Add API to delete spent UTXOs from database

We currently store spent UTXOs in the database
with an `is_spent` field and we don't provide
users with a way to delete these UTXOs. The
issue here is that these could easily bloat the
database or memory. This PR provides methods
that can allow users to delete spent UTXOs
from the database following certain criteria
like size of spent utxos and number of confirmations.
This PR fixes issue #573

src/database/any.rs

@@ -255,6 +255,17 @@ impl Database for AnyDatabase {
  fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error> {
  impl_inner_method!(AnyDatabase, self, increment_last_index, keychain)
  }
+
+ fn del_spent_utxos(&mut self, spent_utxos: Vec<LocalUtxo>) -> Result<Vec<LocalUtxo>, Error> {
+ impl_inner_method!(AnyDatabase, self, del_spent_utxos, spent_utxos)
+ }
+
+ fn del_spent_utxos_by_criteria(
+ &mut self,
+ criteria: DelCriteria,
+ ) -> Result<Vec<LocalUtxo>, Error> {
+ impl_inner_method!(AnyDatabase, self, del_spent_utxos_by_criteria, criteria)
+ }
 }
 
 impl BatchOperations for AnyBatch {

src/database/keyvalue.rs

@@ -18,7 +18,7 @@ use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
 use crate::database::memory::MapKey;
-use crate::database::{BatchDatabase, BatchOperations, Database, SyncTime};
+use crate::database::{BatchDatabase, BatchOperations, Database, DelCriteria, SyncTime};
 use crate::error::Error;
 use crate::types::*;
 
@@ -200,6 +200,29 @@ impl BatchOperations for Batch {
  impl_batch_operations!({}, process_delete_batch);
 }
 
+trait Helpers {
+ fn delete_spent_utxo_set(
+ &mut self,
+ deleted_utxos: &mut Vec<LocalUtxo>,
+ to_del: &[LocalUtxo],
+ ) -> Result<(), Error>;
+}
+
+impl Helpers for Tree {
+ fn delete_spent_utxo_set(
+ &mut self,
+ deleted_utxos: &mut Vec<LocalUtxo>,
+ to_del: &[LocalUtxo],
+ ) -> Result<(), Error> {
+ for spent_utxo in to_del {
+ if let Some(utxo) = self.del_utxo(&spent_utxo.outpoint)? {
+ deleted_utxos.push(utxo);
+ }
+ }
+ Ok(())
+ }
+}
+
 impl Database for Tree {
  fn check_descriptor_checksum<B: AsRef<[u8]>>(
  &mut self,
@@ -379,6 +402,66 @@ impl Database for Tree {
  })?
  .map_or(Ok(0), ivec_to_u32)
  }
+
+ fn del_spent_utxos(&mut self, spent_utxos: Vec<LocalUtxo>) -> Result<Vec<LocalUtxo>, Error> {
+ let mut deleted_utxos = vec![];
+
+ if spent_utxos.is_empty() {
+ let all_spent_utxos: Vec<LocalUtxo> = self
+ .iter_utxos()?
+ .into_iter()
+ .filter(|utxo| utxo.is_spent)
+ .collect();
+ self.delete_spent_utxo_set(&mut deleted_utxos, &all_spent_utxos)?;
+ } else {
+ self.delete_spent_utxo_set(&mut deleted_utxos, &spent_utxos)?;
+ }
+
+ Ok(deleted_utxos)
+ }
+
+ fn del_spent_utxos_by_criteria(
+ &mut self,
+ criteria: DelCriteria,
+ ) -> Result<Vec<LocalUtxo>, Error> {
+ let spent_utxos: Vec<LocalUtxo> = self
+ .iter_utxos()?
+ .into_iter()
+ .filter(|utxo| utxo.is_spent)
+ .collect();
+ let spent_utxo_size = spent_utxos.len();
+ if let Some(threshold_size) = criteria.threshold_size {
+ // if the threshold size is smaller than current size, delete everything
+ if threshold_size < spent_utxo_size as u64 {
+ //delete all spent utxos
+ return self.del_spent_utxos(vec![]);
+ }
+ }
+
+ let mut deleted_utxos: Vec<LocalUtxo> = vec![];
+ match self.get_sync_time()? {
+ Some(sync_time) => {
+ for spent_utxo in spent_utxos {
+ let tx_details = self.get_tx(&spent_utxo.outpoint.txid, false)?;
+ if let Some(tx_details) = tx_details {
+ if let Some(confirmation_time) = tx_details.confirmation_time {
+ if let Some(confirmations) = criteria.confirmations {
+ if (sync_time.block_time.height - confirmation_time.height)
+ < confirmations
+ {
+ //delete it and add to deleted utxos
+ self.del_utxo(&spent_utxo.outpoint)?;
+ deleted_utxos.push(spent_utxo);
+ }
+ }
+ }
+ }
+ }
+ Ok(deleted_utxos)
+ }
+ None => Ok(deleted_utxos),
+ }
+ }
 }
 
 fn ivec_to_u32(b: sled::IVec) -> Result<u32, Error> {

src/database/memory.rs

@@ -22,7 +22,9 @@ use bitcoin::consensus::encode::{deserialize, serialize};
 use bitcoin::hash_types::Txid;
 use bitcoin::{OutPoint, Script, Transaction};
 
-use crate::database::{BatchDatabase, BatchOperations, ConfigurableDatabase, Database, SyncTime};
+use crate::database::{
+ BatchDatabase, BatchOperations, ConfigurableDatabase, Database, DelCriteria, SyncTime,
+};
 use crate::error::Error;
 use crate::types::*;
 
@@ -126,6 +128,45 @@ impl MemoryDatabase {
  deleted_keys: Vec::new(),
  }
  }
+
+ fn select_spent_utxos(&self) -> Vec<LocalUtxo> {
+ let key = MapKey::Utxo(None).as_map_key();
+ self.map
+ .range::<Vec<u8>, _>((Included(&key), Excluded(&after(&key))))
+ .filter_map(|(k, v)| {
+ let outpoint = deserialize(&k[1..]).unwrap();
+ let (txout, keychain, is_spent) = v.downcast_ref().cloned().unwrap();
+ if is_spent {
+ Some(LocalUtxo {
+ outpoint,
+ txout,
+ keychain,
+ is_spent,
+ })
+ } else {
+ None
+ }
+ })
+ .collect()
+ }
+
+ fn delete_spent_utxo_set(
+ &mut self,
+ deleted_utxos: &mut Vec<LocalUtxo>,
+ to_del: &Vec<LocalUtxo>,
+ ) -> Result<(), Error> {
+ for spent_utxo in to_del {
+ if let Some(utxo) = self.del_utxo(&spent_utxo.outpoint)? {
+ deleted_utxos.push(utxo);
+ }
+ }
+
+ Ok(())
+ }
+
+ fn count_spent_utxos(&self) -> usize {
+ self.select_spent_utxos().len()
+ }
 }
 
 impl BatchOperations for MemoryDatabase {
@@ -449,6 +490,54 @@ impl Database for MemoryDatabase {
 
  Ok(*value)
  }
+
+ fn del_spent_utxos(&mut self, spent_utxos: Vec<LocalUtxo>) -> Result<Vec<LocalUtxo>, Error> {
+ let mut deleted_utxos: Vec<LocalUtxo> = vec![];
+ if spent_utxos.is_empty() {
+ let all_spent_utxos: Vec<LocalUtxo> = self.select_spent_utxos();
+ self.delete_spent_utxo_set(&mut deleted_utxos, &all_spent_utxos)?;
+ } else {
+ self.delete_spent_utxo_set(&mut deleted_utxos, &spent_utxos)?;
+ }
+
+ Ok(deleted_utxos)
+ }
+
+ fn del_spent_utxos_by_criteria(
+ &mut self,
+ criteria: DelCriteria,
+ ) -> Result<Vec<LocalUtxo>, Error> {
+ let spent_utxo_size = self.count_spent_utxos();
+ if let Some(threshold_size) = criteria.threshold_size {
+ if threshold_size < spent_utxo_size as u64 {
+ return self.del_spent_utxos(vec![]);
+ }
+ }
+
+ let mut deleted_utxos: Vec<LocalUtxo> = vec![];
+ match self.get_sync_time()? {
+ Some(sync_time) => {
+ let spent_utxos = self.select_spent_utxos();
+ for spent_utxo in spent_utxos {
+ let tx_details = self.get_tx(&spent_utxo.outpoint.txid, false)?;
+ if let Some(tx_details) = tx_details {
+ if let Some(confirmation_time) = tx_details.confirmation_time {
+ if let Some(confirmations) = criteria.confirmations {
+ if (sync_time.block_time.height - confirmation_time.height)
+ < confirmations
+ {
+ self.del_utxo(&spent_utxo.outpoint)?;
+ deleted_utxos.push(spent_utxo);
+ }
+ }
+ }
+ }
+ }
+ Ok(deleted_utxos)
+ }
+ None => Ok(deleted_utxos),
+ }
+ }
 }
 
 impl BatchDatabase for MemoryDatabase {

src/database/mod.rs

@@ -55,6 +55,18 @@ pub struct SyncTime {
  pub block_time: BlockTime,
 }
 
+/// Structure encapsulates criteria used for deleting
+/// spent UTXOs
+#[derive(Clone, Debug)]
+pub struct DelCriteria {
+ /// Number of Confirmation on UTXOs to
+ /// make it eligible for deletion
+ pub confirmations: Option<u32>,
+ /// Max number of allowable spent
+ /// UTXOs in the database
+ pub threshold_size: Option<u64>,
+}
+
 /// Trait for operations that can be batched
 ///
 /// This trait defines the list of operations that must be implemented on the [`Database`] type and
@@ -158,6 +170,17 @@ pub trait Database: BatchOperations {
  ///
  /// It should insert and return `0` if not present in the database
  fn increment_last_index(&mut self, keychain: KeychainKind) -> Result<u32, Error>;
+
+ /// Delete UTXOs provided as params, if `spent_utxos` is empty it deletes all spent
+ /// UTXOs in the database.
+ fn del_spent_utxos(&mut self, spent_utxos: Vec<LocalUtxo>) -> Result<Vec<LocalUtxo>, Error>;
+
+ /// Delete UTXOs based on the number of confirmations or
+ /// threshold size (number of spent utxos in DB) defined in [`DelCriteria`]
+ fn del_spent_utxos_by_criteria(
+ &mut self,
+ criteria: DelCriteria,
+ ) -> Result<Vec<LocalUtxo>, Error>;
 }
 
 /// Trait for a database that supports batch operations