Enhance Mempool performance

.gitignore

@@ -10,3 +10,4 @@ analyzer-target
 .DS_Store
 *.code-workspace
 /setup
+testing/integration/perflogs*

consensus/core/src/api/mod.rs

@@ -4,7 +4,7 @@ use std::sync::Arc;
 
 use crate::{
     acceptance_data::AcceptanceData,
-    block::{Block, BlockTemplate},
+    block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector},
     block_count::BlockCount,
     blockstatus::BlockStatus,
     coinbase::MinerData,
@@ -27,7 +27,12 @@ pub type BlockValidationFuture = BoxFuture<'static, BlockProcessResult<BlockStat
 /// Abstracts the consensus external API
 #[allow(unused_variables)]
 pub trait ConsensusApi: Send + Sync {
-    fn build_block_template(&self, miner_data: MinerData, txs: Vec<Transaction>) -> Result<BlockTemplate, RuleError> {
+    fn build_block_template(
+        &self,
+        miner_data: MinerData,
+        tx_selector: Box<dyn TemplateTransactionSelector>,
+        build_mode: TemplateBuildMode,
+    ) -> Result<BlockTemplate, RuleError> {
         unimplemented!()
     }
 
@@ -40,8 +45,24 @@ pub trait ConsensusApi: Send + Sync {
     }
 
     /// Populates the mempool transaction with maximally found UTXO entry data and proceeds to full transaction
-    /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated
-    fn validate_mempool_transaction_and_populate(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
+    /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated.
+    fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
+        unimplemented!()
+    }
+
+    /// Populates the mempool transactions with maximally found UTXO entry data and proceeds to full transactions
+    /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated.
+    fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec<TxResult<()>> {
+        unimplemented!()
+    }
+
+    /// Populates the mempool transaction with maximally found UTXO entry data.
+    fn populate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
+        unimplemented!()
+    }
+
+    /// Populates the mempool transactions with maximally found UTXO entry data.
+    fn populate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec<TxResult<()>> {
         unimplemented!()
     }
 

consensus/core/src/block.rs

@@ -1,6 +1,10 @@
 use std::sync::Arc;
 
-use crate::{coinbase::MinerData, header::Header, tx::Transaction};
+use crate::{
+    coinbase::MinerData,
+    header::Header,
+    tx::{Transaction, TransactionId},
+};
 use kaspa_hashes::Hash;
 
 /// A mutable block structure where header and transactions within can still be mutated.
@@ -64,6 +68,34 @@ impl Block {
     }
 }
 
+/// An abstraction for a recallable transaction selector with persistent state
+pub trait TemplateTransactionSelector {
+    /// Expected to return a batch of transactions which were not previously selected.
+    /// The batch will typically contain sufficient transactions to fill the block
+    /// mass (along with the previously unrejected txs), or will drain the selector    
+    fn select_transactions(&mut self) -> Vec<Transaction>;
+
+    /// Should be used to report invalid transactions obtained from the *most recent*
+    /// `select_transactions` call. Implementors should use this call to internally
+    /// track the selection state and discard the rejected tx from internal occupation calculations
+    fn reject_selection(&mut self, tx_id: TransactionId);
+
+    /// Determine whether this was an overall successful selection episode
+    fn is_successful(&self) -> bool;
+}
+
+/// Block template build mode
+#[derive(Clone, Copy, Debug)]
+pub enum TemplateBuildMode {
+    /// Block template build can possibly fail if `TemplateTransactionSelector::is_successful` deems the operation unsuccessful.
+    ///
+    /// In such a case, the build fails with `BlockRuleError::InvalidTransactionsInNewBlock`.
+    Standard,
+
+    /// Block template build always succeeds. The built block contains only the validated transactions.
+    Infallible,
+}
+
 /// A block template for miners.
 #[derive(Debug, Clone)]
 pub struct BlockTemplate {

consensus/core/src/config/mod.rs

@@ -61,6 +61,9 @@ pub struct Config {
 
     #[cfg(feature = "devnet-prealloc")]
     pub initial_utxo_set: Arc<UtxoCollection>,
+
+    #[cfg(feature = "devnet-prealloc")]
+    pub block_template_cache_lifetime: Option<u64>,
 }
 
 impl Config {
@@ -85,6 +88,8 @@ impl Config {
 
             #[cfg(feature = "devnet-prealloc")]
             initial_utxo_set: Default::default(),
+            #[cfg(feature = "devnet-prealloc")]
+            block_template_cache_lifetime: None,
         }
     }
 

consensus/core/src/errors/block.rs

@@ -1,4 +1,4 @@
-use std::fmt::Display;
+use std::{collections::HashMap, fmt::Display};
 
 use crate::{
     constants,
@@ -140,7 +140,7 @@ pub enum RuleError {
     InvalidTransactionsInUtxoContext(usize, usize),
 
     #[error("invalid transactions in new block template")]
-    InvalidTransactionsInNewBlock(Vec<(TransactionId, TxRuleError)>),
+    InvalidTransactionsInNewBlock(HashMap<TransactionId, TxRuleError>),
 
     #[error("DAA window data has only {0} entries")]
     InsufficientDaaWindowSize(usize),

consensus/core/src/tx.rs

@@ -340,6 +340,12 @@ impl<T: AsRef<Transaction>> MutableTransaction<T> {
     }
 }
 
+impl<T: AsRef<Transaction>> AsRef<Transaction> for MutableTransaction<T> {
+    fn as_ref(&self) -> &Transaction {
+        self.tx.as_ref()
+    }
+}
+
 /// Private struct used to wrap a [`MutableTransaction`] as a [`VerifiableTransaction`]
 struct MutableTransactionVerifiableWrapper<'a, T: AsRef<Transaction>> {
     inner: &'a MutableTransaction<T>,

consensus/src/consensus/mod.rs

@@ -41,7 +41,7 @@ use crate::{
 use kaspa_consensus_core::{
     acceptance_data::AcceptanceData,
     api::{BlockValidationFuture, ConsensusApi},
-    block::{Block, BlockTemplate},
+    block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector},
     block_count::BlockCount,
     blockhash::BlockHashExtensions,
     blockstatus::BlockStatus,
@@ -353,8 +353,13 @@ impl Consensus {
 }
 
 impl ConsensusApi for Consensus {
-    fn build_block_template(&self, miner_data: MinerData, txs: Vec<Transaction>) -> Result<BlockTemplate, RuleError> {
-        self.virtual_processor.build_block_template(miner_data, txs)
+    fn build_block_template(
+        &self,
+        miner_data: MinerData,
+        tx_selector: Box<dyn TemplateTransactionSelector>,
+        build_mode: TemplateBuildMode,
+    ) -> Result<BlockTemplate, RuleError> {
+        self.virtual_processor.build_block_template(miner_data, tx_selector, build_mode)
     }
 
     fn validate_and_insert_block(&self, block: Block) -> BlockValidationFuture {
@@ -367,11 +372,24 @@ impl ConsensusApi for Consensus {
         Box::pin(result)
     }
 
-    fn validate_mempool_transaction_and_populate(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
-        self.virtual_processor.validate_mempool_transaction_and_populate(transaction)?;
+    fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
+        self.virtual_processor.validate_mempool_transaction(transaction)?;
+        Ok(())
+    }
+
+    fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec<TxResult<()>> {
+        self.virtual_processor.validate_mempool_transactions_in_parallel(transactions)
+    }
+
+    fn populate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> {
+        self.virtual_processor.populate_mempool_transaction(transaction)?;
         Ok(())
     }
 
+    fn populate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec<TxResult<()>> {
+        self.virtual_processor.populate_mempool_transactions_in_parallel(transactions)
+    }
+
     fn calculate_transaction_mass(&self, transaction: &Transaction) -> u64 {
         self.services.mass_calculator.calc_tx_mass(transaction)
     }