From 539af9cc8f82d0adedf4c03763830a5c624b224d Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sun, 18 Aug 2024 20:10:09 +0200 Subject: [PATCH 01/48] fix deprecated `IdbIndexParameters::unique` method warnings --- .../core/src/storage/local/transaction/indexdb.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/wallet/core/src/storage/local/transaction/indexdb.rs b/wallet/core/src/storage/local/transaction/indexdb.rs index e4190aa..232db46 100644 --- a/wallet/core/src/storage/local/transaction/indexdb.rs +++ b/wallet/core/src/storage/local/transaction/indexdb.rs @@ -33,20 +33,29 @@ impl Inner { // Check if the object store exists; create it if it doesn't if !evt.db().object_store_names().any(|n| n == TRANSACTIONS_STORE_NAME) { let object_store = evt.db().create_object_store(TRANSACTIONS_STORE_NAME)?; + + let id_index_params = IdbIndexParameters::new(); + id_index_params.set_unique(true); object_store.create_index_with_params( TRANSACTIONS_STORE_ID_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_ID_INDEX), - IdbIndexParameters::new().unique(true), + &id_index_params, )?; + + let timestamp_index_params = IdbIndexParameters::new(); + timestamp_index_params.set_unique(false); object_store.create_index_with_params( TRANSACTIONS_STORE_TIMESTAMP_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_TIMESTAMP_INDEX), - IdbIndexParameters::new().unique(false), + ×tamp_index_params, )?; + + let data_index_params = IdbIndexParameters::new(); + data_index_params.set_unique(false); object_store.create_index_with_params( TRANSACTIONS_STORE_DATA_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_DATA_INDEX), - IdbIndexParameters::new().unique(false), + &data_index_params, )?; } Ok(()) From 52f91471a90f27b68a9a66f585e654bdb5830d76 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sun, 18 Aug 2024 20:24:44 +0200 Subject: [PATCH 02/48] remove unnecessary conditions in `Matrix::compute_rank` --- consensus/pow/src/matrix.rs | 40 ++++++++++++++++--------------------- 1 file changed, 17 insertions(+), 23 deletions(-) diff --git a/consensus/pow/src/matrix.rs b/consensus/pow/src/matrix.rs index 698947a..1f3efe8 100644 --- a/consensus/pow/src/matrix.rs +++ b/consensus/pow/src/matrix.rs @@ -51,16 +51,15 @@ impl Matrix { #[inline(always)] fn convert_to_float(&self) -> [[f64; 64]; 64] { - // SAFETY: An uninitialized MaybeUninit is always safe. - let mut out: [[MaybeUninit; 64]; 64] = unsafe { MaybeUninit::uninit().assume_init() }; + let mut out: [[f64; 64]; 64] = [[Default::default(); 64]; 64]; out.iter_mut().zip(self.0.iter()).for_each(|(out_row, mat_row)| { out_row.iter_mut().zip(mat_row).for_each(|(out_element, &element)| { - out_element.write(f64::from(element)); + *out_element = f64::from(element); }) }); - // SAFETY: The loop above wrote into all indexes. - unsafe { std::mem::transmute(out) } + + out } pub fn compute_rank(&self) -> usize { @@ -69,30 +68,25 @@ impl Matrix { let mut rank = 0; let mut row_selected = [false; 64]; for i in 0..64 { - if i >= 64 { - // Required for optimization, See https://github.com/rust-lang/rust/issues/90794 - unreachable!() - } let mut j = 0; while j < 64 { if !row_selected[j] && mat_float[j][i].abs() > EPS { - break; - } - j += 1; - } - if j != 64 { - rank += 1; - row_selected[j] = true; - for p in (i + 1)..64 { - mat_float[j][p] /= mat_float[j][i]; - } - for k in 0..64 { - if k != j && mat_float[k][i].abs() > EPS { - for p in (i + 1)..64 { - mat_float[k][p] -= mat_float[j][p] * mat_float[k][i]; + rank += 1; + row_selected[j] = true; + for p in (i + 1)..64 { + mat_float[j][p] /= mat_float[j][i]; + } + for k in 0..64 { + if k != j && mat_float[k][i].abs() > EPS { + for p in (i + 1)..64 { + mat_float[k][p] -= mat_float[j][p] * mat_float[k][i]; + } } } + + break; } + j += 1; } } rank From 87df80ac1062ccded478b848f58eaedbd9a9c7a5 Mon Sep 17 00:00:00 2001 From: 0xA001113 <0xeuler@proton.me> Date: Thu, 17 Oct 2024 10:15:00 +0200 Subject: [PATCH 03/48] TN and TN11 DNS seeders --- consensus/core/src/config/params.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/consensus/core/src/config/params.rs b/consensus/core/src/config/params.rs index 59b04d4..219176e 100644 --- a/consensus/core/src/config/params.rs +++ b/consensus/core/src/config/params.rs @@ -346,7 +346,11 @@ pub const MAINNET_PARAMS: Params = Params { }; pub const TESTNET_PARAMS: Params = Params { - dns_seeders: &[], + dns_seeders: &[ + // Official DNS seeders. + "testnet-dnsseed-1.spectre-network.org", + "testnet-dnsseed-2.spectre-network.org", + ], net: NetworkId::with_suffix(NetworkType::Testnet, 10), genesis: TESTNET_GENESIS, ghostdag_k: LEGACY_DEFAULT_GHOSTDAG_K, @@ -400,7 +404,11 @@ pub const TESTNET_PARAMS: Params = Params { }; pub const TESTNET11_PARAMS: Params = Params { - dns_seeders: &[], + dns_seeders: &[ + // Official DNS seeders. + "testnet11-dnsseed-1.spectre-network.org", + "testnet11-dnsseed-2.spectre-network.org", + ], net: NetworkId::with_suffix(NetworkType::Testnet, 11), genesis: TESTNET11_GENESIS, legacy_timestamp_deviation_tolerance: LEGACY_TIMESTAMP_DEVIATION_TOLERANCE, From 1b0956d04607b7d769c17bf4f55d3f11433c42de Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 01:53:47 +0200 Subject: [PATCH 04/48] replace by fee on mempool --- consensus/core/src/api/args.rs | 47 + consensus/core/src/api/mod.rs | 14 +- consensus/core/src/errors/tx.rs | 5 + consensus/core/src/tx.rs | 13 + consensus/src/consensus/mod.rs | 21 +- .../pipeline/virtual_processor/processor.rs | 22 +- .../virtual_processor/utxo_validation.rs | 6 +- .../transaction_validator_populated.rs | 21 +- crypto/txscript/src/standard.rs | 42 +- mining/errors/src/mempool.rs | 12 +- mining/src/block_template/selector.rs | 6 +- mining/src/manager.rs | 145 +++- mining/src/manager_tests.rs | 815 ++++++++++++++---- mining/src/mempool/mod.rs | 48 ++ mining/src/mempool/model/transactions_pool.rs | 22 +- mining/src/mempool/model/tx.rs | 53 +- mining/src/mempool/model/utxo_set.rs | 32 +- .../populate_entries_and_try_validate.rs | 20 +- mining/src/mempool/replace_by_fee.rs | 149 ++++ .../validate_and_insert_transaction.rs | 47 +- mining/src/model/mod.rs | 1 + mining/src/model/tx_insert.rs | 14 + mining/src/testutils/consensus_mock.rs | 17 +- protocol/flows/src/flow_context.rs | 40 +- protocol/flows/src/v5/txrelay/flow.rs | 4 +- rothschild/src/main.rs | 4 +- rpc/core/src/api/ops.rs | 3 + rpc/core/src/api/rpc.rs | 11 + rpc/core/src/model/message.rs | 25 + rpc/grpc/client/src/lib.rs | 1 + rpc/grpc/core/proto/messages.proto | 6 +- rpc/grpc/core/proto/rpc.proto | 21 +- rpc/grpc/core/src/convert/message.rs | 27 + rpc/grpc/core/src/convert/spectred.rs | 2 + rpc/grpc/core/src/ops.rs | 1 + .../server/src/request_handler/factory.rs | 1 + rpc/grpc/server/src/tests/rpc_core_mock.rs | 7 + rpc/service/src/service.rs | 16 + rpc/wrpc/client/src/client.rs | 1 + rpc/wrpc/server/src/router.rs | 1 + testing/integration/src/common/utils.rs | 4 +- testing/integration/src/rpc_tests.rs | 11 + wallet/core/src/tests/rpc_core_mock.rs | 7 + 43 files changed, 1458 insertions(+), 307 deletions(-) create mode 100644 consensus/core/src/api/args.rs create mode 100644 mining/src/mempool/replace_by_fee.rs create mode 100644 mining/src/model/tx_insert.rs diff --git a/consensus/core/src/api/args.rs b/consensus/core/src/api/args.rs new file mode 100644 index 0000000..ebc76d9 --- /dev/null +++ b/consensus/core/src/api/args.rs @@ -0,0 +1,47 @@ +use std::collections::HashMap; + +use crate::tx::TransactionId; + +/// A struct provided to consensus for transaction validation processing calls +#[derive(Clone, Debug, Default)] +pub struct TransactionValidationArgs { + /// Optional fee/mass threshold above which a bound transaction in not rejected + pub feerate_threshold: Option, +} + +impl TransactionValidationArgs { + pub fn new(feerate_threshold: Option) -> Self { + Self { feerate_threshold } + } +} + +/// A struct provided to consensus for transactions validation batch processing calls +pub struct TransactionValidationBatchArgs { + tx_args: HashMap, +} + +impl TransactionValidationBatchArgs { + const DEFAULT_ARGS: TransactionValidationArgs = TransactionValidationArgs { feerate_threshold: None }; + + pub fn new() -> Self { + Self { tx_args: HashMap::new() } + } + + /// Set some fee/mass threshold for transaction `transaction_id`. + pub fn set_feerate_threshold(&mut self, transaction_id: TransactionId, feerate_threshold: f64) { + self.tx_args + .entry(transaction_id) + .and_modify(|x| x.feerate_threshold = Some(feerate_threshold)) + .or_insert(TransactionValidationArgs::new(Some(feerate_threshold))); + } + + pub fn get(&self, transaction_id: &TransactionId) -> &TransactionValidationArgs { + self.tx_args.get(transaction_id).unwrap_or(&Self::DEFAULT_ARGS) + } +} + +impl Default for TransactionValidationBatchArgs { + fn default() -> Self { + Self::new() + } +} diff --git a/consensus/core/src/api/mod.rs b/consensus/core/src/api/mod.rs index e347158..99ae2be 100644 --- a/consensus/core/src/api/mod.rs +++ b/consensus/core/src/api/mod.rs @@ -4,6 +4,7 @@ use std::sync::Arc; use crate::{ acceptance_data::AcceptanceData, + api::args::{TransactionValidationArgs, TransactionValidationBatchArgs}, block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, blockstatus::BlockStatus, coinbase::MinerData, @@ -25,6 +26,7 @@ use spectre_hashes::Hash; pub use self::stats::{BlockCount, ConsensusStats}; +pub mod args; pub mod counters; pub mod stats; @@ -62,14 +64,18 @@ pub trait ConsensusApi: Send + Sync { } /// Populates the mempool transaction with maximally found UTXO entry data and proceeds to full transaction - /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated. - fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { + /// validation if all are found. If validation is successful, also `transaction.calculated_fee` is expected to be populated. + fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { unimplemented!() } /// Populates the mempool transactions with maximally found UTXO entry data and proceeds to full transactions - /// validation if all are found. If validation is successful, also [`transaction.calculated_fee`] is expected to be populated. - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { + /// validation if all are found. If validation is successful, also `transaction.calculated_fee` is expected to be populated. + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { unimplemented!() } diff --git a/consensus/core/src/errors/tx.rs b/consensus/core/src/errors/tx.rs index 339e11a..3f41b69 100644 --- a/consensus/core/src/errors/tx.rs +++ b/consensus/core/src/errors/tx.rs @@ -88,6 +88,11 @@ pub enum TxRuleError { #[error("calculated contextual mass (including storage mass) {0} is not equal to the committed mass field {1}")] WrongMass(u64, u64), + + /// [`TxRuleError::FeerateTooLow`] is not a consensus error but a mempool error triggered by the + /// fee/mass RBF validation rule + #[error("fee rate per contextual mass gram is not greater than the fee rate of the replaced transaction")] + FeerateTooLow, } pub type TxResult = std::result::Result; diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 28a0d0d..64fd83f 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -406,6 +406,19 @@ impl> MutableTransaction { *entry = None; } } + + /// Returns the calculated feerate. The feerate is calculated as the amount of fee + /// this transactions pays per gram of the full contextual (compute & storage) mass. The + /// function returns a value when calculated fee exists and the contextual mass is greater + /// than zero, otherwise `None` is returned. + pub fn calculated_feerate(&self) -> Option { + let contextual_mass = self.tx.as_ref().mass(); + if contextual_mass > 0 { + self.calculated_fee.map(|fee| fee as f64 / contextual_mass as f64) + } else { + None + } + } } impl> AsRef for MutableTransaction { diff --git a/consensus/src/consensus/mod.rs b/consensus/src/consensus/mod.rs index 7964d78..328b856 100644 --- a/consensus/src/consensus/mod.rs +++ b/consensus/src/consensus/mod.rs @@ -40,7 +40,11 @@ use crate::{ }; use spectre_consensus_core::{ acceptance_data::AcceptanceData, - api::{stats::BlockCount, BlockValidationFutures, ConsensusApi, ConsensusStats}, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + stats::BlockCount, + BlockValidationFutures, ConsensusApi, ConsensusStats, + }, block::{Block, BlockTemplate, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, blockhash::BlockHashExtensions, blockstatus::BlockStatus, @@ -49,9 +53,10 @@ use spectre_consensus_core::{ errors::{ coinbase::CoinbaseResult, consensus::{ConsensusError, ConsensusResult}, + difficulty::DifficultyError, + pruning::PruningImportError, tx::TxResult, }, - errors::{difficulty::DifficultyError, pruning::PruningImportError}, header::Header, muhash::MuHashExtensions, network::NetworkType, @@ -419,13 +424,17 @@ impl ConsensusApi for Consensus { BlockValidationFutures { block_task: Box::pin(block_task), virtual_state_task: Box::pin(virtual_state_task) } } - fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { - self.virtual_processor.validate_mempool_transaction(transaction)?; + fn validate_mempool_transaction(&self, transaction: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { + self.virtual_processor.validate_mempool_transaction(transaction, args)?; Ok(()) } - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - self.virtual_processor.validate_mempool_transactions_in_parallel(transactions) + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { + self.virtual_processor.validate_mempool_transactions_in_parallel(transactions, args) } fn populate_mempool_transaction(&self, transaction: &mut MutableTransaction) -> TxResult<()> { diff --git a/consensus/src/pipeline/virtual_processor/processor.rs b/consensus/src/pipeline/virtual_processor/processor.rs index ea099a5..bccdd6c 100644 --- a/consensus/src/pipeline/virtual_processor/processor.rs +++ b/consensus/src/pipeline/virtual_processor/processor.rs @@ -48,6 +48,7 @@ use crate::{ }; use spectre_consensus_core::{ acceptance_data::AcceptanceData, + api::args::{TransactionValidationArgs, TransactionValidationBatchArgs}, block::{BlockTemplate, MutableBlock, TemplateBuildMode, TemplateTransactionSelector}, blockstatus::BlockStatus::{StatusDisqualifiedFromChain, StatusUTXOValid}, coinbase::MinerData, @@ -757,23 +758,28 @@ impl VirtualStateProcessor { virtual_utxo_view: &impl UtxoView, virtual_daa_score: u64, virtual_past_median_time: u64, + args: &TransactionValidationArgs, ) -> TxResult<()> { self.transaction_validator.validate_tx_in_isolation(&mutable_tx.tx)?; self.transaction_validator.utxo_free_tx_validation(&mutable_tx.tx, virtual_daa_score, virtual_past_median_time)?; - self.validate_mempool_transaction_in_utxo_context(mutable_tx, virtual_utxo_view, virtual_daa_score)?; + self.validate_mempool_transaction_in_utxo_context(mutable_tx, virtual_utxo_view, virtual_daa_score, args)?; Ok(()) } - pub fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction) -> TxResult<()> { + pub fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction, args: &TransactionValidationArgs) -> TxResult<()> { let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; let virtual_daa_score = virtual_state.daa_score; let virtual_past_median_time = virtual_state.past_median_time; - self.validate_mempool_transaction_impl(mutable_tx, virtual_utxo_view, virtual_daa_score, virtual_past_median_time) + self.validate_mempool_transaction_impl(mutable_tx, virtual_utxo_view, virtual_daa_score, virtual_past_median_time, args) } - pub fn validate_mempool_transactions_in_parallel(&self, mutable_txs: &mut [MutableTransaction]) -> Vec> { + pub fn validate_mempool_transactions_in_parallel( + &self, + mutable_txs: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, + ) -> Vec> { let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; @@ -784,7 +790,13 @@ impl VirtualStateProcessor { mutable_txs .par_iter_mut() .map(|mtx| { - self.validate_mempool_transaction_impl(mtx, &virtual_utxo_view, virtual_daa_score, virtual_past_median_time) + self.validate_mempool_transaction_impl( + mtx, + &virtual_utxo_view, + virtual_daa_score, + virtual_past_median_time, + args.get(&mtx.id()), + ) }) .collect::>>() }) diff --git a/consensus/src/pipeline/virtual_processor/utxo_validation.rs b/consensus/src/pipeline/virtual_processor/utxo_validation.rs index 9ab7055..a3f8815 100644 --- a/consensus/src/pipeline/virtual_processor/utxo_validation.rs +++ b/consensus/src/pipeline/virtual_processor/utxo_validation.rs @@ -15,6 +15,7 @@ use crate::{ }; use spectre_consensus_core::{ acceptance_data::{AcceptedTxEntry, MergesetBlockAcceptanceData}, + api::args::TransactionValidationArgs, coinbase::*, hashing, header::Header, @@ -248,7 +249,7 @@ impl VirtualStateProcessor { } } let populated_tx = PopulatedTransaction::new(transaction, entries); - let res = self.transaction_validator.validate_populated_transaction_and_get_fee(&populated_tx, pov_daa_score, flags); + let res = self.transaction_validator.validate_populated_transaction_and_get_fee(&populated_tx, pov_daa_score, flags, None); match res { Ok(calculated_fee) => Ok(ValidatedTransaction::new(populated_tx, calculated_fee)), Err(tx_rule_error) => { @@ -290,6 +291,7 @@ impl VirtualStateProcessor { mutable_tx: &mut MutableTransaction, utxo_view: &impl UtxoView, pov_daa_score: u64, + args: &TransactionValidationArgs, ) -> TxResult<()> { self.populate_mempool_transaction_in_utxo_context(mutable_tx, utxo_view)?; @@ -308,10 +310,12 @@ impl VirtualStateProcessor { mutable_tx.tx.set_mass(contextual_mass); // At this point we know all UTXO entries are populated, so we can safely pass the tx as verifiable + let mass_and_feerate_threshold = args.feerate_threshold.map(|threshold| (contextual_mass, threshold)); let calculated_fee = self.transaction_validator.validate_populated_transaction_and_get_fee( &mutable_tx.as_verifiable(), pov_daa_score, TxValidationFlags::SkipMassCheck, // we can skip the mass check since we just set it + mass_and_feerate_threshold, )?; mutable_tx.calculated_fee = Some(calculated_fee); Ok(()) diff --git a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs index a6ecaf4..16ed8cc 100644 --- a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs +++ b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs @@ -27,10 +27,12 @@ impl TransactionValidator { tx: &impl VerifiableTransaction, pov_daa_score: u64, flags: TxValidationFlags, + mass_and_feerate_threshold: Option<(u64, f64)>, ) -> TxResult { self.check_transaction_coinbase_maturity(tx, pov_daa_score)?; let total_in = self.check_transaction_input_amounts(tx)?; let total_out = Self::check_transaction_output_values(tx, total_in)?; + let fee = total_in - total_out; if flags != TxValidationFlags::SkipMassCheck && pov_daa_score > self.storage_mass_activation_daa_score { // Storage mass hardfork was activated self.check_mass_commitment(tx)?; @@ -40,6 +42,11 @@ impl TransactionValidator { } } Self::check_sequence_lock(tx, pov_daa_score)?; + + // The following call is not a consensus check (it could not be one in the first place since it uses floating number) + // but rather a mempool Replace by Fee validation rule. It was placed here purposely for avoiding unneeded script checks. + Self::check_feerate_threshold(fee, mass_and_feerate_threshold)?; + match flags { TxValidationFlags::Full | TxValidationFlags::SkipMassCheck => { Self::check_sig_op_counts(tx)?; @@ -47,7 +54,19 @@ impl TransactionValidator { } TxValidationFlags::SkipScriptChecks => {} } - Ok(total_in - total_out) + Ok(fee) + } + + fn check_feerate_threshold(fee: u64, mass_and_feerate_threshold: Option<(u64, f64)>) -> TxResult<()> { + // An actual check can only occur if some mass and threshold are provided, + // otherwise, the check does not verify anything and exits successfully. + if let Some((contextual_mass, feerate_threshold)) = mass_and_feerate_threshold { + assert!(contextual_mass > 0); + if fee as f64 / contextual_mass as f64 <= feerate_threshold { + return Err(TxRuleError::FeerateTooLow); + } + } + Ok(()) } fn check_transaction_coinbase_maturity(&self, tx: &impl VerifiableTransaction, pov_daa_score: u64) -> TxResult<()> { diff --git a/crypto/txscript/src/standard.rs b/crypto/txscript/src/standard.rs index 4ea4e40..bdb011d 100644 --- a/crypto/txscript/src/standard.rs +++ b/crypto/txscript/src/standard.rs @@ -100,9 +100,9 @@ pub mod test_helpers { (script_public_key, redeem_script) } - // Creates a transaction that spends the first output of provided transaction. - // Assumes that the output being spent has opTrueScript as it's scriptPublicKey. - // Creates the value of the spent output minus provided `fee` (in sompi). + /// Creates a transaction that spends the first output of provided transaction. + /// Assumes that the output being spent has opTrueScript as its scriptPublicKey. + /// Creates the value of the spent output minus provided `fee` (in sompi). pub fn create_transaction(tx_to_spend: &Transaction, fee: u64) -> Transaction { let (script_public_key, redeem_script) = op_true_script(); let signature_script = pay_to_script_hash_signature_script(redeem_script, vec![]).expect("the script is canonical"); @@ -111,6 +111,42 @@ pub mod test_helpers { let output = TransactionOutput::new(tx_to_spend.outputs[0].value - fee, script_public_key); Transaction::new(TX_VERSION, vec![input], vec![output], 0, SUBNETWORK_ID_NATIVE, 0, vec![]) } + + /// Creates a transaction that spends the outputs of specified indexes (if they exist) of every provided transaction and returns an optional change. + /// Assumes that the outputs being spent have opTrueScript as their scriptPublicKey. + /// + /// If some change is provided, creates two outputs, first one with the value of the spent outputs minus `change` + /// and `fee` (in sompi) and second one of `change` amount. + /// + /// If no change is provided, creates only one output with the value of the spent outputs minus and `fee` (in sompi) + pub fn create_transaction_with_change<'a>( + txs_to_spend: impl Iterator, + output_indexes: Vec, + change: Option, + fee: u64, + ) -> Transaction { + let (script_public_key, redeem_script) = op_true_script(); + let signature_script = pay_to_script_hash_signature_script(redeem_script, vec![]).expect("the script is canonical"); + let mut inputs_value: u64 = 0; + let mut inputs = vec![]; + for tx_to_spend in txs_to_spend { + for i in output_indexes.iter().copied() { + if i < tx_to_spend.outputs.len() { + let previous_outpoint = TransactionOutpoint::new(tx_to_spend.id(), i as u32); + inputs.push(TransactionInput::new(previous_outpoint, signature_script.clone(), MAX_TX_IN_SEQUENCE_NUM, 1)); + inputs_value += tx_to_spend.outputs[i].value; + } + } + } + let outputs = match change { + Some(change) => vec![ + TransactionOutput::new(inputs_value - fee - change, script_public_key.clone()), + TransactionOutput::new(change, script_public_key), + ], + None => vec![TransactionOutput::new(inputs_value - fee, script_public_key.clone())], + }; + Transaction::new(TX_VERSION, inputs, outputs, 0, SUBNETWORK_ID_NATIVE, 0, vec![]) + } } #[cfg(test)] diff --git a/mining/errors/src/mempool.rs b/mining/errors/src/mempool.rs index 9a194b5..08d6649 100644 --- a/mining/errors/src/mempool.rs +++ b/mining/errors/src/mempool.rs @@ -4,7 +4,7 @@ use spectre_consensus_core::{ }; use thiserror::Error; -#[derive(Error, Debug, Clone)] +#[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum RuleError { /// A consensus transaction rule error /// @@ -24,9 +24,15 @@ pub enum RuleError { #[error("transaction {0} is already in the mempool")] RejectDuplicate(TransactionId), - #[error("output {0} already spent by transaction {1} in the memory pool")] + #[error("output {0} already spent by transaction {1} in the mempool")] RejectDoubleSpendInMempool(TransactionOutpoint, TransactionId), + #[error("replace by fee found no double spending transaction in the mempool")] + RejectRbfNoDoubleSpend, + + #[error("replace by fee found more than one double spending transaction in the mempool")] + RejectRbfTooManyDoubleSpendingTransactions, + /// New behavior: a transaction is rejected if the mempool is full #[error("number of high-priority transactions in mempool ({0}) has reached the maximum allowed ({1})")] RejectMempoolIsFull(usize, u64), @@ -95,7 +101,7 @@ impl From for RuleError { pub type RuleResult = std::result::Result; -#[derive(Error, Debug, Clone)] +#[derive(Error, Debug, Clone, PartialEq, Eq)] pub enum NonStandardError { #[error("transaction version {1} is not in the valid range of {2}-{3}")] RejectVersion(TransactionId, u16, u16, u16), diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index 602d56f..37b6b3a 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -182,11 +182,7 @@ impl TransactionsSelector { self.total_mass += selected_tx.calculated_mass; self.total_fees += selected_tx.calculated_fee; - trace!( - "Adding tx {0} (fee per megagram: {1})", - selected_tx.tx.id(), - selected_tx.calculated_fee * 1_000_000 / selected_tx.calculated_mass - ); + trace!("Adding tx {0} (fee per gram: {1})", selected_tx.tx.id(), selected_tx.calculated_fee / selected_tx.calculated_mass); // Mark for deletion selected_candidate.is_marked_for_deletion = true; diff --git a/mining/src/manager.rs b/mining/src/manager.rs index 34eeb8c..64de72c 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -4,17 +4,18 @@ use crate::{ errors::MiningManagerResult, mempool::{ config::Config, - model::tx::{MempoolTransaction, TxRemovalReason}, + model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, populate_entries_and_try_validate::{ populate_mempool_transactions_in_parallel, validate_mempool_transaction, validate_mempool_transactions_in_parallel, }, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, Mempool, }, model::{ candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, topological_sort::IntoIterTopologically, + tx_insert::TransactionInsertion, tx_query::TransactionQuery, }, MempoolCountersSnapshot, MiningCounters, P2pTxCountSample, @@ -22,7 +23,10 @@ use crate::{ use itertools::Itertools; use parking_lot::RwLock; use spectre_consensus_core::{ - api::ConsensusApi, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, block::{BlockTemplate, TemplateBuildMode}, coinbase::MinerData, errors::{block::RuleError as BlockRuleError, tx::TxRuleError}, @@ -172,6 +176,9 @@ impl MiningManager { if let Err(err) = removal_result { // Original golang comment: // mempool.remove_transactions might return errors in situations that are perfectly fine in this context. + // TODO: Once the mempool invariants are clear, this might return an error: + // https://github.com/spectrenet/spectred/issues/1553 + // NOTE: unlike golang, here we continue removing also if an error was found error!("Error from mempool.remove_transactions: {:?}", err); } }); @@ -209,47 +216,58 @@ impl MiningManager { /// adds it to the set of known transactions that have not yet been /// added to any block. /// - /// The returned transactions are clones of objects owned by the mempool. + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// + /// On success, returns transactions that where unorphaned following the insertion + /// of the provided transaction. + /// + /// The returned transactions are references of objects owned by the mempool. pub fn validate_and_insert_transaction( &self, consensus: &dyn ConsensusApi, transaction: Transaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { - self.validate_and_insert_mutable_transaction(consensus, MutableTransaction::from_tx(transaction), priority, orphan) + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { + self.validate_and_insert_mutable_transaction(consensus, MutableTransaction::from_tx(transaction), priority, orphan, rbf_policy) } - /// Exposed only for tests. Ordinary users should call `validate_and_insert_transaction` instead - pub fn validate_and_insert_mutable_transaction( + /// Exposed for tests only + /// + /// See `validate_and_insert_transaction` + pub(crate) fn validate_and_insert_mutable_transaction( &self, consensus: &dyn ConsensusApi, transaction: MutableTransaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { // read lock on mempool - let mut transaction = self.mempool.read().pre_validate_and_populate_transaction(consensus, transaction)?; + let TransactionPreValidation { mut transaction, feerate_threshold } = + self.mempool.read().pre_validate_and_populate_transaction(consensus, transaction, rbf_policy)?; + let args = TransactionValidationArgs::new(feerate_threshold); // no lock on mempool - let validation_result = validate_mempool_transaction(consensus, &mut transaction); + let validation_result = validate_mempool_transaction(consensus, &mut transaction, &args); // write lock on mempool let mut mempool = self.mempool.write(); - if let Some(accepted_transaction) = - mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan)? - { - let unorphaned_transactions = mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction); - drop(mempool); - - // The capacity used here may be exceeded since accepted unorphaned transaction may themselves unorphan other transactions. - let mut accepted_transactions = Vec::with_capacity(unorphaned_transactions.len() + 1); - // We include the original accepted transaction as well - accepted_transactions.push(accepted_transaction); - accepted_transactions.extend(self.validate_and_insert_unorphaned_transactions(consensus, unorphaned_transactions)); - self.counters.increase_tx_counts(1, priority); - - Ok(accepted_transactions) - } else { - Ok(vec![]) + match mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan, rbf_policy)? { + TransactionPostValidation { removed, accepted: Some(accepted_transaction) } => { + let unorphaned_transactions = mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction); + drop(mempool); + + // The capacity used here may be exceeded since accepted unorphaned transaction may themselves unorphan other transactions. + let mut accepted_transactions = Vec::with_capacity(unorphaned_transactions.len() + 1); + // We include the original accepted transaction as well + accepted_transactions.push(accepted_transaction); + accepted_transactions.extend(self.validate_and_insert_unorphaned_transactions(consensus, unorphaned_transactions)); + self.counters.increase_tx_counts(1, priority); + + Ok(TransactionInsertion::new(removed, accepted_transactions)) + } + TransactionPostValidation { removed, accepted: None } => Ok(TransactionInsertion::new(removed, vec![])), } } @@ -260,6 +278,9 @@ impl MiningManager { ) -> Vec> { // The capacity used here may be exceeded (see next comment). let mut accepted_transactions = Vec::with_capacity(incoming_transactions.len()); + // The validation args map is immutably empty since unorphaned transactions do not require pre processing so there + // are no feerate thresholds to use. Instead, we rely on this being checked during post processing. + let args = TransactionValidationBatchArgs::new(); // We loop as long as incoming unorphaned transactions do unorphan other transactions when they // get validated and inserted into the mempool. while !incoming_transactions.is_empty() { @@ -274,8 +295,11 @@ impl MiningManager { let mut validation_results = Vec::with_capacity(transactions.len()); while let Some(upper_bound) = self.next_transaction_chunk_upper_bound(&transactions, lower_bound) { assert!(lower_bound < upper_bound, "the chunk is never empty"); - validation_results - .extend(validate_mempool_transactions_in_parallel(consensus, &mut transactions[lower_bound..upper_bound])); + validation_results.extend(validate_mempool_transactions_in_parallel( + consensus, + &mut transactions[lower_bound..upper_bound], + &args, + )); lower_bound = upper_bound; } assert_eq!(transactions.len(), validation_results.len(), "every transaction should have a matching validation result"); @@ -288,19 +312,21 @@ impl MiningManager { .zip(validation_results) .flat_map(|((transaction, priority), validation_result)| { let orphan_id = transaction.id(); + let rbf_policy = Mempool::get_orphan_transaction_rbf_policy(priority); match mempool.post_validate_and_insert_transaction( consensus, validation_result, transaction, priority, Orphan::Forbidden, + rbf_policy, ) { - Ok(Some(accepted_transaction)) => { + Ok(TransactionPostValidation { removed: _, accepted: Some(accepted_transaction) }) => { accepted_transactions.push(accepted_transaction.clone()); self.counters.increase_tx_counts(1, priority); mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction) } - Ok(None) => vec![], + Ok(TransactionPostValidation { removed: _, accepted: None }) => vec![], Err(err) => { debug!("Failed to unorphan transaction {0} due to rule error: {1}", orphan_id, err); vec![] @@ -316,14 +342,18 @@ impl MiningManager { /// Validates a batch of transactions, handling iteratively only the independent ones, and /// adds those to the set of known transactions that have not yet been added to any block. /// + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// /// Returns transactions that where unorphaned following the insertion of the provided - /// transactions. The returned transactions are clones of objects owned by the mempool. + /// transactions. The returned transactions are references of objects owned by the mempool. pub fn validate_and_insert_transaction_batch( &self, consensus: &dyn ConsensusApi, transactions: Vec, priority: Priority, orphan: Orphan, + rbf_policy: RbfPolicy, ) -> Vec>> { const TRANSACTION_CHUNK_SIZE: usize = 250; @@ -337,12 +367,18 @@ impl MiningManager { // read lock on mempool // Here, we simply log and drop all erroneous transactions since the caller doesn't care about those anyway let mut transactions = Vec::with_capacity(sorted_transactions.len()); + let mut args = TransactionValidationBatchArgs::new(); for chunk in &sorted_transactions.chunks(TRANSACTION_CHUNK_SIZE) { let mempool = self.mempool.read(); let txs = chunk.filter_map(|tx| { let transaction_id = tx.id(); - match mempool.pre_validate_and_populate_transaction(consensus, tx) { - Ok(tx) => Some(tx), + match mempool.pre_validate_and_populate_transaction(consensus, tx, rbf_policy) { + Ok(TransactionPreValidation { transaction, feerate_threshold }) => { + if let Some(threshold) = feerate_threshold { + args.set_feerate_threshold(transaction.id(), threshold); + } + Some(transaction) + } Err(RuleError::RejectAlreadyAccepted(transaction_id)) => { debug!("Ignoring already accepted transaction {}", transaction_id); None @@ -371,8 +407,11 @@ impl MiningManager { let mut validation_results = Vec::with_capacity(transactions.len()); while let Some(upper_bound) = self.next_transaction_chunk_upper_bound(&transactions, lower_bound) { assert!(lower_bound < upper_bound, "the chunk is never empty"); - validation_results - .extend(validate_mempool_transactions_in_parallel(consensus, &mut transactions[lower_bound..upper_bound])); + validation_results.extend(validate_mempool_transactions_in_parallel( + consensus, + &mut transactions[lower_bound..upper_bound], + &args, + )); lower_bound = upper_bound; } assert_eq!(transactions.len(), validation_results.len(), "every transaction should have a matching validation result"); @@ -383,13 +422,20 @@ impl MiningManager { let mut mempool = self.mempool.write(); let txs = chunk.flat_map(|(transaction, validation_result)| { let transaction_id = transaction.id(); - match mempool.post_validate_and_insert_transaction(consensus, validation_result, transaction, priority, orphan) { - Ok(Some(accepted_transaction)) => { + match mempool.post_validate_and_insert_transaction( + consensus, + validation_result, + transaction, + priority, + orphan, + rbf_policy, + ) { + Ok(TransactionPostValidation { removed: _, accepted: Some(accepted_transaction) }) => { insert_results.push(Ok(accepted_transaction.clone())); self.counters.increase_tx_counts(1, priority); mempool.get_unorphaned_transactions_after_accepted_transaction(&accepted_transaction) } - Ok(None) => { + Ok(TransactionPostValidation { removed: _, accepted: None }) | Err(RuleError::RejectDuplicate(_)) => { // Either orphaned or already existing in the mempool vec![] } @@ -756,32 +802,43 @@ impl MiningManagerProxy { /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// - /// The returned transactions are clones of objects owned by the mempool. + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// + /// The returned transactions are references of objects owned by the mempool. pub async fn validate_and_insert_transaction( self, consensus: &ConsensusProxy, transaction: Transaction, priority: Priority, orphan: Orphan, - ) -> MiningManagerResult>> { - consensus.clone().spawn_blocking(move |c| self.inner.validate_and_insert_transaction(c, transaction, priority, orphan)).await + rbf_policy: RbfPolicy, + ) -> MiningManagerResult { + consensus + .clone() + .spawn_blocking(move |c| self.inner.validate_and_insert_transaction(c, transaction, priority, orphan, rbf_policy)) + .await } /// Validates a batch of transactions, handling iteratively only the independent ones, and /// adds those to the set of known transactions that have not yet been added to any block. /// + /// The validation is constrained by a Replace by fee policy applied + /// to double spends in the mempool. For more information, see [`RbfPolicy`]. + /// /// Returns transactions that where unorphaned following the insertion of the provided - /// transactions. The returned transactions are clones of objects owned by the mempool. + /// transactions. The returned transactions are references of objects owned by the mempool. pub async fn validate_and_insert_transaction_batch( self, consensus: &ConsensusProxy, transactions: Vec, priority: Priority, orphan: Orphan, + rbf_policy: RbfPolicy, ) -> Vec>> { consensus .clone() - .spawn_blocking(move |c| self.inner.validate_and_insert_transaction_batch(c, transactions, priority, orphan)) + .spawn_blocking(move |c| self.inner.validate_and_insert_transaction_batch(c, transactions, priority, orphan, rbf_policy)) .await } diff --git a/mining/src/manager_tests.rs b/mining/src/manager_tests.rs index 51d4f25..272f6fc 100644 --- a/mining/src/manager_tests.rs +++ b/mining/src/manager_tests.rs @@ -7,19 +7,20 @@ mod tests { mempool::{ config::{Config, DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE}, errors::RuleError, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, }, model::{candidate_tx::CandidateTransaction, tx_query::TransactionQuery}, testutils::consensus_mock::ConsensusMock, MiningCounters, }; + use itertools::Itertools; use spectre_addresses::{Address, Prefix, Version}; use spectre_consensus_core::{ api::ConsensusApi, block::TemplateBuildMode, coinbase::MinerData, constants::{MAX_TX_IN_SEQUENCE_NUM, SOMPI_PER_SPECTRE, TX_VERSION}, - errors::tx::{TxResult, TxRuleError}, + errors::tx::TxRuleError, mass::transaction_estimated_serialized_size, subnets::SUBNETWORK_ID_NATIVE, tx::{ @@ -28,11 +29,12 @@ mod tests { }, }; use spectre_hashes::Hash; + use spectre_mining_errors::mempool::RuleResult; use spectre_txscript::{ pay_to_address_script, pay_to_script_hash_signature_script, - test_helpers::{create_transaction, op_true_script}, + test_helpers::{create_transaction, create_transaction_with_change, op_true_script}, }; - use std::sync::Arc; + use std::{iter::once, sync::Arc}; use tokio::sync::mpsc::{error::TryRecvError, unbounded_channel}; const TARGET_TIME_PER_BLOCK: u64 = 1_000; @@ -42,72 +44,106 @@ mod tests { #[test] fn test_validate_and_insert_transaction() { const TX_COUNT: u32 = 10; - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transactions_to_insert = (0..TX_COUNT).map(|i| create_transaction_with_utxo_entry(i, 0)).collect::>(); - for transaction in transactions_to_insert.iter() { - let result = mining_manager.validate_and_insert_mutable_transaction( - consensus.as_ref(), - transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "inserting a valid transaction failed"); - } - // The UtxoEntry was filled manually for those transactions, so the transactions won't be considered orphans. - // Therefore, all the transactions expected to be contained in the mempool. - let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); - assert_eq!( - transactions_to_insert.len(), - transactions_from_pool.len(), - "wrong number of transactions in mempool: expected: {}, got: {}", - transactions_to_insert.len(), - transactions_from_pool.len() - ); - transactions_to_insert.iter().for_each(|tx_to_insert| { - let found_exact_match = transactions_from_pool.contains(tx_to_insert); - let tx_from_pool = transactions_from_pool.iter().find(|tx_from_pool| tx_from_pool.id() == tx_to_insert.id()); - let found_transaction_id = tx_from_pool.is_some(); - if found_transaction_id && !found_exact_match { - let tx = tx_from_pool.unwrap(); - assert_eq!( - tx_to_insert.calculated_fee.unwrap(), - tx.calculated_fee.unwrap(), - "wrong fee in transaction {}: expected: {}, got: {}", - tx.id(), - tx_to_insert.calculated_fee.unwrap(), - tx.calculated_fee.unwrap() - ); - assert_eq!( - tx_to_insert.calculated_compute_mass.unwrap(), - tx.calculated_compute_mass.unwrap(), - "wrong mass in transaction {}: expected: {}, got: {}", - tx.id(), - tx_to_insert.calculated_compute_mass.unwrap(), - tx.calculated_compute_mass.unwrap() - ); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + let transactions_to_insert = (0..TX_COUNT).map(|i| create_transaction_with_utxo_entry(i, 0)).collect::>(); + for transaction in transactions_to_insert.iter() { + let result = into_mempool_result(mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy, + )); + match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => { + assert!(result.is_ok(), "({priority:?}, {orphan:?}, {rbf_policy:?}) inserting a valid transaction failed"); + } + RbfPolicy::Mandatory => { + assert!(result.is_err(), "({priority:?}, {orphan:?}, {rbf_policy:?}) replacing a valid transaction without replacement in mempool should fail"); + let err = result.unwrap_err(); + assert_eq!( + RuleError::RejectRbfNoDoubleSpend, + err, + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong error: expected {} got: {}", + RuleError::RejectRbfNoDoubleSpend, + err, + ); + } + } } - assert!(found_exact_match, "missing transaction {} in the mempool, no exact match", tx_to_insert.id()); - }); - // The parent's transaction was inserted into the consensus, so we want to verify that - // the child transaction is not considered an orphan and inserted into the mempool. - let transaction_not_an_orphan = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction_not_an_orphan.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "inserting the child transaction {} into the mempool failed", transaction_not_an_orphan.id()); - let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); - assert!( - contained_by(transaction_not_an_orphan.id(), &transactions_from_pool), - "missing transaction {} in the mempool", - transaction_not_an_orphan.id() - ); + // The UtxoEntry was filled manually for those transactions, so the transactions won't be considered orphans. + // Therefore, all the transactions expected to be contained in the mempool if replace by fee policy allowed it. + let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); + let transactions_inserted = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => transactions_to_insert.clone(), + RbfPolicy::Mandatory => { + vec![] + } + }; + assert_eq!( + transactions_inserted.len(), + transactions_from_pool.len(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong number of transactions in mempool: expected: {}, got: {}", + transactions_inserted.len(), + transactions_from_pool.len() + ); + transactions_inserted.iter().for_each(|tx_to_insert| { + let found_exact_match = transactions_from_pool.contains(tx_to_insert); + let tx_from_pool = transactions_from_pool.iter().find(|tx_from_pool| tx_from_pool.id() == tx_to_insert.id()); + let found_transaction_id = tx_from_pool.is_some(); + if found_transaction_id && !found_exact_match { + let tx = tx_from_pool.unwrap(); + assert_eq!( + tx_to_insert.calculated_fee.unwrap(), + tx.calculated_fee.unwrap(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong fee in transaction {}: expected: {}, got: {}", + tx.id(), + tx_to_insert.calculated_fee.unwrap(), + tx.calculated_fee.unwrap() + ); + assert_eq!( + tx_to_insert.calculated_compute_mass.unwrap(), + tx.calculated_compute_mass.unwrap(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) wrong mass in transaction {}: expected: {}, got: {}", + tx.id(), + tx_to_insert.calculated_compute_mass.unwrap(), + tx.calculated_compute_mass.unwrap() + ); + } + assert!( + found_exact_match, + "({priority:?}, {orphan:?}, {rbf_policy:?}) missing transaction {} in the mempool, no exact match", + tx_to_insert.id() + ); + }); + + // The parent's transaction was inserted into the consensus, so we want to verify that + // the child transaction is not considered an orphan and inserted into the mempool. + let transaction_not_an_orphan = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction_not_an_orphan.clone(), + priority, + orphan, + RbfPolicy::Forbidden, + ); + assert!( + result.is_ok(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) inserting the child transaction {} into the mempool failed", + transaction_not_an_orphan.id() + ); + let (transactions_from_pool, _) = mining_manager.get_all_transactions(TransactionQuery::TransactionsOnly); + assert!( + contained_by(transaction_not_an_orphan.id(), &transactions_from_pool), + "({priority:?}, {orphan:?}, {rbf_policy:?}) missing transaction {} in the mempool", + transaction_not_an_orphan.id() + ); + } } /// test_simulated_error_in_consensus verifies that a predefined result is actually @@ -115,127 +151,397 @@ mod tests { /// insert a transaction. #[test] fn test_simulated_error_in_consensus() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - - // Build an invalid transaction with some gas and inform the consensus mock about the result it should return - // when the mempool will submit this transaction for validation. - let mut transaction = create_transaction_with_utxo_entry(0, 1); - Arc::make_mut(&mut transaction.tx).gas = 1000; - let status = Err(TxRuleError::TxHasGas); - consensus.set_status(transaction.id(), status.clone()); - - // Try validate and insert the transaction into the mempool - let result = into_status(mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction.tx.as_ref().clone(), - Priority::Low, - Orphan::Allowed, - )); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + + // Build an invalid transaction with some gas and inform the consensus mock about the result it should return + // when the mempool will submit this transaction for validation. + let mut transaction = create_transaction_with_utxo_entry(0, 1); + Arc::make_mut(&mut transaction.tx).gas = 1000; + let tx_err = TxRuleError::TxHasGas; + let expected = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Allowed => Err(RuleError::from(tx_err.clone())), + RbfPolicy::Mandatory => Err(RuleError::RejectRbfNoDoubleSpend), + }; + consensus.set_status(transaction.id(), Err(tx_err)); + + // Try validate and insert the transaction into the mempool + let result = into_mempool_result(mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy, + )); - assert_eq!( - status, result, - "Unexpected result when trying to insert an invalid transaction: expected: {status:?}, got: {result:?}", - ); - let pool_tx = mining_manager.get_transaction(&transaction.id(), TransactionQuery::All); - assert!(pool_tx.is_none(), "Mempool contains a transaction that should have been rejected"); + assert_eq!( + expected, result, + "({priority:?}, {orphan:?}, {rbf_policy:?}) unexpected result when trying to insert an invalid transaction: expected: {expected:?}, got: {result:?}", + ); + let pool_tx = mining_manager.get_transaction(&transaction.id(), TransactionQuery::All); + assert!( + pool_tx.is_none(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) mempool contains a transaction that should have been rejected" + ); + } } /// test_insert_double_transactions_to_mempool verifies that an attempt to insert a transaction /// more than once into the mempool will result in raising an appropriate error. #[test] fn test_insert_double_transactions_to_mempool() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transaction = create_transaction_with_utxo_entry(0, 0); + let transaction = create_transaction_with_utxo_entry(0, 0); - // submit the transaction to the mempool - let result = mining_manager.validate_and_insert_mutable_transaction( - consensus.as_ref(), - transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_ok(), "mempool should have accepted a valid transaction but did not"); - - // submit the same transaction again to the mempool - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - transaction.tx.as_ref().clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_err(), "mempool should refuse a double submit of the same transaction but accepts it"); - if let Err(MiningManagerError::MempoolError(RuleError::RejectDuplicate(transaction_id))) = result { - assert_eq!( - transaction.id(), - transaction_id, - "the error returned by the mempool should include id {} but provides {}", - transaction.id(), - transaction_id + // submit the transaction to the mempool + let result = mining_manager.validate_and_insert_mutable_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + rbf_policy.for_insert(), ); - } else { - panic!( - "the nested error returned by the mempool should be variant RuleError::RejectDuplicate but is {:?}", - result.err().unwrap() + assert!( + result.is_ok(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should have accepted a valid transaction but did not" ); + + // submit the same transaction again to the mempool + let result = into_mempool_result(mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.tx.as_ref().clone(), + priority, + orphan, + rbf_policy, + )); + match result { + Err(RuleError::RejectDuplicate(transaction_id)) => { + assert_eq!( + transaction.id(), + transaction_id, + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should include transaction id {} but provides {}", + transaction.id(), + transaction_id + ); + } + Err(err) => { + panic!( + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should be {:?} but is {err:?}", + RuleError::RejectDuplicate(transaction.id()) + ); + } + Ok(()) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should refuse a double submit of the same transaction but accepts it"); + } + } } } - // test_double_spend_in_mempool verifies that an attempt to insert a transaction double-spending - // another transaction already in the mempool will result in raising an appropriate error. + /// test_double_spend_in_mempool verifies that an attempt to insert a transaction double-spending + /// another transaction already in the mempool will result in raising an appropriate error. #[test] fn test_double_spend_in_mempool() { - let consensus = Arc::new(ConsensusMock::new()); - let counters = Arc::new(MiningCounters::default()); - let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + for (priority, orphan, rbf_policy) in all_priority_orphan_rbf_policy_combinations() { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); - let transaction = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); - assert!( - consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), - "the consensus mock should have spendable UTXOs for the newly created transaction {}", - transaction.id() - ); + let transaction = create_child_and_parent_txs_and_add_parent_to_consensus(&consensus); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), + "({priority:?}, {orphan:?}, {rbf_policy:?}) the consensus mock should have spendable UTXOs for the newly created transaction {}", + transaction.id() + ); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), transaction.clone(), Priority::Low, Orphan::Allowed); - assert!(result.is_ok(), "the mempool should accept a valid transaction when it is able to populate its UTXO entries"); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.clone(), + priority, + orphan, + RbfPolicy::Forbidden, + ); + assert!(result.is_ok(), "({priority:?}, {orphan:?}, {rbf_policy:?}) the mempool should accept a valid transaction when it is able to populate its UTXO entries"); - let mut double_spending_transaction = transaction.clone(); - double_spending_transaction.outputs[0].value -= 1; // do some minor change so that txID is different - double_spending_transaction.finalize(); - assert_ne!( - transaction.id(), - double_spending_transaction.id(), - "two transactions differing by only one output value should have different ids" - ); - let result = mining_manager.validate_and_insert_transaction( - consensus.as_ref(), - double_spending_transaction.clone(), - Priority::Low, - Orphan::Allowed, - ); - assert!(result.is_err(), "mempool should refuse a double spend transaction but accepts it"); - if let Err(MiningManagerError::MempoolError(RuleError::RejectDoubleSpendInMempool(_, transaction_id))) = result { - assert_eq!( + let mut double_spending_transaction = transaction.clone(); + double_spending_transaction.outputs[0].value += 1; // do some minor change so that txID is different while not increasing fee + double_spending_transaction.finalize(); + assert_ne!( transaction.id(), - transaction_id, - "the error returned by the mempool should include id {} but provides {}", - transaction.id(), - transaction_id - ); - } else { - panic!( - "the nested error returned by the mempool should be variant RuleError::RejectDoubleSpendInMempool but is {:?}", - result.err().unwrap() + double_spending_transaction.id(), + "({priority:?}, {orphan:?}, {rbf_policy:?}) two transactions differing by only one output value should have different ids" ); + let result = into_mempool_result(mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + double_spending_transaction.clone(), + priority, + orphan, + rbf_policy, + )); + match result { + Err(RuleError::RejectDoubleSpendInMempool(_, transaction_id)) => { + assert_eq!( + transaction.id(), + transaction_id, + "({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should include id {} but provides {}", + transaction.id(), + transaction_id + ); + } + Err(err) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) the error returned by the mempool should be RuleError::RejectDoubleSpendInMempool but is {err:?}"); + } + Ok(()) => { + panic!("({priority:?}, {orphan:?}, {rbf_policy:?}) mempool should refuse a double spend transaction ineligible to RBF but accepts it"); + } + } + } + } + + /// test_replace_by_fee_in_mempool verifies that an attempt to insert a double-spending transaction + /// will cause or not the transaction(s) double spending in the mempool to be replaced/removed, + /// depending on varying factors. + #[test] + fn test_replace_by_fee_in_mempool() { + const BASE_FEE: u64 = DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE; + + struct TxOp { + /// Funding transaction indexes + tx: Vec, + /// Funding transaction output indexes + output: Vec, + /// Add a change output to the transaction + change: bool, + /// Transaction fee + fee: u64, + /// Children binary tree depth + depth: usize, + } + + impl TxOp { + fn change(&self) -> Option { + self.change.then_some(900 * SOMPI_PER_SPECTRE) + } + } + + struct Test { + name: &'static str, + /// Initial transactions in the mempool + starts: Vec, + /// Replacement transaction submitted to the mempool + replacement: TxOp, + /// Expected RBF result for the 3 policies [Forbidden, Allowed, Mandatory] + expected: [bool; 3], + } + + impl Test { + fn run_rbf(&self, rbf_policy: RbfPolicy, expected: bool) { + let consensus = Arc::new(ConsensusMock::new()); + let counters = Arc::new(MiningCounters::default()); + let mining_manager = MiningManager::new(TARGET_TIME_PER_BLOCK, false, MAX_BLOCK_MASS, None, counters); + let funding_transactions = create_and_add_funding_transactions(&consensus, 10); + + // RPC submit the initial transactions + let (transactions, children): (Vec<_>, Vec<_>) = + self.starts + .iter() + .map(|tx_op| { + let transaction = create_funded_transaction( + select_transactions(&funding_transactions, &tx_op.tx), + tx_op.output.clone(), + tx_op.change(), + tx_op.fee, + ); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction.clone())), + "[{}, {:?}] the consensus should have spendable UTXOs for the newly created transaction {}", + self.name, rbf_policy, transaction.id() + ); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction.clone(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); + assert!( + result.is_ok(), + "[{}, {:?}] the mempool should accept a valid transaction when it is able to populate its UTXO entries", + self.name, rbf_policy, + ); + let children = create_children_tree(&transaction, tx_op.depth); + let children_count = (2_usize.pow(tx_op.depth as u32) - 1) * transaction.outputs.len(); + assert_eq!( + children.len(), children_count, + "[{}, {:?}] a parent transaction with {} output(s) should generate a binary children tree of depth {} with {} children but got {}", + self.name, rbf_policy, transaction.outputs.len(), tx_op.depth, children_count, children.len(), + ); + validate_and_insert_transactions( + &mining_manager, + consensus.as_ref(), + children.iter(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); + (transaction, children) + }) + .unzip(); + + // RPC submit transaction replacement + let transaction_replacement = create_funded_transaction( + select_transactions(&funding_transactions, &self.replacement.tx), + self.replacement.output.clone(), + self.replacement.change(), + self.replacement.fee, + ); + assert!( + consensus.can_finance_transaction(&MutableTransaction::from_tx(transaction_replacement.clone())), + "[{}, {:?}] the consensus should have spendable UTXOs for the newly created transaction {}", + self.name, + rbf_policy, + transaction_replacement.id() + ); + let tx_count = mining_manager.transaction_count(TransactionQuery::TransactionsOnly); + let expected_tx_count = match expected { + true => tx_count + 1 - transactions.len() - children.iter().map(|x| x.len()).sum::(), + false => tx_count, + }; + let priority = match rbf_policy { + RbfPolicy::Forbidden | RbfPolicy::Mandatory => Priority::High, + RbfPolicy::Allowed => Priority::Low, + }; + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + transaction_replacement.clone(), + priority, + Orphan::Forbidden, + rbf_policy, + ); + if expected { + assert!(result.is_ok(), "[{}, {:?}] mempool should accept a RBF transaction", self.name, rbf_policy,); + let tx_insertion = result.unwrap(); + assert_eq!( + tx_insertion.removed.as_ref().unwrap().id(), + transactions[0].id(), + "[{}, {:?}] RBF should return the removed transaction", + self.name, + rbf_policy, + ); + transactions.iter().for_each(|x| { + assert!( + !mining_manager.has_transaction(&x.id(), TransactionQuery::All), + "[{}, {:?}] RBF replaced transaction should no longer be in the mempool", + self.name, + rbf_policy, + ); + }); + assert_transaction_count( + &mining_manager, + expected_tx_count, + &format!( + "[{}, {:?}] RBF should remove all chained transactions of the removed mempool transaction(s)", + self.name, rbf_policy + ), + ); + } else { + assert!(result.is_err(), "[{}, {:?}] mempool should reject the RBF transaction", self.name, rbf_policy); + transactions.iter().for_each(|x| { + assert!( + mining_manager.has_transaction(&x.id(), TransactionQuery::All), + "[{}, {:?}] RBF transaction target is no longer in the mempool", + self.name, + rbf_policy + ); + }); + assert_transaction_count( + &mining_manager, + expected_tx_count, + &format!("[{}, {:?}] a failing RBF should leave the mempool unchanged", self.name, rbf_policy), + ); + } + } + + fn run(&self) { + [RbfPolicy::Forbidden, RbfPolicy::Allowed, RbfPolicy::Mandatory].iter().copied().enumerate().for_each( + |(i, rbf_policy)| { + self.run_rbf(rbf_policy, self.expected[i]); + }, + ) + } + } + + let tests = vec![ + Test { + name: "1 input, 1 output <=> 1 input, 1 output, constant fee", + starts: vec![TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }], + replacement: TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "1 input, 1 output <=> 1 input, 1 output, increased fee", + starts: vec![TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }], + replacement: TxOp { tx: vec![0], output: vec![0], change: false, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 2 outputs, increased fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "4 inputs, 2 outputs <=> 2 inputs, 2 outputs, constant fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 1 output, constant fee", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "2 inputs, 2 outputs <=> 4 inputs, 2 output, constant fee (MUST FAIL on fee/mass)", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "2 inputs, 1 output <=> 4 inputs, 2 output, increased fee (MUST FAIL on fee/mass)", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: false, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE + 10, depth: 0 }, + expected: [false, false, false], + }, + Test { + name: "2 inputs, 2 outputs <=> 2 inputs, 1 output, constant fee, partial double spend overlap", + starts: vec![TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }], + replacement: TxOp { tx: vec![0, 2], output: vec![0], change: false, fee: BASE_FEE, depth: 0 }, + expected: [false, true, true], + }, + Test { + name: "(2 inputs, 2 outputs) * 2 <=> 4 inputs, 2 outputs, increased fee, 2 double spending mempool transactions (MUST FAIL on Mandatory)", + starts: vec![ + TxOp { tx: vec![0, 1], output: vec![0], change: true, fee: BASE_FEE, depth: 2 }, + TxOp { tx: vec![0, 1], output: vec![1], change: true, fee: BASE_FEE, depth: 2 }, + ], + replacement: TxOp { tx: vec![0, 1], output: vec![0, 1], change: true, fee: BASE_FEE * 2, depth: 0 }, + expected: [false, true, false], + }, + ]; + + for test in tests { + test.run(); } } - // test_handle_new_block_transactions verifies that all the transactions in the block were successfully removed from the mempool. + /// test_handle_new_block_transactions verifies that all the transactions in the block were successfully removed from the mempool. #[test] fn test_handle_new_block_transactions() { let consensus = Arc::new(ConsensusMock::new()); @@ -250,6 +556,7 @@ mod tests { transaction.tx.as_ref().clone(), Priority::Low, Orphan::Allowed, + RbfPolicy::Forbidden, ); assert!(result.is_ok(), "the insertion of a new valid transaction in the mempool failed"); } @@ -295,8 +602,8 @@ mod tests { } #[test] - // test_double_spend_with_block verifies that any transactions which are now double spends as a result of the block's new transactions - // will be removed from the mempool. + /// test_double_spend_with_block verifies that any transactions which are now double spends as a result of the block's new transactions + /// will be removed from the mempool. fn test_double_spend_with_block() { let consensus = Arc::new(ConsensusMock::new()); let counters = Arc::new(MiningCounters::default()); @@ -308,6 +615,7 @@ mod tests { transaction_in_the_mempool.tx.as_ref().clone(), Priority::Low, Orphan::Allowed, + RbfPolicy::Forbidden, ); assert!(result.is_ok()); @@ -326,7 +634,7 @@ mod tests { ); } - // test_orphan_transactions verifies that a transaction could be a part of a new block template only if it's not an orphan. + /// test_orphan_transactions verifies that a transaction could be a part of a new block template only if it's not an orphan. #[test] fn test_orphan_transactions() { let consensus = Arc::new(ConsensusMock::new()); @@ -340,8 +648,13 @@ mod tests { assert_eq!(parent_txs.len(), TX_PAIRS_COUNT); assert_eq!(child_txs.len(), TX_PAIRS_COUNT); for orphan in child_txs.iter() { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), orphan.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + orphan.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid orphan transaction {}", orphan.id()); } let (populated_txs, orphans) = mining_manager.get_all_transactions(TransactionQuery::All); @@ -485,10 +798,15 @@ mod tests { ); // Add the remaining parent transaction into the mempool - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), parent_txs[0].clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + parent_txs[0].clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the insertion of the remaining parent transaction in the mempool failed"); - let unorphaned_txs = result.unwrap(); + let unorphaned_txs = result.unwrap().accepted; let (populated_txs, orphans) = mining_manager.get_all_transactions(TransactionQuery::All); assert_eq!( unorphaned_txs.len(), SKIPPED_TXS + 1, @@ -592,8 +910,13 @@ mod tests { // Try submit children while rejecting orphans for (tx, test) in child_txs.iter().zip(tests.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Forbidden); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Forbidden, + RbfPolicy::Forbidden, + ); assert!(result.is_err(), "mempool should reject an orphan transaction with {:?} when asked to do so", test.priority); if let Err(MiningManagerError::MempoolError(RuleError::RejectDisallowedOrphan(transaction_id))) = result { assert_eq!( @@ -613,8 +936,13 @@ mod tests { // Try submit children while accepting orphans for (tx, test) in child_txs.iter().zip(tests.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert_eq!( test.should_enter_orphan_pool, result.is_ok(), @@ -623,7 +951,7 @@ mod tests { test.insert_result() ); if let Ok(unorphaned_txs) = result { - assert!(unorphaned_txs.is_empty(), "mempool should unorphan no transaction since it only contains orphans"); + assert!(unorphaned_txs.accepted.is_empty(), "mempool should unorphan no transaction since it only contains orphans"); } else if let Err(MiningManagerError::MempoolError(RuleError::RejectOrphanPoolIsFull(pool_len, config_len))) = result { assert_eq!( (config.maximum_orphan_transaction_count as usize, config.maximum_orphan_transaction_count), @@ -642,10 +970,15 @@ mod tests { // Submit all the parents for (i, (tx, test)) in parent_txs.iter().zip(tests.iter()).enumerate() { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), tx.clone(), test.priority, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + tx.clone(), + test.priority, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "mempool should accept a valid transaction with {:?} when asked to do so", test.priority,); - let unorphaned_txs = result.as_ref().unwrap(); + let unorphaned_txs = &result.as_ref().unwrap().accepted; assert_eq!( test.should_unorphan, unorphaned_txs.len() > 1, @@ -682,8 +1015,13 @@ mod tests { // Add to mempool a transaction that spends child_tx_2 (as high priority) let spending_tx = create_transaction(&child_tx_2, 1_000); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), spending_tx.clone(), Priority::High, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + spending_tx.clone(), + Priority::High, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the insertion in the mempool of the spending transaction failed"); // Revalidate, to make sure spending_tx is still valid @@ -725,7 +1063,7 @@ mod tests { assert!(orphan_txs.is_empty(), "orphan pool should be empty"); } - // test_modify_block_template verifies that modifying a block template changes coinbase data correctly. + /// test_modify_block_template verifies that modifying a block template changes coinbase data correctly. #[test] fn test_modify_block_template() { let consensus = Arc::new(ConsensusMock::new()); @@ -737,11 +1075,21 @@ mod tests { let (parent_txs, child_txs) = create_arrays_of_parent_and_children_transactions(&consensus, TX_PAIRS_COUNT); for (parent_tx, child_tx) in parent_txs.iter().zip(child_txs.iter()) { - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), parent_tx.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + parent_tx.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid parent transaction {}", parent_tx.id()); - let result = - mining_manager.validate_and_insert_transaction(consensus.as_ref(), child_tx.clone(), Priority::Low, Orphan::Allowed); + let result = mining_manager.validate_and_insert_transaction( + consensus.as_ref(), + child_tx.clone(), + Priority::Low, + Orphan::Allowed, + RbfPolicy::Forbidden, + ); assert!(result.is_ok(), "the mempool should accept the valid child transaction {}", parent_tx.id()); } @@ -933,6 +1281,69 @@ mod tests { mutable_tx } + fn create_and_add_funding_transactions(consensus: &Arc, count: usize) -> Vec { + // Make the funding amounts always different so that funding txs have different ids + (0..count) + .map(|i| { + let funding_tx = + create_transaction_without_input(vec![1_000 * SOMPI_PER_SPECTRE, 2_500 * SOMPI_PER_SPECTRE + i as u64]); + consensus.add_transaction(funding_tx.clone(), 1); + funding_tx + }) + .collect_vec() + } + + fn select_transactions<'a>(transactions: &'a [Transaction], indexes: &'a [usize]) -> impl Iterator { + indexes.iter().map(|i| &transactions[*i]) + } + + fn create_funded_transaction<'a>( + txs_to_spend: impl Iterator, + output_indexes: Vec, + change: Option, + fee: u64, + ) -> Transaction { + create_transaction_with_change(txs_to_spend, output_indexes, change, fee) + } + + fn create_children_tree(parent: &Transaction, depth: usize) -> Vec { + let mut tree = vec![]; + let root = [parent.clone()]; + let mut parents = &root[..]; + let mut first_child = 0; + for _ in 0..depth { + let mut children = vec![]; + for parent in parents { + children.extend(parent.outputs.iter().enumerate().map(|(i, output)| { + create_transaction_with_change( + once(parent), + vec![i], + Some(output.value / 2), + DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + ) + })); + } + tree.extend(children); + parents = &tree[first_child..]; + first_child = tree.len() + } + tree + } + + fn validate_and_insert_transactions<'a>( + mining_manager: &MiningManager, + consensus: &dyn ConsensusApi, + transactions: impl Iterator, + priority: Priority, + orphan: Orphan, + rbf_policy: RbfPolicy, + ) { + transactions.for_each(|transaction| { + let result = mining_manager.validate_and_insert_transaction(consensus, transaction.clone(), priority, orphan, rbf_policy); + assert!(result.is_ok(), "the mempool should accept a valid transaction when it is able to populate its UTXO entries"); + }); + } + fn create_arrays_of_parent_and_children_transactions( consensus: &Arc, count: usize, @@ -974,11 +1385,13 @@ mod tests { transactions.iter().any(|x| x.as_ref().id() == transaction_id) } - fn into_status(result: MiningManagerResult) -> TxResult<()> { + fn into_mempool_result(result: MiningManagerResult) -> RuleResult<()> { match result { Ok(_) => Ok(()), - Err(MiningManagerError::MempoolError(RuleError::RejectTxRule(err))) => Err(err), - _ => Ok(()), + Err(MiningManagerError::MempoolError(err)) => Err(err), + _ => { + panic!("result is an unsupported error"); + } } } @@ -1000,4 +1413,26 @@ mod tests { let script = pay_to_address_script(&address); MinerData::new(script, vec![]) } + + #[allow(dead_code)] + fn all_priority_orphan_combinations() -> impl Iterator { + [Priority::Low, Priority::High] + .iter() + .flat_map(|priority| [Orphan::Allowed, Orphan::Forbidden].iter().map(|orphan| (*priority, *orphan))) + } + + fn all_priority_orphan_rbf_policy_combinations() -> impl Iterator { + [Priority::Low, Priority::High].iter().flat_map(|priority| { + [Orphan::Allowed, Orphan::Forbidden].iter().flat_map(|orphan| { + [RbfPolicy::Forbidden, RbfPolicy::Allowed, RbfPolicy::Mandatory] + .iter() + .map(|rbf_policy| (*priority, *orphan, *rbf_policy)) + }) + }) + } + + fn assert_transaction_count(mining_manager: &MiningManager, expected_count: usize, message: &str) { + let count = mining_manager.transaction_count(TransactionQuery::TransactionsOnly); + assert_eq!(expected_count, count, "{message} mempool transaction count: expected {}, got {}", expected_count, count); + } } diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index 2dc05d4..8c6afa2 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -23,6 +23,7 @@ pub(crate) mod handle_new_block_transactions; pub(crate) mod model; pub(crate) mod populate_entries_and_try_validate; pub(crate) mod remove_transaction; +pub(crate) mod replace_by_fee; pub(crate) mod validate_and_insert_transaction; /// Mempool contains transactions intended to be inserted into a block and mined. @@ -158,4 +159,51 @@ pub mod tx { Forbidden, Allowed, } + + /// Replace by Fee (RBF) policy + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub enum RbfPolicy { + /// ### RBF is forbidden + /// + /// Inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - no double spend + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend error. + Forbidden, + + /// ### RBF may occur + /// + /// Identifies double spends in mempool and their owning transactions checking in order every input of the incoming + /// transaction. + /// + /// Removes all mempool transactions owning double spends and inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - on absence of double spends, always succeeds + /// - on double spends, the incoming transaction has a higher fee/mass ratio than the mempool transaction owning + /// the first double spend + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend or a tx fee/mass too low error. + Allowed, + + /// ### RBF must occur + /// + /// Identifies double spends in mempool and their owning transactions checking in order every input of the incoming + /// transaction. + /// + /// Removes the mempool transaction owning the double spends and inserts the incoming transaction. + /// + /// Conditions of success: + /// + /// - at least one double spend + /// - all double spends belong to the same mempool transaction + /// - the incoming transaction has a higher fee/mass ratio than the mempool double spending transaction. + /// + /// If conditions are not met, leaves the mempool unchanged and fails with a double spend or a tx fee/mass too low error. + Mandatory, + } } diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index 67462e0..bf17928 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -5,7 +5,7 @@ use crate::{ model::{ map::MempoolTransactionCollection, pool::{Pool, TransactionsEdges}, - tx::MempoolTransaction, + tx::{DoubleSpend, MempoolTransaction}, utxo_set::MempoolUtxoSet, }, tx::Priority, @@ -182,6 +182,7 @@ impl TransactionsPool { } false } + /// Returns the exceeding low-priority transactions having the lowest fee rates in order /// to have room for at least `free_slots` new transactions. The returned transactions /// are guaranteed to be unchained (no successor in mempool) and to not be parent of @@ -245,10 +246,29 @@ impl TransactionsPool { self.utxo_set.get_outpoint_owner_id(outpoint) } + /// Make sure no other transaction in the mempool is already spending an output which one of this transaction inputs spends pub(crate) fn check_double_spends(&self, transaction: &MutableTransaction) -> RuleResult<()> { self.utxo_set.check_double_spends(transaction) } + /// Returns the first double spend of every transaction in the mempool double spending on `transaction` + pub(crate) fn get_double_spend_transaction_ids(&self, transaction: &MutableTransaction) -> Vec { + self.utxo_set.get_double_spend_transaction_ids(transaction) + } + + pub(crate) fn get_double_spend_owner<'a>(&'a self, double_spend: &DoubleSpend) -> RuleResult<&'a MempoolTransaction> { + match self.get(&double_spend.owner_id) { + Some(transaction) => Ok(transaction), + None => { + // This case should never arise in the first place. + // Anyway, in case it does, if a double spent transaction id is found but the matching + // transaction cannot be located in the mempool a replacement is no longer possible + // so a double spend error is returned. + Err(double_spend.into()) + } + } + } + pub(crate) fn collect_expired_low_priority_transactions(&mut self, virtual_daa_score: u64) -> Vec { let now = unix_now(); if virtual_daa_score < self.last_expire_scan_daa_score + self.config.transaction_expire_scan_interval_daa_score diff --git a/mining/src/mempool/model/tx.rs b/mining/src/mempool/model/tx.rs index 4b9dd72..f596f78 100644 --- a/mining/src/mempool/model/tx.rs +++ b/mining/src/mempool/model/tx.rs @@ -1,8 +1,10 @@ -use crate::mempool::tx::Priority; -use spectre_consensus_core::{tx::MutableTransaction, tx::TransactionId}; +use crate::mempool::tx::{Priority, RbfPolicy}; +use spectre_consensus_core::tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint}; +use spectre_mining_errors::mempool::RuleError; use std::{ cmp::Ordering, fmt::{Display, Formatter}, + sync::Arc, }; pub(crate) struct MempoolTransaction { @@ -53,6 +55,51 @@ impl PartialEq for MempoolTransaction { } } +impl RbfPolicy { + #[cfg(test)] + /// Returns an alternate policy accepting a transaction insertion in case the policy requires a replacement + pub(crate) fn for_insert(&self) -> RbfPolicy { + match self { + RbfPolicy::Forbidden | RbfPolicy::Allowed => *self, + RbfPolicy::Mandatory => RbfPolicy::Allowed, + } + } +} + +pub(crate) struct DoubleSpend { + pub outpoint: TransactionOutpoint, + pub owner_id: TransactionId, +} + +impl DoubleSpend { + pub fn new(outpoint: TransactionOutpoint, owner_id: TransactionId) -> Self { + Self { outpoint, owner_id } + } +} + +impl From for RuleError { + fn from(value: DoubleSpend) -> Self { + RuleError::RejectDoubleSpendInMempool(value.outpoint, value.owner_id) + } +} + +impl From<&DoubleSpend> for RuleError { + fn from(value: &DoubleSpend) -> Self { + RuleError::RejectDoubleSpendInMempool(value.outpoint, value.owner_id) + } +} + +pub(crate) struct TransactionPreValidation { + pub transaction: MutableTransaction, + pub feerate_threshold: Option, +} + +#[derive(Default)] +pub(crate) struct TransactionPostValidation { + pub removed: Option>, + pub accepted: Option>, +} + #[derive(PartialEq, Eq)] pub(crate) enum TxRemovalReason { Muted, @@ -63,6 +110,7 @@ pub(crate) enum TxRemovalReason { DoubleSpend, InvalidInBlockTemplate, RevalidationWithMissingOutpoints, + ReplacedByFee, } impl TxRemovalReason { @@ -76,6 +124,7 @@ impl TxRemovalReason { TxRemovalReason::DoubleSpend => "double spend", TxRemovalReason::InvalidInBlockTemplate => "invalid in block template", TxRemovalReason::RevalidationWithMissingOutpoints => "revalidation with missing outpoints", + TxRemovalReason::ReplacedByFee => "replaced by fee", } } diff --git a/mining/src/mempool/model/utxo_set.rs b/mining/src/mempool/model/utxo_set.rs index 6b647d2..32f026a 100644 --- a/mining/src/mempool/model/utxo_set.rs +++ b/mining/src/mempool/model/utxo_set.rs @@ -1,7 +1,9 @@ +use std::collections::HashSet; + use crate::{ mempool::{ - errors::{RuleError, RuleResult}, - model::map::OutpointIndex, + errors::RuleResult, + model::{map::OutpointIndex, tx::DoubleSpend}, }, model::TransactionIdSet, }; @@ -70,14 +72,36 @@ impl MempoolUtxoSet { /// Make sure no other transaction in the mempool is already spending an output which one of this transaction inputs spends pub(crate) fn check_double_spends(&self, transaction: &MutableTransaction) -> RuleResult<()> { + match self.get_first_double_spend(transaction) { + Some(double_spend) => Err(double_spend.into()), + None => Ok(()), + } + } + + pub(crate) fn get_first_double_spend(&self, transaction: &MutableTransaction) -> Option { let transaction_id = transaction.id(); for input in transaction.tx.inputs.iter() { if let Some(existing_transaction_id) = self.get_outpoint_owner_id(&input.previous_outpoint) { if *existing_transaction_id != transaction_id { - return Err(RuleError::RejectDoubleSpendInMempool(input.previous_outpoint, *existing_transaction_id)); + return Some(DoubleSpend::new(input.previous_outpoint, *existing_transaction_id)); + } + } + } + None + } + + /// Returns the first double spend of every transaction in the mempool double spending on `transaction` + pub(crate) fn get_double_spend_transaction_ids(&self, transaction: &MutableTransaction) -> Vec { + let transaction_id = transaction.id(); + let mut double_spends = vec![]; + let mut visited = HashSet::new(); + for input in transaction.tx.inputs.iter() { + if let Some(existing_transaction_id) = self.get_outpoint_owner_id(&input.previous_outpoint) { + if *existing_transaction_id != transaction_id && visited.insert(*existing_transaction_id) { + double_spends.push(DoubleSpend::new(input.previous_outpoint, *existing_transaction_id)); } } } - Ok(()) + double_spends } } diff --git a/mining/src/mempool/populate_entries_and_try_validate.rs b/mining/src/mempool/populate_entries_and_try_validate.rs index 48078c4..4d31ac9 100644 --- a/mining/src/mempool/populate_entries_and_try_validate.rs +++ b/mining/src/mempool/populate_entries_and_try_validate.rs @@ -1,5 +1,12 @@ use crate::mempool::{errors::RuleResult, model::pool::Pool, Mempool}; -use spectre_consensus_core::{api::ConsensusApi, constants::UNACCEPTED_DAA_SCORE, tx::MutableTransaction, tx::UtxoEntry}; +use spectre_consensus_core::{ + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, + constants::UNACCEPTED_DAA_SCORE, + tx::{MutableTransaction, UtxoEntry}, +}; use spectre_mining_errors::mempool::RuleError; impl Mempool { @@ -14,15 +21,20 @@ impl Mempool { } } -pub(crate) fn validate_mempool_transaction(consensus: &dyn ConsensusApi, transaction: &mut MutableTransaction) -> RuleResult<()> { - Ok(consensus.validate_mempool_transaction(transaction)?) +pub(crate) fn validate_mempool_transaction( + consensus: &dyn ConsensusApi, + transaction: &mut MutableTransaction, + args: &TransactionValidationArgs, +) -> RuleResult<()> { + Ok(consensus.validate_mempool_transaction(transaction, args)?) } pub(crate) fn validate_mempool_transactions_in_parallel( consensus: &dyn ConsensusApi, transactions: &mut [MutableTransaction], + args: &TransactionValidationBatchArgs, ) -> Vec> { - consensus.validate_mempool_transactions_in_parallel(transactions).into_iter().map(|x| x.map_err(RuleError::from)).collect() + consensus.validate_mempool_transactions_in_parallel(transactions, args).into_iter().map(|x| x.map_err(RuleError::from)).collect() } pub(crate) fn populate_mempool_transactions_in_parallel( diff --git a/mining/src/mempool/replace_by_fee.rs b/mining/src/mempool/replace_by_fee.rs new file mode 100644 index 0000000..7b9d66b --- /dev/null +++ b/mining/src/mempool/replace_by_fee.rs @@ -0,0 +1,149 @@ +use crate::mempool::{ + errors::{RuleError, RuleResult}, + model::tx::{DoubleSpend, MempoolTransaction, TxRemovalReason}, + tx::RbfPolicy, + Mempool, +}; +use spectre_consensus_core::tx::{MutableTransaction, Transaction}; +use std::sync::Arc; + +impl Mempool { + /// Returns the replace by fee (RBF) constraint fee/mass threshold for an incoming transaction and a policy. + /// + /// Fails if the transaction does not meet some condition of the RBF policy, excluding the fee/mass condition. + /// + /// See [`RbfPolicy`] variants for details of each policy process and success conditions. + pub(super) fn get_replace_by_fee_constraint( + &self, + transaction: &MutableTransaction, + rbf_policy: RbfPolicy, + ) -> RuleResult> { + match rbf_policy { + RbfPolicy::Forbidden => { + // When RBF is forbidden, fails early on any double spend + self.transaction_pool.check_double_spends(transaction)?; + Ok(None) + } + + RbfPolicy::Allowed => { + // When RBF is allowed, never fails since both insertion and replacement are possible + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + if double_spends.is_empty() { + Ok(None) + } else { + let mut feerate_threshold = 0f64; + for double_spend in double_spends { + // We take the max over all double spends as the required threshold + feerate_threshold = feerate_threshold.max(self.get_double_spend_feerate(&double_spend)?); + } + Ok(Some(feerate_threshold)) + } + } + + RbfPolicy::Mandatory => { + // When RBF is mandatory, fails early if we do not have exactly one double spending transaction + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.len() { + 0 => Err(RuleError::RejectRbfNoDoubleSpend), + 1 => { + let feerate_threshold = self.get_double_spend_feerate(&double_spends[0])?; + Ok(Some(feerate_threshold)) + } + _ => Err(RuleError::RejectRbfTooManyDoubleSpendingTransactions), + } + } + } + } + + /// Executes replace by fee (RBF) for an incoming transaction and a policy. + /// + /// See [`RbfPolicy`] variants for details of each policy process and success conditions. + /// + /// On success, `transaction` is guaranteed to embed no double spend with the mempool. + /// + /// On success with the [`RbfPolicy::Mandatory`] policy, some removed transaction is always returned. + pub(super) fn execute_replace_by_fee( + &mut self, + transaction: &MutableTransaction, + rbf_policy: RbfPolicy, + ) -> RuleResult>> { + match rbf_policy { + RbfPolicy::Forbidden => { + self.transaction_pool.check_double_spends(transaction)?; + Ok(None) + } + + RbfPolicy::Allowed => { + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.is_empty() { + true => Ok(None), + false => { + let removed = self.validate_double_spending_transaction(transaction, &double_spends[0])?.mtx.tx.clone(); + for double_spend in double_spends.iter().skip(1) { + // Validate the feerate threshold is passed for all double spends + self.validate_double_spending_transaction(transaction, double_spend)?; + } + // We apply consequences such as removal only after we fully validate against all double spends + for double_spend in double_spends { + self.remove_transaction( + &double_spend.owner_id, + true, + TxRemovalReason::ReplacedByFee, + format!("by {}", transaction.id()).as_str(), + )?; + } + Ok(Some(removed)) + } + } + } + + RbfPolicy::Mandatory => { + let double_spends = self.transaction_pool.get_double_spend_transaction_ids(transaction); + match double_spends.len() { + 0 => Err(RuleError::RejectRbfNoDoubleSpend), + 1 => { + let removed = self.validate_double_spending_transaction(transaction, &double_spends[0])?.mtx.tx.clone(); + self.remove_transaction( + &double_spends[0].owner_id, + true, + TxRemovalReason::ReplacedByFee, + format!("by {}", transaction.id()).as_str(), + )?; + Ok(Some(removed)) + } + _ => Err(RuleError::RejectRbfTooManyDoubleSpendingTransactions), + } + } + } + } + + fn get_double_spend_feerate(&self, double_spend: &DoubleSpend) -> RuleResult { + let owner = self.transaction_pool.get_double_spend_owner(double_spend)?; + match owner.mtx.calculated_feerate() { + Some(double_spend_feerate) => Ok(double_spend_feerate), + // Getting here is unexpected since a mempool owned tx should be populated with fee + // and mass at this stage but nonetheless we fail gracefully + None => Err(double_spend.into()), + } + } + + fn validate_double_spending_transaction<'a>( + &'a self, + transaction: &MutableTransaction, + double_spend: &DoubleSpend, + ) -> RuleResult<&'a MempoolTransaction> { + let owner = self.transaction_pool.get_double_spend_owner(double_spend)?; + if let (Some(transaction_feerate), Some(double_spend_feerate)) = + (transaction.calculated_feerate(), owner.mtx.calculated_feerate()) + { + if transaction_feerate > double_spend_feerate { + return Ok(owner); + } else { + return Err(double_spend.into()); + } + } + // Getting here is unexpected since both txs should be populated with + // fee and mass at this stage but nonetheless we fail gracefully + Err(double_spend.into()) + } +} diff --git a/mining/src/mempool/validate_and_insert_transaction.rs b/mining/src/mempool/validate_and_insert_transaction.rs index 41dd9cf..a413d3a 100644 --- a/mining/src/mempool/validate_and_insert_transaction.rs +++ b/mining/src/mempool/validate_and_insert_transaction.rs @@ -2,9 +2,9 @@ use crate::mempool::{ errors::{RuleError, RuleResult}, model::{ pool::Pool, - tx::{MempoolTransaction, TxRemovalReason}, + tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, }, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, Mempool, }; use spectre_consensus_core::{ @@ -13,21 +13,21 @@ use spectre_consensus_core::{ tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint, UtxoEntry}, }; use spectre_core::{debug, info}; -use std::sync::Arc; impl Mempool { pub(crate) fn pre_validate_and_populate_transaction( &self, consensus: &dyn ConsensusApi, mut transaction: MutableTransaction, - ) -> RuleResult { + rbf_policy: RbfPolicy, + ) -> RuleResult { self.validate_transaction_unacceptance(&transaction)?; // Populate mass in the beginning, it will be used in multiple places throughout the validation and insertion. transaction.calculated_compute_mass = Some(consensus.calculate_transaction_compute_mass(&transaction.tx)); self.validate_transaction_in_isolation(&transaction)?; - self.transaction_pool.check_double_spends(&transaction)?; + let feerate_threshold = self.get_replace_by_fee_constraint(&transaction, rbf_policy)?; self.populate_mempool_entries(&mut transaction); - Ok(transaction) + Ok(TransactionPreValidation { transaction, feerate_threshold }) } pub(crate) fn post_validate_and_insert_transaction( @@ -37,7 +37,8 @@ impl Mempool { transaction: MutableTransaction, priority: Priority, orphan: Orphan, - ) -> RuleResult>> { + rbf_policy: RbfPolicy, + ) -> RuleResult { let transaction_id = transaction.id(); // First check if the transaction was not already added to the mempool. @@ -46,28 +47,29 @@ impl Mempool { // concurrently. if self.transaction_pool.has(&transaction_id) { debug!("Transaction {0} is not post validated since already in the mempool", transaction_id); - return Ok(None); + return Err(RuleError::RejectDuplicate(transaction_id)); } self.validate_transaction_unacceptance(&transaction)?; - // Re-check double spends since validate_and_insert_transaction is no longer atomic - self.transaction_pool.check_double_spends(&transaction)?; - match validation_result { Ok(_) => {} Err(RuleError::RejectMissingOutpoint) => { if orphan == Orphan::Forbidden { return Err(RuleError::RejectDisallowedOrphan(transaction_id)); } + let _ = self.get_replace_by_fee_constraint(&transaction, rbf_policy)?; self.orphan_pool.try_add_orphan(consensus.get_virtual_daa_score(), transaction, priority)?; - return Ok(None); + return Ok(TransactionPostValidation::default()); } Err(err) => { return Err(err); } } + // Check double spends and try to remove them if the RBF policy requires it + let removed_transaction = self.execute_replace_by_fee(&transaction, rbf_policy)?; + self.validate_transaction_in_context(&transaction)?; // Before adding the transaction, check if there is room in the pool @@ -78,7 +80,7 @@ impl Mempool { // Add the transaction to the mempool as a MempoolTransaction and return a clone of the embedded Arc let accepted_transaction = self.transaction_pool.add_transaction(transaction, consensus.get_virtual_daa_score(), priority)?.mtx.tx.clone(); - Ok(Some(accepted_transaction)) + Ok(TransactionPostValidation { removed: removed_transaction, accepted: Some(accepted_transaction) }) } /// Validates that the transaction wasn't already accepted into the DAG @@ -184,9 +186,26 @@ impl Mempool { // The one we just removed from the orphan pool. assert_eq!(transactions.len(), 1, "the list returned by remove_orphan is expected to contain exactly one transaction"); let transaction = transactions.pop().unwrap(); + let rbf_policy = Self::get_orphan_transaction_rbf_policy(transaction.priority); self.validate_transaction_unacceptance(&transaction.mtx)?; - self.transaction_pool.check_double_spends(&transaction.mtx)?; + let _ = self.get_replace_by_fee_constraint(&transaction.mtx, rbf_policy)?; Ok(transaction) } + + /// Returns the RBF policy to apply to an orphan/unorphaned transaction by inferring it from the transaction priority. + pub(crate) fn get_orphan_transaction_rbf_policy(priority: Priority) -> RbfPolicy { + // The RBF policy applied to an orphaned transaction is not recorded in the orphan pool + // but we can infer it from the priority: + // + // - high means a submitted tx via RPC which forbids RBF + // - low means a tx arrived via P2P which allows RBF + // + // Note that the RPC submit transaction replacement case, implying a mandatory RBF, forbids orphans + // so is excluded here. + match priority { + Priority::High => RbfPolicy::Forbidden, + Priority::Low => RbfPolicy::Allowed, + } + } } diff --git a/mining/src/model/mod.rs b/mining/src/model/mod.rs index a98e9e1..0c6caee 100644 --- a/mining/src/model/mod.rs +++ b/mining/src/model/mod.rs @@ -5,6 +5,7 @@ pub(crate) mod candidate_tx; pub mod owner_txs; pub mod topological_index; pub mod topological_sort; +pub mod tx_insert; pub mod tx_query; /// A set of unique transaction ids diff --git a/mining/src/model/tx_insert.rs b/mining/src/model/tx_insert.rs new file mode 100644 index 0000000..d8abdf8 --- /dev/null +++ b/mining/src/model/tx_insert.rs @@ -0,0 +1,14 @@ +use spectre_consensus_core::tx::Transaction; +use std::sync::Arc; + +#[derive(Debug)] +pub struct TransactionInsertion { + pub removed: Option>, + pub accepted: Vec>, +} + +impl TransactionInsertion { + pub fn new(removed: Option>, accepted: Vec>) -> Self { + Self { removed, accepted } + } +} diff --git a/mining/src/testutils/consensus_mock.rs b/mining/src/testutils/consensus_mock.rs index 4647507..09be800 100644 --- a/mining/src/testutils/consensus_mock.rs +++ b/mining/src/testutils/consensus_mock.rs @@ -1,6 +1,9 @@ use super::coinbase_mock::CoinbaseManagerMock; use spectre_consensus_core::{ - api::ConsensusApi, + api::{ + args::{TransactionValidationArgs, TransactionValidationBatchArgs}, + ConsensusApi, + }, block::{BlockTemplate, MutableBlock, TemplateBuildMode, TemplateTransactionSelector, VirtualStateApproxId}, coinbase::MinerData, constants::BLOCK_VERSION, @@ -103,7 +106,7 @@ impl ConsensusApi for ConsensusMock { Ok(BlockTemplate::new(mutable_block, miner_data, coinbase.has_red_reward, now, 0, ZERO_HASH)) } - fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction) -> TxResult<()> { + fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction, _: &TransactionValidationArgs) -> TxResult<()> { // If a predefined status was registered to simulate an error, return it right away if let Some(status) = self.statuses.read().get(&mutable_tx.id()) { if status.is_err() { @@ -138,12 +141,16 @@ impl ConsensusApi for ConsensusMock { Ok(()) } - fn validate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - transactions.iter_mut().map(|x| self.validate_mempool_transaction(x)).collect() + fn validate_mempool_transactions_in_parallel( + &self, + transactions: &mut [MutableTransaction], + _: &TransactionValidationBatchArgs, + ) -> Vec> { + transactions.iter_mut().map(|x| self.validate_mempool_transaction(x, &Default::default())).collect() } fn populate_mempool_transactions_in_parallel(&self, transactions: &mut [MutableTransaction]) -> Vec> { - transactions.iter_mut().map(|x| self.validate_mempool_transaction(x)).collect() + transactions.iter_mut().map(|x| self.validate_mempool_transaction(x, &Default::default())).collect() } fn calculate_transaction_compute_mass(&self, transaction: &Transaction) -> u64 { diff --git a/protocol/flows/src/flow_context.rs b/protocol/flows/src/flow_context.rs index 82b8341..940d8b1 100644 --- a/protocol/flows/src/flow_context.rs +++ b/protocol/flows/src/flow_context.rs @@ -26,8 +26,8 @@ use spectre_core::{ }; use spectre_core::{time::unix_now, warn}; use spectre_hashes::Hash; -use spectre_mining::manager::MiningManagerProxy; use spectre_mining::mempool::tx::{Orphan, Priority}; +use spectre_mining::{manager::MiningManagerProxy, mempool::tx::RbfPolicy}; use spectre_notify::notifier::Notify; use spectre_p2p_lib::{ common::ProtocolError, @@ -618,16 +618,48 @@ impl FlowContext { transaction: Transaction, orphan: Orphan, ) -> Result<(), ProtocolError> { - let accepted_transactions = - self.mining_manager().clone().validate_and_insert_transaction(consensus, transaction, Priority::High, orphan).await?; + let transaction_insertion = self + .mining_manager() + .clone() + .validate_and_insert_transaction(consensus, transaction, Priority::High, orphan, RbfPolicy::Forbidden) + .await?; self.broadcast_transactions( - accepted_transactions.iter().map(|x| x.id()), + transaction_insertion.accepted.iter().map(|x| x.id()), false, // RPC transactions are considered high priority, so we don't want to throttle them ) .await; Ok(()) } + /// Replaces the rpc-submitted transaction into the mempool and propagates it to peers. + /// + /// Returns the removed mempool transaction on successful replace by fee. + /// + /// Transactions submitted through rpc are considered high priority. This definition does not affect the tx selection algorithm + /// but only changes how we manage the lifetime of the tx. A high-priority tx does not expire and is repeatedly rebroadcasted to + /// peers + pub async fn submit_rpc_transaction_replacement( + &self, + consensus: &ConsensusProxy, + transaction: Transaction, + ) -> Result, ProtocolError> { + let transaction_insertion = self + .mining_manager() + .clone() + .validate_and_insert_transaction(consensus, transaction, Priority::High, Orphan::Forbidden, RbfPolicy::Mandatory) + .await?; + self.broadcast_transactions( + transaction_insertion.accepted.iter().map(|x| x.id()), + false, // RPC transactions are considered high priority, so we don't want to throttle them + ) + .await; + // The combination of args above of Orphan::Forbidden and RbfPolicy::Mandatory should always result + // in a removed transaction returned, however we prefer failing gracefully in case of future internal mempool changes + transaction_insertion.removed.ok_or(ProtocolError::Other( + "Replacement transaction was actually accepted but the *replaced* transaction was not returned from the mempool", + )) + } + /// Returns true if the time has come for running the task cleaning mempool transactions. async fn should_run_mempool_scanning_task(&self) -> bool { self.transactions_spread.write().await.should_run_mempool_scanning_task() diff --git a/protocol/flows/src/v5/txrelay/flow.rs b/protocol/flows/src/v5/txrelay/flow.rs index 7911d55..43829fa 100644 --- a/protocol/flows/src/v5/txrelay/flow.rs +++ b/protocol/flows/src/v5/txrelay/flow.rs @@ -10,7 +10,7 @@ use spectre_mining::{ errors::MiningManagerError, mempool::{ errors::RuleError, - tx::{Orphan, Priority}, + tx::{Orphan, Priority, RbfPolicy}, }, model::tx_query::TransactionQuery, P2pTxCountSample, @@ -219,7 +219,7 @@ impl RelayTransactionsFlow { .ctx .mining_manager() .clone() - .validate_and_insert_transaction_batch(&consensus, transactions, Priority::Low, Orphan::Allowed) + .validate_and_insert_transaction_batch(&consensus, transactions, Priority::Low, Orphan::Allowed, RbfPolicy::Allowed) .await; for res in insert_results.iter() { diff --git a/rothschild/src/main.rs b/rothschild/src/main.rs index 2f2b735..6f9256f 100644 --- a/rothschild/src/main.rs +++ b/rothschild/src/main.rs @@ -21,7 +21,7 @@ use spectre_txscript::pay_to_address_script; use tokio::time::{interval, MissedTickBehavior}; const DEFAULT_SEND_AMOUNT: u64 = 10 * SOMPI_PER_SPECTRE; -const FEE_PER_MASS: u64 = 10; +const FEE_RATE: u64 = 10; const MILLIS_PER_TICK: u64 = 10; const ADDRESS_PREFIX: Prefix = Prefix::Testnet; const ADDRESS_VERSION: Version = Version::PubKey; @@ -438,7 +438,7 @@ fn clean_old_pending_outpoints(pending: &mut HashMap) } fn required_fee(num_utxos: usize, num_outs: u64) -> u64 { - FEE_PER_MASS * estimated_mass(num_utxos, num_outs) + FEE_RATE * estimated_mass(num_utxos, num_outs) } fn estimated_mass(num_utxos: usize, num_outs: u64) -> u64 { diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index f29cf62..e8df3dc 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -114,6 +114,9 @@ pub enum RpcApiOps { VirtualDaaScoreChangedNotification, PruningPointUtxoSetOverrideNotification, NewBlockTemplateNotification, + + /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy + SubmitTransactionReplacement, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index a48e709..f4588bb 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -132,6 +132,17 @@ pub trait RpcApi: Sync + Send + AnySync { } async fn submit_transaction_call(&self, request: SubmitTransactionRequest) -> RpcResult; + /// Submits a transaction replacement to the mempool, applying a mandatory Replace by Fee policy. + /// + /// Returns the ID of the inserted transaction and the transaction the submission replaced in the mempool. + async fn submit_transaction_replacement(&self, transaction: RpcTransaction) -> RpcResult { + self.submit_transaction_replacement_call(SubmitTransactionReplacementRequest { transaction }).await + } + async fn submit_transaction_replacement_call( + &self, + request: SubmitTransactionReplacementRequest, + ) -> RpcResult; + /// Requests information about a specific block. async fn get_block(&self, hash: RpcHash, include_transactions: bool) -> RpcResult { Ok(self.get_block_call(GetBlockRequest::new(hash, include_transactions)).await?.block) diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index 14d0ef4..f1cdc66 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -299,6 +299,31 @@ impl SubmitTransactionResponse { } } +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmitTransactionReplacementRequest { + pub transaction: RpcTransaction, +} + +impl SubmitTransactionReplacementRequest { + pub fn new(transaction: RpcTransaction) -> Self { + Self { transaction } + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct SubmitTransactionReplacementResponse { + pub transaction_id: RpcTransactionId, + pub replaced_transaction: RpcTransaction, +} + +impl SubmitTransactionReplacementResponse { + pub fn new(transaction_id: RpcTransactionId, replaced_transaction: RpcTransaction) -> Self { + Self { transaction_id, replaced_transaction } + } +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct GetSubnetworkRequest { diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index b96ccf1..1902e5c 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -253,6 +253,7 @@ impl RpcApi for GrpcClient { route!(get_connected_peer_info_call, GetConnectedPeerInfo); route!(add_peer_call, AddPeer); route!(submit_transaction_call, SubmitTransaction); + route!(submit_transaction_replacement_call, SubmitTransactionReplacement); route!(get_subnetwork_call, GetSubnetwork); route!(get_virtual_chain_from_block_call, GetVirtualChainFromBlock); route!(get_blocks_call, GetBlocks); diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index 80ea46d..21c9332 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -58,7 +58,8 @@ message SpectredRequest { GetMetricsRequestMessage getMetricsRequest = 1090; GetServerInfoRequestMessage getServerInfoRequest = 1092; GetSyncStatusRequestMessage getSyncStatusRequest = 1094; - GetDaaScoreTimestampEstimateRequestMessage GetDaaScoreTimestampEstimateRequest = 1096; + GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; + SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; } } @@ -117,7 +118,8 @@ message SpectredResponse { GetMetricsResponseMessage getMetricsResponse= 1091; GetServerInfoResponseMessage getServerInfoResponse = 1093; GetSyncStatusResponseMessage getSyncStatusResponse = 1095; - GetDaaScoreTimestampEstimateResponseMessage GetDaaScoreTimestampEstimateResponse = 1097; + GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; + SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; } } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 30d3896..5c0f8b1 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -307,6 +307,21 @@ message SubmitTransactionResponseMessage{ RPCError error = 1000; } +// SubmitTransactionReplacementRequestMessage submits a transaction to the mempool, applying a mandatory Replace by Fee policy +message SubmitTransactionReplacementRequestMessage{ + RpcTransaction transaction = 1; +} + +message SubmitTransactionReplacementResponseMessage{ + // The transaction ID of the submitted transaction + string transactionId = 1; + + // The previous transaction replaced in the mempool by the newly submitted one + RpcTransaction replacedTransaction = 2; + + RPCError error = 1000; +} + // NotifyVirtualChainChangedRequestMessage registers this connection for virtualChainChanged notifications. // // See: VirtualChainChangedNotificationMessage @@ -844,10 +859,10 @@ message GetSyncStatusResponseMessage{ } message GetDaaScoreTimestampEstimateRequestMessage { - repeated uint64 daa_scores = 1; + repeated uint64 daa_scores = 1; } message GetDaaScoreTimestampEstimateResponseMessage{ - repeated uint64 timestamps = 1; - RPCError error = 1000; + repeated uint64 timestamps = 1; + RPCError error = 1000; } diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index bfe7501..1f737ed 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -248,6 +248,13 @@ from!(item: RpcResult<&spectre_rpc_core::SubmitTransactionResponse>, protowire:: Self { transaction_id: item.transaction_id.to_string(), error: None } }); +from!(item: &spectre_rpc_core::SubmitTransactionReplacementRequest, protowire::SubmitTransactionReplacementRequestMessage, { + Self { transaction: Some((&item.transaction).into()) } +}); +from!(item: RpcResult<&spectre_rpc_core::SubmitTransactionReplacementResponse>, protowire::SubmitTransactionReplacementResponseMessage, { + Self { transaction_id: item.transaction_id.to_string(), replaced_transaction: Some((&item.replaced_transaction).into()), error: None } +}); + from!(item: &spectre_rpc_core::GetSubnetworkRequest, protowire::GetSubnetworkRequestMessage, { Self { subnetwork_id: item.subnetwork_id.to_string() } }); @@ -647,6 +654,26 @@ try_from!(item: &protowire::SubmitTransactionResponseMessage, RpcResult, { + Self { + transaction_id: RpcHash::from_str(&item.transaction_id)?, + replaced_transaction: item + .replaced_transaction + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("SubmitTransactionReplacementRequestMessage".to_string(), "replaced_transaction".to_string()))? + .try_into()?, + } +}); + try_from!(item: &protowire::GetSubnetworkRequestMessage, spectre_rpc_core::GetSubnetworkRequest, { Self { subnetwork_id: spectre_rpc_core::RpcSubnetworkId::from_str(&item.subnetwork_id)? } }); diff --git a/rpc/grpc/core/src/convert/spectred.rs b/rpc/grpc/core/src/convert/spectred.rs index 3fd6fbe..195de3d 100644 --- a/rpc/grpc/core/src/convert/spectred.rs +++ b/rpc/grpc/core/src/convert/spectred.rs @@ -36,6 +36,7 @@ pub mod spectred_request_convert { impl_into_spectred_request!(GetConnectedPeerInfo); impl_into_spectred_request!(AddPeer); impl_into_spectred_request!(SubmitTransaction); + impl_into_spectred_request!(SubmitTransactionReplacement); impl_into_spectred_request!(GetSubnetwork); impl_into_spectred_request!(GetVirtualChainFromBlock); impl_into_spectred_request!(GetBlocks); @@ -167,6 +168,7 @@ pub mod spectred_response_convert { impl_into_spectred_response!(GetConnectedPeerInfo); impl_into_spectred_response!(AddPeer); impl_into_spectred_response!(SubmitTransaction); + impl_into_spectred_response!(SubmitTransactionReplacement); impl_into_spectred_response!(GetSubnetwork); impl_into_spectred_response!(GetVirtualChainFromBlock); impl_into_spectred_response!(GetBlocks); diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index ead310e..1607645 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -61,6 +61,7 @@ pub enum SpectredPayloadOps { GetConnectedPeerInfo, AddPeer, SubmitTransaction, + SubmitTransactionReplacement, GetSubnetwork, GetVirtualChainFromBlock, GetBlockCount, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index a13e9b6..a2cfd2a 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -55,6 +55,7 @@ impl Factory { GetConnectedPeerInfo, AddPeer, SubmitTransaction, + SubmitTransactionReplacement, GetSubnetwork, GetVirtualChainFromBlock, GetBlockCount, diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index ba1de2f..0ab19a0 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -134,6 +134,13 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn submit_transaction_replacement_call( + &self, + _request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { Err(RpcError::NotImplemented) } diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index 633b322..2d9b23f 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -506,6 +506,22 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(SubmitTransactionResponse::new(transaction_id)) } + async fn submit_transaction_replacement_call( + &self, + request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + let transaction: Transaction = (&request.transaction).try_into()?; + let transaction_id = transaction.id(); + let session = self.consensus_manager.consensus().unguarded_session(); + let replaced_transaction = + self.flow_context.submit_rpc_transaction_replacement(&session, transaction).await.map_err(|err| { + let err = RpcError::RejectedTransaction(transaction_id, err.to_string()); + debug!("{err}"); + err + })?; + Ok(SubmitTransactionReplacementResponse::new(transaction_id, (&*replaced_transaction).into())) + } + async fn get_current_network_call(&self, _: GetCurrentNetworkRequest) -> RpcResult { Ok(GetCurrentNetworkResponse::new(*self.config.net)) } diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index ca84e53..775c1f9 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -617,6 +617,7 @@ impl RpcApi for SpectreRpcClient { Shutdown, SubmitBlock, SubmitTransaction, + SubmitTransactionReplacement, Unban, ] ); diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index 6cbe349..abf806a 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -66,6 +66,7 @@ impl Router { Shutdown, SubmitBlock, SubmitTransaction, + SubmitTransactionReplacement, Unban, ] ); diff --git a/testing/integration/src/common/utils.rs b/testing/integration/src/common/utils.rs index 4049fcd..96e39d9 100644 --- a/testing/integration/src/common/utils.rs +++ b/testing/integration/src/common/utils.rs @@ -36,8 +36,8 @@ const fn estimated_mass(num_inputs: usize, num_outputs: u64) -> u64 { } pub const fn required_fee(num_inputs: usize, num_outputs: u64) -> u64 { - const FEE_PER_MASS: u64 = 10; - FEE_PER_MASS * estimated_mass(num_inputs, num_outputs) + const FEE_RATE: u64 = 10; + FEE_RATE * estimated_mass(num_inputs, num_outputs) } /// Builds a TX DAG based on the initial UTXO set and on constant params diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index 0d0c87f..c2505ea 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -301,6 +301,17 @@ async fn sanity_test() { }) } + SpectredPayloadOps::SubmitTransactionReplacement => { + let rpc_client = client.clone(); + tst!(op, { + // Build an erroneous transaction... + let transaction = Transaction::new(0, vec![], vec![], 0, SubnetworkId::default(), 0, vec![]); + let result = rpc_client.submit_transaction_replacement((&transaction).into()).await; + // ...that gets rejected by the consensus + assert!(result.is_err()); + }) + } + SpectredPayloadOps::GetSubnetwork => { let rpc_client = client.clone(); tst!(op, { diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index 9ac1e22..171a08b 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -151,6 +151,13 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn submit_transaction_replacement_call( + &self, + _request: SubmitTransactionReplacementRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { Err(RpcError::NotImplemented) } From 817a3db7b89c7f364298952e73dea4bec8c0b13f Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:08:52 +0200 Subject: [PATCH 05/48] input signature_script checking to submitTransaction --- consensus/Cargo.toml | 1 + consensus/core/src/errors/tx.rs | 3 + .../transaction_validator_populated.rs | 18 +- rpc/core/src/error.rs | 304 +++++++++--------- 4 files changed, 171 insertions(+), 155 deletions(-) diff --git a/consensus/Cargo.toml b/consensus/Cargo.toml index f3a2efb..940686c 100644 --- a/consensus/Cargo.toml +++ b/consensus/Cargo.toml @@ -30,6 +30,7 @@ spectre-muhash.workspace = true spectre-notify.workspace = true spectre-pow.workspace = true spectre-txscript.workspace = true +spectre-txscript-errors.workspace = true spectre-utils.workspace = true log.workspace = true once_cell.workspace = true diff --git a/consensus/core/src/errors/tx.rs b/consensus/core/src/errors/tx.rs index 3f41b69..25e215e 100644 --- a/consensus/core/src/errors/tx.rs +++ b/consensus/core/src/errors/tx.rs @@ -80,6 +80,9 @@ pub enum TxRuleError { #[error("failed to verify the signature script: {0}")] SignatureInvalid(TxScriptError), + #[error("failed to verify empty signature script. Inner error: {0}")] + SignatureEmpty(TxScriptError), + #[error("input {0} sig op count is {1}, but the calculated value is {2}")] WrongSigOpCount(usize, u64, u64), diff --git a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs index 16ed8cc..a71ae44 100644 --- a/consensus/src/processes/transaction_validator/transaction_validator_populated.rs +++ b/consensus/src/processes/transaction_validator/transaction_validator_populated.rs @@ -1,7 +1,11 @@ use crate::constants::{MAX_SOMPI, SEQUENCE_LOCK_TIME_DISABLED, SEQUENCE_LOCK_TIME_MASK}; -use spectre_consensus_core::{hashing::sighash::SigHashReusedValues, tx::VerifiableTransaction}; +use spectre_consensus_core::{ + hashing::sighash::SigHashReusedValues, + tx::{TransactionInput, VerifiableTransaction}, +}; use spectre_core::warn; use spectre_txscript::{get_sig_op_count, TxScriptEngine}; +use spectre_txscript_errors::TxScriptError; use super::{ errors::{TxResult, TxRuleError}, @@ -167,14 +171,22 @@ impl TransactionValidator { let mut reused_values = SigHashReusedValues::new(); for (i, (input, entry)) in tx.populated_inputs().enumerate() { let mut engine = TxScriptEngine::from_transaction_input(tx, input, i, entry, &mut reused_values, &self.sig_cache) - .map_err(TxRuleError::SignatureInvalid)?; - engine.execute().map_err(TxRuleError::SignatureInvalid)?; + .map_err(|err| map_script_err(err, input))?; + engine.execute().map_err(|err| map_script_err(err, input))?; } Ok(()) } } +fn map_script_err(script_err: TxScriptError, input: &TransactionInput) -> TxRuleError { + if input.signature_script.is_empty() { + TxRuleError::SignatureEmpty(script_err) + } else { + TxRuleError::SignatureInvalid(script_err) + } +} + #[cfg(test)] mod tests { use super::super::errors::TxRuleError; diff --git a/rpc/core/src/error.rs b/rpc/core/src/error.rs index 0a42a17..7f7906d 100644 --- a/rpc/core/src/error.rs +++ b/rpc/core/src/error.rs @@ -1,156 +1,156 @@ use spectre_consensus_core::{subnets::SubnetworkConversionError, tx::TransactionId}; -use spectre_utils::networking::IpAddress; -use std::{net::AddrParseError, num::TryFromIntError}; -use thiserror::Error; -use workflow_core::channel::ChannelError; - -use crate::{api::ctl::RpcState, RpcHash, RpcTransactionId, SubmitBlockRejectReason}; - -#[derive(Clone, Debug, Error)] -pub enum RpcError { - #[error("Not implemented")] - NotImplemented, - - #[error("Integer downsize conversion error {0}")] - IntConversionError(#[from] TryFromIntError), - - #[error("Hex parsing error: {0}")] - HexParsingError(#[from] faster_hex::Error), - - #[error("Blue work parsing error {0}")] - RpcBlueWorkTypeParseError(std::num::ParseIntError), - - #[error("Integer parsing error: {0}")] - ParseIntError(#[from] std::num::ParseIntError), - - #[error("Ip address parsing error {0}")] - ParseIpAddressError(#[from] AddrParseError), - - #[error("Wrong rpc api version format")] - RpcApiVersionFormatError, - - #[error("Invalid script class: {0}")] - InvalidRpcScriptClass(String), - - #[error("Missing required field {0}.{1}")] - MissingRpcFieldError(String, String), - - #[error("Feature not supported")] - UnsupportedFeature, - - #[error("Primitive to enum conversion error")] - PrimitiveToEnumConversionError, - - #[error("Coinbase payload is above max length ({0}). Try to shorten the extra data.")] - CoinbasePayloadLengthAboveMax(usize), - - #[error("Rejected transaction {0}: {1}")] - RejectedTransaction(RpcTransactionId, String), - - #[error("Block {0} is invalid. No verbose data can be built.")] - InvalidBlock(RpcHash), - - #[error("If includeTransactions is set, then includeBlockVerboseData must be set as well.")] - InvalidGetBlocksRequest, - - #[error("Transaction {0} not found")] - TransactionNotFound(TransactionId), - - #[error("Method unavailable. Run the node with the --utxoindex argument.")] - NoUtxoIndex, - - #[error("Method unavailable. No connection manager is currently available.")] - NoConnectionManager, - - #[error("Requested window size {0} is larger than max {1} allowed in RPC safe mode.")] - WindowSizeExceedingMaximum(u32, u32), - - #[error("Requested window size {0} is larger than pruning point depth {1}.")] - WindowSizeExceedingPruningDepth(u32, u64), - - #[error("Method unavailable in safe mode. Run the node with --unsaferpc argument.")] - UnavailableInSafeMode, - - #[error("Cannot ban IP {0} because it has some permanent connection.")] - IpHasPermanentConnection(IpAddress), - - #[error("IP {0} is not registered as banned.")] - IpIsNotBanned(IpAddress), - - #[error("Block was not submitted: {0}")] - SubmitBlockError(SubmitBlockRejectReason), - - #[error(transparent)] - AddressError(#[from] spectre_addresses::AddressError), - - #[error(transparent)] - NetworkTypeError(#[from] spectre_consensus_core::network::NetworkTypeError), - - #[error(transparent)] - NetworkIdError(#[from] spectre_consensus_core::network::NetworkIdError), - - #[error(transparent)] - NotificationError(#[from] spectre_notify::error::Error), - - #[error(transparent)] - MiningManagerError(#[from] spectre_mining_errors::manager::MiningManagerError), - - #[error(transparent)] - ConsensusError(#[from] spectre_consensus_core::errors::consensus::ConsensusError), - - #[error(transparent)] - ScriptClassError(#[from] spectre_txscript::script_class::Error), - - #[error(transparent)] - NodeIdError(#[from] uuid::Error), - - #[error("RPC Server (remote error) -> {0}")] - RpcSubsystem(String), - - #[error("{0}")] - General(String), - - #[error("RpcCtl dispatch error")] - RpcCtlDispatchError, - - #[error("transaction query must either not filter transactions or include orphans")] - InconsistentMempoolTxQuery, - +use spectre_utils::networking::IpAddress; +use std::{net::AddrParseError, num::TryFromIntError}; +use thiserror::Error; +use workflow_core::channel::ChannelError; + +use crate::{api::ctl::RpcState, RpcHash, RpcTransactionId, SubmitBlockRejectReason}; + +#[derive(Clone, Debug, Error)] +pub enum RpcError { + #[error("Not implemented")] + NotImplemented, + + #[error("Integer downsize conversion error {0}")] + IntConversionError(#[from] TryFromIntError), + + #[error("Hex parsing error: {0}")] + HexParsingError(#[from] faster_hex::Error), + + #[error("Blue work parsing error {0}")] + RpcBlueWorkTypeParseError(std::num::ParseIntError), + + #[error("Integer parsing error: {0}")] + ParseIntError(#[from] std::num::ParseIntError), + + #[error("Ip address parsing error {0}")] + ParseIpAddressError(#[from] AddrParseError), + + #[error("Wrong rpc api version format")] + RpcApiVersionFormatError, + + #[error("Invalid script class: {0}")] + InvalidRpcScriptClass(String), + + #[error("Missing required field {0}.{1}")] + MissingRpcFieldError(String, String), + + #[error("Feature not supported")] + UnsupportedFeature, + + #[error("Primitive to enum conversion error")] + PrimitiveToEnumConversionError, + + #[error("Coinbase payload is above max length ({0}). Try to shorten the extra data.")] + CoinbasePayloadLengthAboveMax(usize), + + #[error("Rejected transaction {0}: {1}")] + RejectedTransaction(RpcTransactionId, String), + + #[error("Block {0} is invalid. No verbose data can be built.")] + InvalidBlock(RpcHash), + + #[error("If includeTransactions is set, then includeBlockVerboseData must be set as well.")] + InvalidGetBlocksRequest, + + #[error("Transaction {0} not found")] + TransactionNotFound(TransactionId), + + #[error("Method unavailable. Run the node with the --utxoindex argument.")] + NoUtxoIndex, + + #[error("Method unavailable. No connection manager is currently available.")] + NoConnectionManager, + + #[error("Requested window size {0} is larger than max {1} allowed in RPC safe mode.")] + WindowSizeExceedingMaximum(u32, u32), + + #[error("Requested window size {0} is larger than pruning point depth {1}.")] + WindowSizeExceedingPruningDepth(u32, u64), + + #[error("Method unavailable in safe mode. Run the node with --unsaferpc argument.")] + UnavailableInSafeMode, + + #[error("Cannot ban IP {0} because it has some permanent connection.")] + IpHasPermanentConnection(IpAddress), + + #[error("IP {0} is not registered as banned.")] + IpIsNotBanned(IpAddress), + + #[error("Block was not submitted: {0}")] + SubmitBlockError(SubmitBlockRejectReason), + + #[error(transparent)] + AddressError(#[from] spectre_addresses::AddressError), + + #[error(transparent)] + NetworkTypeError(#[from] spectre_consensus_core::network::NetworkTypeError), + + #[error(transparent)] + NetworkIdError(#[from] spectre_consensus_core::network::NetworkIdError), + + #[error(transparent)] + NotificationError(#[from] spectre_notify::error::Error), + + #[error(transparent)] + MiningManagerError(#[from] spectre_mining_errors::manager::MiningManagerError), + + #[error(transparent)] + ConsensusError(#[from] spectre_consensus_core::errors::consensus::ConsensusError), + + #[error(transparent)] + ScriptClassError(#[from] spectre_txscript::script_class::Error), + + #[error(transparent)] + NodeIdError(#[from] uuid::Error), + + #[error("RPC Server (remote error) -> {0}")] + RpcSubsystem(String), + + #[error("{0}")] + General(String), + + #[error("RpcCtl dispatch error")] + RpcCtlDispatchError, + + #[error("transaction query must either not filter transactions or include orphans")] + InconsistentMempoolTxQuery, + #[error(transparent)] SubnetParsingError(#[from] SubnetworkConversionError), - #[error(transparent)] - WasmError(#[from] workflow_wasm::error::Error), - - #[error("{0}")] - SerdeWasmBindgen(String), - - #[error(transparent)] - ConsensusClient(#[from] spectre_consensus_client::error::Error), -} - -impl From for RpcError { - fn from(value: String) -> Self { - RpcError::General(value) - } -} - -impl From<&str> for RpcError { - fn from(value: &str) -> Self { - RpcError::General(value.to_string()) - } -} - -impl From> for RpcError { - fn from(_: ChannelError) -> Self { - RpcError::RpcCtlDispatchError - } -} - -impl From for RpcError { - fn from(value: serde_wasm_bindgen::Error) -> Self { - RpcError::SerdeWasmBindgen(value.to_string()) - } -} - -pub type RpcResult = std::result::Result; + #[error(transparent)] + WasmError(#[from] workflow_wasm::error::Error), + + #[error("{0}")] + SerdeWasmBindgen(String), + + #[error(transparent)] + ConsensusClient(#[from] spectre_consensus_client::error::Error), +} + +impl From for RpcError { + fn from(value: String) -> Self { + RpcError::General(value) + } +} + +impl From<&str> for RpcError { + fn from(value: &str) -> Self { + RpcError::General(value.to_string()) + } +} + +impl From> for RpcError { + fn from(_: ChannelError) -> Self { + RpcError::RpcCtlDispatchError + } +} + +impl From for RpcError { + fn from(value: serde_wasm_bindgen::Error) -> Self { + RpcError::SerdeWasmBindgen(value.to_string()) + } +} + +pub type RpcResult = std::result::Result; From a7ff129bbc6b4c0c2c8d2800f9a2ca869ce34aa6 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:13:19 +0200 Subject: [PATCH 06/48] hint message for P2P reject reason block not found --- protocol/p2p/src/common.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/protocol/p2p/src/common.rs b/protocol/p2p/src/common.rs index 4d19b7a..0fb0ea0 100644 --- a/protocol/p2p/src/common.rs +++ b/protocol/p2p/src/common.rs @@ -100,6 +100,10 @@ impl ProtocolError { pub fn from_reject_message(reason: String) -> Self { if reason == LOOPBACK_CONNECTION_MESSAGE || reason == DUPLICATE_CONNECTION_MESSAGE { ProtocolError::IgnorableReject(reason) + } else if reason.contains("cannot find full block") { + let hint = "Hint: If this error persists, it might be due to the other peer having pruned block data after syncing headers and UTXOs. In such a case, you may need to reset the database."; + let detailed_reason = format!("{}. {}", reason, hint); + ProtocolError::Rejected(detailed_reason) } else { ProtocolError::Rejected(reason) } From 91f81179de9c1fe81ac38d0e752d4b1ce2a5106b Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:17:04 +0200 Subject: [PATCH 07/48] lazy load origin children --- consensus/src/model/services/reachability.rs | 9 +++++ .../pipeline/pruning_processor/processor.rs | 31 +++++++++++++---- consensus/src/processes/parents_builder.rs | 34 ++++++++++--------- 3 files changed, 52 insertions(+), 22 deletions(-) diff --git a/consensus/src/model/services/reachability.rs b/consensus/src/model/services/reachability.rs index 91f1317..8098ab4 100644 --- a/consensus/src/model/services/reachability.rs +++ b/consensus/src/model/services/reachability.rs @@ -17,6 +17,7 @@ pub trait ReachabilityService { fn is_any_dag_ancestor_result(&self, list: &mut impl Iterator, queried: Hash) -> Result; fn get_next_chain_ancestor(&self, descendant: Hash, ancestor: Hash) -> Hash; fn get_chain_parent(&self, this: Hash) -> Hash; + fn has_reachability_data(&self, this: Hash) -> bool; } impl ReachabilityService for T { @@ -56,6 +57,10 @@ impl ReachabilityService for T { fn get_chain_parent(&self, this: Hash) -> Hash { self.get_parent(this).unwrap() } + + fn has_reachability_data(&self, this: Hash) -> bool { + self.has(this).unwrap() + } } /// Multi-threaded reachability service imp @@ -108,6 +113,10 @@ impl ReachabilityService for MTReachability fn get_chain_parent(&self, this: Hash) -> Hash { self.store.read().get_parent(this).unwrap() } + + fn has_reachability_data(&self, this: Hash) -> bool { + self.store.read().has(this).unwrap() + } } impl MTReachabilityService { diff --git a/consensus/src/pipeline/pruning_processor/processor.rs b/consensus/src/pipeline/pruning_processor/processor.rs index ef33b85..a537b47 100644 --- a/consensus/src/pipeline/pruning_processor/processor.rs +++ b/consensus/src/pipeline/pruning_processor/processor.rs @@ -268,6 +268,12 @@ impl PruningProcessor { .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) .chain(proof.iter().flatten().map(|h| h.hash)) .collect(); + let keep_level_zero_relations: BlockHashSet = std::iter::empty() + .chain(data.anticone.iter().copied()) + .chain(data.daa_window_blocks.iter().map(|th| th.header.hash)) + .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) + .chain(proof[0].iter().map(|h| h.hash)) + .collect(); let keep_headers: BlockHashSet = self.past_pruning_points(); info!("Header and Block pruning: waiting for consensus write permissions..."); @@ -281,16 +287,16 @@ impl PruningProcessor { { let mut counter = 0; let mut batch = WriteBatch::default(); - for kept in keep_relations.iter().copied() { + for kept in keep_level_zero_relations.iter().copied() { let Some(ghostdag) = self.ghostdag_primary_store.get_data(kept).unwrap_option() else { continue; }; - if ghostdag.unordered_mergeset().any(|h| !keep_relations.contains(&h)) { + if ghostdag.unordered_mergeset().any(|h| !keep_level_zero_relations.contains(&h)) { let mut mutable_ghostdag: ExternalGhostdagData = ghostdag.as_ref().into(); - mutable_ghostdag.mergeset_blues.retain(|h| keep_relations.contains(h)); - mutable_ghostdag.mergeset_reds.retain(|h| keep_relations.contains(h)); - mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_relations.contains(k)); - if !keep_relations.contains(&mutable_ghostdag.selected_parent) { + mutable_ghostdag.mergeset_blues.retain(|h| keep_level_zero_relations.contains(h)); + mutable_ghostdag.mergeset_reds.retain(|h| keep_level_zero_relations.contains(h)); + mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_level_zero_relations.contains(k)); + if !keep_level_zero_relations.contains(&mutable_ghostdag.selected_parent) { mutable_ghostdag.selected_parent = ORIGIN; } counter += 1; @@ -396,6 +402,19 @@ impl PruningProcessor { // other parts of the code assume the existence of GD data etc.) statuses_write.set_batch(&mut batch, current, StatusHeaderOnly).unwrap(); } + + // Delete level-0 relations for blocks which only belong to higher proof levels. + // Note: it is also possible to delete level relations for level x > 0 for any block that only belongs + // to proof levels higher than x, but this requires maintaining such per level usage mapping. + // Since the main motivation of this deletion step is to reduce the + // number of origin's children in level 0, and this is not a bottleneck in any other + // level, we currently chose to only delete level-0 redundant relations. + if !keep_level_zero_relations.contains(¤t) { + let mut staging_level_relations = StagingRelationsStore::new(&mut level_relations_write[0]); + relations::delete_level_relations(MemoryWriter, &mut staging_level_relations, current).unwrap_option(); + staging_level_relations.commit(&mut batch).unwrap(); + self.ghostdag_stores[0].delete_batch(&mut batch, current).unwrap_option(); + } } else { // Count only blocks which get fully pruned including DAG relations counter += 1; diff --git a/consensus/src/processes/parents_builder.rs b/consensus/src/processes/parents_builder.rs index 62819f9..89028eb 100644 --- a/consensus/src/processes/parents_builder.rs +++ b/consensus/src/processes/parents_builder.rs @@ -10,8 +10,6 @@ use crate::model::{ stores::{headers::HeaderStoreReader, reachability::ReachabilityStoreReader, relations::RelationsStoreReader}, }; -use super::reachability::ReachabilityResultExtensions; - #[derive(Clone)] pub struct ParentsManager { max_block_level: BlockLevel, @@ -52,10 +50,7 @@ impl .expect("at least one of the parents is expected to be in the future of the pruning point"); direct_parent_headers.swap(0, first_parent_in_future_of_pruning_point); - let origin_children = self.relations_service.get_children(ORIGIN).unwrap().read().iter().copied().collect_vec(); - let origin_children_headers = - origin_children.iter().copied().map(|parent| self.headers_store.get_header(parent).unwrap()).collect_vec(); - + let mut origin_children_headers = None; let mut parents = Vec::with_capacity(self.max_block_level as usize); for block_level in 0..self.max_block_level { @@ -97,11 +92,7 @@ impl }; for (i, parent) in grandparents.into_iter().enumerate() { - let is_in_origin_children_future = self - .reachability_service - .is_any_dag_ancestor_result(&mut origin_children.iter().copied(), parent) - .unwrap_option() - .is_some_and(|r| r); + let has_reachability_data = self.reachability_service.has_reachability_data(parent); // Reference blocks are the blocks that are used in reachability queries to check if // a candidate is in the future of another candidate. In most cases this is just the @@ -110,13 +101,24 @@ impl // If we make sure to add a parent in the future of the pruning point first, we can // know that any pruned candidate that is in the past of some blocks in the pruning // point anticone should be a parent (in the relevant level) of one of - // the virtual genesis children in the pruning point anticone. So we can check which - // virtual genesis children have this block as parent and use those block as + // the origin children in the pruning point anticone. So we can check which + // origin children have this block as parent and use those block as // reference blocks. - let reference_blocks = if is_in_origin_children_future { + let reference_blocks = if has_reachability_data { smallvec![parent] } else { - let mut reference_blocks = SmallVec::with_capacity(origin_children.len()); + // Here we explicitly declare the type because otherwise Rust would make it mutable. + let origin_children_headers: &Vec<_> = origin_children_headers.get_or_insert_with(|| { + self.relations_service + .get_children(ORIGIN) + .unwrap() + .read() + .iter() + .copied() + .map(|parent| self.headers_store.get_header(parent).unwrap()) + .collect_vec() + }); + let mut reference_blocks = SmallVec::with_capacity(origin_children_headers.len()); for child_header in origin_children_headers.iter() { if self.parents_at_level(child_header, block_level).contains(&parent) { reference_blocks.push(child_header.hash); @@ -133,7 +135,7 @@ impl continue; } - if !is_in_origin_children_future { + if !has_reachability_data { continue; } From 93fdc316f5bd71ff3759c37927815da3299d62fb Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:32:29 +0200 Subject: [PATCH 08/48] O(k log n) mempool transaction sampler + Fee estimation API --- cli/src/modules/rpc.rs | 13 +- consensus/core/src/config/params.rs | 3 +- mining/Cargo.toml | 3 +- mining/benches/bench.rs | 222 +++++++- mining/src/block_template/builder.rs | 23 +- mining/src/block_template/policy.rs | 6 +- mining/src/block_template/selector.rs | 76 ++- mining/src/feerate/fee_estimation.ipynb | 496 ++++++++++++++++++ mining/src/feerate/mod.rs | 231 ++++++++ mining/src/lib.rs | 5 + mining/src/manager.rs | 55 +- mining/src/manager_tests.rs | 19 +- mining/src/mempool/config.rs | 23 +- mining/src/mempool/mod.rs | 27 +- mining/src/mempool/model/feerate_key.rs | 79 +++ mining/src/mempool/model/frontier.rs | 454 ++++++++++++++++ .../mempool/model/frontier/feerate_weight.rs | 265 ++++++++++ .../src/mempool/model/frontier/search_tree.rs | 335 ++++++++++++ .../src/mempool/model/frontier/selectors.rs | 162 ++++++ mining/src/mempool/model/mod.rs | 1 + mining/src/mempool/model/transactions_pool.rs | 47 +- mining/src/mempool/model/tx.rs | 21 - mining/src/model/candidate_tx.rs | 11 +- mining/src/model/mod.rs | 2 +- mining/src/monitor.rs | 8 +- rpc/core/src/api/ops.rs | 4 + rpc/core/src/api/rpc.rs | 16 + rpc/core/src/model/feerate_estimate.rs | 55 ++ rpc/core/src/model/message.rs | 29 + rpc/core/src/model/mod.rs | 2 + rpc/grpc/client/src/lib.rs | 2 + rpc/grpc/core/proto/messages.proto | 4 + rpc/grpc/core/proto/rpc.proto | 56 ++ rpc/grpc/core/src/convert/feerate_estimate.rs | 66 +++ rpc/grpc/core/src/convert/message.rs | 43 ++ rpc/grpc/core/src/convert/mod.rs | 1 + rpc/grpc/core/src/convert/spectred.rs | 4 + rpc/grpc/core/src/ops.rs | 2 + .../server/src/request_handler/factory.rs | 2 + rpc/grpc/server/src/tests/rpc_core_mock.rs | 11 + rpc/service/src/converter/feerate_estimate.rs | 49 ++ rpc/service/src/converter/mod.rs | 1 + rpc/service/src/service.rs | 31 ++ rpc/wrpc/client/src/client.rs | 12 +- rpc/wrpc/server/src/router.rs | 10 +- spectred/src/args.rs | 2 +- spectred/src/daemon.rs | 5 +- testing/integration/src/mempool_benchmarks.rs | 4 +- testing/integration/src/rpc_tests.rs | 27 + testing/integration/src/tasks/tx/sender.rs | 2 +- utils/Cargo.toml | 1 + utils/src/expiring_cache.rs | 152 ++++++ utils/src/lib.rs | 1 + utils/src/rand/mod.rs | 1 + utils/src/rand/seq.rs | 76 +++ utils/src/vec.rs | 9 + wallet/core/src/tests/rpc_core_mock.rs | 11 + 57 files changed, 3133 insertions(+), 145 deletions(-) create mode 100644 mining/src/feerate/fee_estimation.ipynb create mode 100644 mining/src/feerate/mod.rs create mode 100644 mining/src/mempool/model/feerate_key.rs create mode 100644 mining/src/mempool/model/frontier.rs create mode 100644 mining/src/mempool/model/frontier/feerate_weight.rs create mode 100644 mining/src/mempool/model/frontier/search_tree.rs create mode 100644 mining/src/mempool/model/frontier/selectors.rs create mode 100644 rpc/core/src/model/feerate_estimate.rs create mode 100644 rpc/grpc/core/src/convert/feerate_estimate.rs create mode 100644 rpc/service/src/converter/feerate_estimate.rs create mode 100644 utils/src/expiring_cache.rs create mode 100644 utils/src/rand/mod.rs create mode 100644 utils/src/rand/seq.rs diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index b05fd4a..47cd1a8 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -30,7 +30,7 @@ impl Rpc { let op_str_uc = op_str.to_case(Case::UpperCamel).to_string(); // tprintln!(ctx, "uc: '{op_str_uc}'"); - let op = RpcApiOps::from_str(op_str_uc.as_str()).ok_or(Error::custom(format!("No such RPC method: '{op_str}'")))?; + let op = RpcApiOps::from_str(op_str_uc.as_str()).ok_or(Error::custom(format!("No such rpc method: '{op_str}'")))?; match op { RpcApiOps::Ping => { @@ -229,8 +229,17 @@ impl Rpc { } } } + RpcApiOps::GetFeeEstimate => { + let result = rpc.get_fee_estimate_call(GetFeeEstimateRequest {}).await?; + self.println(&ctx, result); + } + RpcApiOps::GetFeeEstimateExperimental => { + let verbose = if argv.is_empty() { false } else { argv.remove(0).parse().unwrap_or(false) }; + let result = rpc.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await?; + self.println(&ctx, result); + } _ => { - tprintln!(ctx, "RPC method exists but is not supported by the CLI: '{op_str}'\r\n"); + tprintln!(ctx, "rpc method exists but is not supported by the cli: '{op_str}'\r\n"); return Ok(()); } } diff --git a/consensus/core/src/config/params.rs b/consensus/core/src/config/params.rs index 59b04d4..ecc672b 100644 --- a/consensus/core/src/config/params.rs +++ b/consensus/core/src/config/params.rs @@ -472,7 +472,8 @@ pub const SIMNET_PARAMS: Params = Params { target_time_per_block: Testnet11Bps::target_time_per_block(), past_median_time_sample_rate: Testnet11Bps::past_median_time_sample_rate(), difficulty_sample_rate: Testnet11Bps::difficulty_adjustment_sample_rate(), - max_block_parents: Testnet11Bps::max_block_parents(), + // For simnet, we deviate from TN11 configuration and allow at least 64 parents in order to support mempool benchmarks out of the box + max_block_parents: if Testnet11Bps::max_block_parents() > 64 { Testnet11Bps::max_block_parents() } else { 64 }, mergeset_size_limit: Testnet11Bps::mergeset_size_limit(), merge_depth: Testnet11Bps::merge_depth_bound(), finality_depth: Testnet11Bps::finality_depth(), diff --git a/mining/Cargo.toml b/mining/Cargo.toml index 91ffca2..b5d50d5 100644 --- a/mining/Cargo.toml +++ b/mining/Cargo.toml @@ -27,8 +27,9 @@ parking_lot.workspace = true rand.workspace = true serde.workspace = true smallvec.workspace = true +sweep-bptree = "0.4.1" thiserror.workspace = true -tokio = { workspace = true, features = [ "rt-multi-thread", "macros", "signal" ] } +tokio = { workspace = true, features = ["rt-multi-thread", "macros", "signal"] } [dev-dependencies] spectre-txscript.workspace = true diff --git a/mining/benches/bench.rs b/mining/benches/bench.rs index df58096..8b07650 100644 --- a/mining/benches/bench.rs +++ b/mining/benches/bench.rs @@ -1,6 +1,16 @@ use criterion::{black_box, criterion_group, criterion_main, Criterion}; -use spectre_mining::model::topological_index::TopologicalIndex; -use std::collections::{hash_set::Iter, HashMap, HashSet}; +use itertools::Itertools; +use rand::{thread_rng, Rng}; +use spectre_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, +}; +use spectre_hashes::{HasherBase, TransactionID}; +use spectre_mining::{model::topological_index::TopologicalIndex, FeerateTransactionKey, Frontier, Policy}; +use std::{ + collections::{hash_set::Iter, HashMap, HashSet}, + sync::Arc, +}; #[derive(Default)] pub struct Dag @@ -68,5 +78,211 @@ pub fn bench_compare_topological_index_fns(c: &mut Criterion) { group.finish(); } -criterion_group!(benches, bench_compare_topological_index_fns); +fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) +} + +fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) +} + +pub fn bench_mempool_sampling(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool sampling"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100000) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + group.bench_function("mempool one-shot sample", |b| { + b.iter(|| { + black_box({ + let selected = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + selected.iter().map(|k| k.mass).sum::() + }) + }) + }); + + // Benchmark frontier insertions and removals (see comparisons below) + let remove = map.values().take(map.len() / 10).cloned().collect_vec(); + group.bench_function("frontier remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + frontier.remove(r).then_some(()).unwrap(); + } + for r in remove.iter().cloned() { + frontier.insert(r).then_some(()).unwrap(); + } + 0 + }) + }) + }); + + // Benchmark hashmap insertions and removals for comparison + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("map remove/add", |b| { + b.iter(|| { + black_box({ + for r in remove.iter() { + map.remove(&r.0).unwrap(); + } + for r in remove.iter().cloned() { + map.insert(r.0, r.1.clone()); + } + 0 + }) + }) + }); + + // Benchmark std btree set insertions and removals for comparison + // Results show that frontier (sweep bptree) and std btree set are roughly the same. + // The slightly higher cost for sweep bptree should be attributed to subtree weight + // maintenance (see FeerateWeight) + #[allow(clippy::mutable_key_type)] + let mut std_btree = std::collections::BTreeSet::from_iter(map.values().cloned()); + let remove = map.iter().take(map.len() / 10).map(|(&k, v)| (k, v.clone())).collect_vec(); + group.bench_function("std btree remove/add", |b| { + b.iter(|| { + black_box({ + for (_, key) in remove.iter() { + std_btree.remove(key).then_some(()).unwrap(); + } + for (_, key) in remove.iter() { + std_btree.insert(key.clone()); + } + 0 + }) + }) + }); + group.finish(); +} + +pub fn bench_mempool_selectors(c: &mut Criterion) { + let mut rng = thread_rng(); + let mut group = c.benchmark_group("mempool selectors"); + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [100, 300, 350, 500, 1000, 2000, 5000, 10_000, 100_000, 500_000, 1_000_000].into_iter().rev() { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + group.bench_function(format!("rebalancing selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("sample inplace selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + + if frontier.total_mass() <= 500_000 { + group.bench_function(format!("take all selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.bench_function(format!("dynamic selector ({})", len), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + } + + group.finish(); +} + +pub fn bench_inplace_sampling_worst_case(c: &mut Criterion) { + let mut group = c.benchmark_group("mempool inplace sampling"); + let max_fee = u64::MAX; + let fee_steps = (0..10).map(|i| max_fee / 100u64.pow(i)).collect_vec(); + for subgroup_size in [300, 200, 100, 80, 50, 30] { + let cap = 1_000_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i < 300 { fee_steps[i as usize / subgroup_size] } else { 1 }; + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut collisions = 0; + let mut n = 0; + + group.bench_function(format!("inplace sampling worst case (subgroup size: {})", subgroup_size), |b| { + b.iter(|| { + black_box({ + let mut selector = frontier.build_selector_sample_inplace(&mut collisions); + n += 1; + selector.select_transactions().iter().map(|k| k.gas).sum::() + }) + }) + }); + + if n > 0 { + println!("---------------------- \n Avg collisions: {}", collisions / n); + } + } + + group.finish(); +} + +criterion_group!( + benches, + bench_mempool_sampling, + bench_mempool_selectors, + bench_inplace_sampling_worst_case, + bench_compare_topological_index_fns +); criterion_main!(benches); diff --git a/mining/src/block_template/builder.rs b/mining/src/block_template/builder.rs index b31b35b..fe44197 100644 --- a/mining/src/block_template/builder.rs +++ b/mining/src/block_template/builder.rs @@ -1,25 +1,18 @@ -use super::{errors::BuilderResult, policy::Policy}; -use crate::{block_template::selector::TransactionsSelector, model::candidate_tx::CandidateTransaction}; +use super::errors::BuilderResult; use spectre_consensus_core::{ api::ConsensusApi, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, merkle::calc_hash_merkle_root, tx::COINBASE_TRANSACTION_INDEX, }; -use spectre_core::{ - debug, - time::{unix_now, Stopwatch}, -}; +use spectre_core::time::{unix_now, Stopwatch}; -pub(crate) struct BlockTemplateBuilder { - policy: Policy, -} +pub(crate) struct BlockTemplateBuilder {} impl BlockTemplateBuilder { - pub(crate) fn new(max_block_mass: u64) -> Self { - let policy = Policy::new(max_block_mass); - Self { policy } + pub(crate) fn new() -> Self { + Self {} } /// BuildBlockTemplate creates a block template for a miner to consume @@ -89,12 +82,10 @@ impl BlockTemplateBuilder { &self, consensus: &dyn ConsensusApi, miner_data: &MinerData, - transactions: Vec, + selector: Box, build_mode: TemplateBuildMode, ) -> BuilderResult { let _sw = Stopwatch::<20>::with_threshold("build_block_template op"); - debug!("Considering {} transactions for a new block template", transactions.len()); - let selector = Box::new(TransactionsSelector::new(self.policy.clone(), transactions)); Ok(consensus.build_block_template(miner_data.clone(), selector, build_mode)?) } diff --git a/mining/src/block_template/policy.rs b/mining/src/block_template/policy.rs index ff51972..12ee98e 100644 --- a/mining/src/block_template/policy.rs +++ b/mining/src/block_template/policy.rs @@ -1,14 +1,14 @@ /// Policy houses the policy (configuration parameters) which is used to control /// the generation of block templates. See the documentation for -/// NewBlockTemplate for more details on each of these parameters are used. +/// NewBlockTemplate for more details on how each of these parameters are used. #[derive(Clone)] -pub(crate) struct Policy { +pub struct Policy { /// max_block_mass is the maximum block mass to be used when generating a block template. pub(crate) max_block_mass: u64, } impl Policy { - pub(crate) fn new(max_block_mass: u64) -> Self { + pub fn new(max_block_mass: u64) -> Self { Self { max_block_mass } } } diff --git a/mining/src/block_template/selector.rs b/mining/src/block_template/selector.rs index 37b6b3a..134e405 100644 --- a/mining/src/block_template/selector.rs +++ b/mining/src/block_template/selector.rs @@ -18,7 +18,7 @@ use spectre_consensus_core::{ /// candidate transactions should be. A smaller alpha makes the distribution /// more uniform. ALPHA is used when determining a candidate transaction's /// initial p value. -const ALPHA: i32 = 3; +pub(crate) const ALPHA: i32 = 3; /// REBALANCE_THRESHOLD is the percentage of candidate transactions under which /// we don't rebalance. Rebalancing is a heavy operation so we prefer to avoid @@ -28,7 +28,7 @@ const ALPHA: i32 = 3; /// if REBALANCE_THRESHOLD is 0.95, there's a 1-in-20 chance of collision. const REBALANCE_THRESHOLD: f64 = 0.95; -pub(crate) struct TransactionsSelector { +pub struct RebalancingWeightedTransactionSelector { policy: Policy, /// Transaction store transactions: Vec, @@ -52,8 +52,8 @@ pub(crate) struct TransactionsSelector { gas_usage_map: HashMap, } -impl TransactionsSelector { - pub(crate) fn new(policy: Policy, mut transactions: Vec) -> Self { +impl RebalancingWeightedTransactionSelector { + pub fn new(policy: Policy, mut transactions: Vec) -> Self { let _sw = Stopwatch::<100>::with_threshold("TransactionsSelector::new op"); // Sort the transactions by subnetwork_id. transactions.sort_by(|a, b| a.tx.subnetwork_id.cmp(&b.tx.subnetwork_id)); @@ -103,7 +103,7 @@ impl TransactionsSelector { /// select_transactions loops over the candidate transactions /// and appends the ones that will be included in the next block into /// selected_txs. - pub(crate) fn select_transactions(&mut self) -> Vec { + pub fn select_transactions(&mut self) -> Vec { let _sw = Stopwatch::<15>::with_threshold("select_transaction op"); let mut rng = rand::thread_rng(); @@ -225,7 +225,7 @@ impl TransactionsSelector { } } -impl TemplateTransactionSelector for TransactionsSelector { +impl TemplateTransactionSelector for RebalancingWeightedTransactionSelector { fn select_transactions(&mut self) -> Vec { self.select_transactions() } @@ -269,7 +269,13 @@ mod tests { use spectre_txscript::{pay_to_script_hash_signature_script, test_helpers::op_true_script}; use std::{collections::HashSet, sync::Arc}; - use crate::{mempool::config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, model::candidate_tx::CandidateTransaction}; + use crate::{ + mempool::{ + config::DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, + model::frontier::selectors::{SequenceSelector, SequenceSelectorInput, SequenceSelectorTransaction}, + }, + model::candidate_tx::CandidateTransaction, + }; #[test] fn test_reject_transaction() { @@ -277,29 +283,43 @@ mod tests { // Create a vector of transactions differing by output value so they have unique ids let transactions = (0..TX_INITIAL_COUNT).map(|i| create_transaction(SOMPI_PER_SPECTRE * (i + 1) as u64)).collect_vec(); + let masses: HashMap<_, _> = transactions.iter().map(|tx| (tx.tx.id(), tx.calculated_mass)).collect(); + let sequence: SequenceSelectorInput = + transactions.iter().map(|tx| SequenceSelectorTransaction::new(tx.tx.clone(), tx.calculated_mass)).collect(); + let policy = Policy::new(100_000); - let mut selector = TransactionsSelector::new(policy, transactions); - let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); - let mut reject_count = 32; - for i in 0..10 { - let selected_txs = selector.select_transactions(); - if i > 0 { - assert_eq!( - selected_txs.len(), - reject_count, - "subsequent select calls are expected to only refill the previous rejections" - ); - reject_count /= 2; - } - for tx in selected_txs.iter() { - kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); - assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + let selectors: [Box; 2] = [ + Box::new(RebalancingWeightedTransactionSelector::new(policy.clone(), transactions)), + Box::new(SequenceSelector::new(sequence, policy.clone())), + ]; + + for mut selector in selectors { + let (mut kept, mut rejected) = (HashSet::new(), HashSet::new()); + let mut reject_count = 32; + let mut total_mass = 0; + for i in 0..10 { + let selected_txs = selector.select_transactions(); + if i > 0 { + assert_eq!( + selected_txs.len(), + reject_count, + "subsequent select calls are expected to only refill the previous rejections" + ); + reject_count /= 2; + } + for tx in selected_txs.iter() { + total_mass += masses[&tx.id()]; + kept.insert(tx.id()).then_some(()).expect("selected txs should never repeat themselves"); + assert!(!rejected.contains(&tx.id()), "selected txs should never repeat themselves"); + } + assert!(total_mass <= policy.max_block_mass); + selected_txs.iter().take(reject_count).for_each(|x| { + total_mass -= masses[&x.id()]; + selector.reject_selection(x.id()); + kept.remove(&x.id()).then_some(()).expect("was just inserted"); + rejected.insert(x.id()).then_some(()).expect("was just verified"); + }); } - selected_txs.iter().take(reject_count).for_each(|x| { - selector.reject_selection(x.id()); - kept.remove(&x.id()).then_some(()).expect("was just inserted"); - rejected.insert(x.id()).then_some(()).expect("was just verified"); - }); } } diff --git a/mining/src/feerate/fee_estimation.ipynb b/mining/src/feerate/fee_estimation.ipynb new file mode 100644 index 0000000..694f474 --- /dev/null +++ b/mining/src/feerate/fee_estimation.ipynb @@ -0,0 +1,496 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerates\n", + "\n", + "The feerate value represents the fee/mass ratio of a transaction in `sompi/gram` units.\n", + "Given a feerate value recommendation, one should calculate the required fee by taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)`. \n", + "\n", + "This notebook makes an effort to implement and illustrate the feerate estimator method we used. The corresponding Rust implementation is more comprehensive and addresses some additional edge cases, but the code in this notebook highly reflects it." + ] + }, + { + "cell_type": "code", + "execution_count": 97, + "metadata": {}, + "outputs": [], + "source": [ + "import numpy as np\n", + "import matplotlib.pyplot as plt" + ] + }, + { + "cell_type": "code", + "execution_count": 98, + "metadata": {}, + "outputs": [], + "source": [ + "feerates = [1.0, 1.1, 1.2]*10 + [1.5]*3000 + [2]*3000 + [2.1]*3000 + [3, 4, 5]*10\n", + "# feerates = [1.0, 1.1, 1.2] + [1.1]*100 + [1.2]*100 + [1.3]*100 # + [3, 4, 5, 100]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We compute the probability weight of each transaction by raising `feerate` to the power of `alpha` (currently set to `3`). Essentially, alpha represents the amount of bias we want towards higher feerate transactions. " + ] + }, + { + "cell_type": "code", + "execution_count": 99, + "metadata": {}, + "outputs": [], + "source": [ + "ALPHA = 3.0" + ] + }, + { + "cell_type": "code", + "execution_count": 100, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Total mempool weight: 64108.589999995806\n" + ] + } + ], + "source": [ + "total_weight = sum(np.array(feerates)**ALPHA)\n", + "print('Total mempool weight: ', total_weight)" + ] + }, + { + "cell_type": "code", + "execution_count": 101, + "metadata": {}, + "outputs": [], + "source": [ + "avg_mass = 2000\n", + "bps = 1\n", + "block_mass_limit = 500_000\n", + "network_mass_rate = bps * block_mass_limit" + ] + }, + { + "cell_type": "code", + "execution_count": 102, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Inclusion interval: 0.004\n" + ] + } + ], + "source": [ + "print('Inclusion interval: ', avg_mass/network_mass_rate)" + ] + }, + { + "cell_type": "code", + "execution_count": 109, + "metadata": {}, + "outputs": [], + "source": [ + "class FeerateBucket:\n", + " def __init__(self, feerate, estimated_seconds):\n", + " self.feerate = feerate\n", + " self.estimated_seconds = estimated_seconds\n", + " \n", + "\n", + "class FeerateEstimations:\n", + " def __init__(self, low_bucket, mid_bucket, normal_bucket, priority_bucket):\n", + " self.low_bucket = low_bucket \n", + " self.mid_bucket = mid_bucket \n", + " self.normal_bucket = normal_bucket\n", + " self.priority_bucket = priority_bucket\n", + " \n", + " def __repr__(self):\n", + " return 'Feerates:\\t{}, {}, {}, {} \\nTimes:\\t\\t{}, {}, {}, {}'.format(\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate, \n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds, \n", + " self.priority_bucket.estimated_seconds)\n", + " def feerates(self):\n", + " return np.array([\n", + " self.low_bucket.feerate, \n", + " self.mid_bucket.feerate, \n", + " self.normal_bucket.feerate,\n", + " self.priority_bucket.feerate\n", + " ])\n", + " \n", + " def times(self):\n", + " return np.array([\n", + " self.low_bucket.estimated_seconds, \n", + " self.mid_bucket.estimated_seconds, \n", + " self.normal_bucket.estimated_seconds,\n", + " self.priority_bucket.estimated_seconds\n", + " ])\n", + " \n", + "class FeerateEstimator:\n", + " \"\"\"\n", + " `total_weight`: The total probability weight of all current mempool ready \n", + " transactions, i.e., Σ_{tx in mempool}(tx.fee/tx.mass)^ALPHA\n", + " \n", + " 'inclusion_interval': The amortized time between transactions given the current \n", + " transaction masses present in the mempool, i.e., the inverse \n", + " of the transaction inclusion rate. For instance, if the average \n", + " transaction mass is 2500 grams, the block mass limit is 500,000\n", + " and the network has 10 BPS, then this number would be 1/2000 seconds.\n", + " \"\"\"\n", + " def __init__(self, total_weight, inclusion_interval):\n", + " self.total_weight = total_weight\n", + " self.inclusion_interval = inclusion_interval\n", + "\n", + " \"\"\"\n", + " Feerate to time function: f(feerate) = inclusion_interval * (1/p(feerate))\n", + " where p(feerate) = feerate^ALPHA/(total_weight + feerate^ALPHA) represents \n", + " the probability function for drawing `feerate` from the mempool\n", + " in a single trial. The inverse 1/p is the expected number of trials until\n", + " success (with repetition), thus multiplied by inclusion_interval it provides an\n", + " approximation to the overall expected waiting time\n", + " \"\"\"\n", + " def feerate_to_time(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / feerate**ALPHA + c1\n", + "\n", + " \"\"\"\n", + " The inverse function of `feerate_to_time`\n", + " \"\"\"\n", + " def time_to_feerate(self, time):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return ((c1 * c2 / time) / (1 - c1 / time))**(1 / ALPHA)\n", + " \n", + " \"\"\"\n", + " The antiderivative function of \n", + " feerate_to_time excluding the constant shift `+ c1`\n", + " \"\"\"\n", + " def feerate_to_time_antiderivative(self, feerate):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " return c1 * c2 / (-2.0 * feerate**(ALPHA - 1))\n", + " \n", + " \"\"\"\n", + " Returns the feerate value for which the integral area is `frac` of the total area.\n", + " See figures below for illustration\n", + " \"\"\"\n", + " def quantile(self, lower, upper, frac):\n", + " c1, c2 = self.inclusion_interval, self.total_weight\n", + " z1 = self.feerate_to_time_antiderivative(lower)\n", + " z2 = self.feerate_to_time_antiderivative(upper)\n", + " z = frac * z2 + (1.0 - frac) * z1\n", + " return ((c1 * c2) / (-2 * z))**(1.0 / (ALPHA - 1.0))\n", + " \n", + " def calc_estimations(self):\n", + " # Choose `high` such that it provides sub-second waiting time\n", + " high = self.time_to_feerate(1.0)\n", + " \n", + " # Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile\n", + " low = max(self.time_to_feerate(3600.0), self.quantile(1.0, high, 0.25))\n", + " \n", + " # Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66\n", + " # quantile between low and high\n", + " normal = max(self.time_to_feerate(60.0), self.quantile(low, high, 0.66))\n", + " \n", + " # Choose an additional point between normal and low\n", + " mid = max(self.time_to_feerate(1800.0), self.quantile(1.0, high, 0.5))\n", + " \n", + " return FeerateEstimations(\n", + " FeerateBucket(low, self.feerate_to_time(low)),\n", + " FeerateBucket(mid, self.feerate_to_time(mid)),\n", + " FeerateBucket(normal, self.feerate_to_time(normal)),\n", + " FeerateBucket(high, self.feerate_to_time(high)))" + ] + }, + { + "cell_type": "code", + "execution_count": 104, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "0.0" + ] + }, + "execution_count": 104, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=0, inclusion_interval=1/100)\n", + "# estimator.quantile(2, 3, 0.5)\n", + "estimator.time_to_feerate(1)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Feerate estimation\n", + "\n", + "The figure below illustrates the estimator selection. We first estimate the `feerate_to_time` function and then select 3 meaningfull points by analyzing the curve and its integral (see `calc_estimations`). " + ] + }, + { + "cell_type": "code", + "execution_count": 105, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHVlJREFUeJzt3XuQnHW95/H3t+8990lmMplkQhIwoIAJlxj14PFwRNR4OcA5LoVb60HLLU4VeNTaU7Wl7JaHtWTX2rPKrq5yFgTFEnVTikcUvLABBY5ASBASyIUk5DKT20xuc0vm0tPf/aOfCZNkkpnMdOeZfvrzqup6nufXTz/9bS6f3zO//j1Pm7sjIiLRFQu7ABERKS0FvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4hT0IiIRp6AXEYm4RNgFADQ1NfmiRYvCLkNEpKysW7fuoLs3T7TfjAj6RYsWsXbt2rDLEBEpK2a2azL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgr66Dfsr+Xf/rtZo4eGwq7FBGRGausg37XoX6+/dR22g8fD7sUEZEZa8KgN7MFZvaUmW0ys9fM7PNB+11mtsfMXg4eHx7zmi+Z2TYz22JmHyxV8S11GQAO9AyU6i1ERMreZG6BkAP+wd1fMrNaYJ2ZPRE8d4+7/4+xO5vZpcAtwGXAPOD/mdnF7j5SzMJhTND3KuhFRM5kwjN6d9/n7i8F673AJmD+WV5yA/ATdx909x3ANmBFMYo9VVNNCgMO9AyW4vAiIpFwTmP0ZrYIuBJ4IWj6rJmtN7MHzawxaJsPtI95WQdn7ximLBGP0VCVpFNDNyIiZzTpoDezGuBnwBfcvQe4F7gIuALYB3x9dNdxXu7jHO82M1trZmu7urrOufBR9dmkxuhFRM5iUkFvZkkKIf+wuz8C4O4H3H3E3fPA/bw5PNMBLBjz8jZg76nHdPf73H25uy9vbp7wdspnVJ9Nsl9BLyJyRpOZdWPAA8Amd//GmPbWMbvdBLwarD8K3GJmaTNbDCwB1hSv5JM1VCU1Ri8ichaTmXVzDfBJYIOZvRy03Ql8wsyuoDAssxP4OwB3f83MVgEbKczYuaMUM25GNVSlONw/xFAuTypR1pcFiIiUxIRB7+7PMv64++Nnec3dwN3TqGvSGqqSAHT1DTK/IXs+3lJEpKyU/SlwYxD0+kJWRGR8ZR/0DdkUgKZYioicQfkH/Ykzen0hKyIynrIP+ppMgphp6EZE5EzKPuhjZtSkEzqjFxE5g7IPeoDqdIJO3dhMRGRckQj6qlScfUcV9CIi44lE0FenE7oNgojIGUQi6GvSCfoGcxwbyoVdiojIjBOJoK/NFC7w3avhGxGR00Qj6NOFufR7j+q3Y0VEThWNoD9xRq+gFxE5VSSCvjqdwFDQi4iMJxJBH48ZtZkEe7s1Ri8icqpIBD0UboWgM3oRkdNFJ+hTCfYcUdCLiJwqOkGfSbCvewD3036HXESkokUm6GszSYZG8hzqHwq7FBGRGSVCQa8pliIi44lO0Kd1dayIyHiiE/QZXR0rIjKeyAR9JhkjGTcFvYjIKSIT9GZGXSbJPl00JSJyksgEPUB1Ok7HkWNhlyEiMqNEKuhrM0k6dNGUiMhJIhX0ddkkh/qH9AMkIiJjRCro64OZNzqrFxF5U7SCPlsI+t2HNE4vIjIqUkFfly1cNNWuL2RFRE6IVNBnk3FSiRi7DyvoRURGTRj0ZrbAzJ4ys01m9pqZfT5on2VmT5jZ1mDZGLSbmX3TzLaZ2Xozu6rUH2JMrdRnk7Qr6EVETpjMGX0O+Ad3fxvwLuAOM7sU+CKw2t2XAKuDbYCVwJLgcRtwb9GrPovadEJn9CIiY0wY9O6+z91fCtZ7gU3AfOAG4KFgt4eAG4P1G4AfeMHzQIOZtRa98jOoyybZffiY7ksvIhI4pzF6M1sEXAm8ALS4+z4odAbAnGC3+UD7mJd1BG2nHus2M1trZmu7urrOvfIzqM8mGRjWfelFREZNOujNrAb4GfAFd+85267jtJ12eu3u97n7cndf3tzcPNkyJjQ680bDNyIiBZMKejNLUgj5h939kaD5wOiQTLDsDNo7gAVjXt4G7C1OuRMbvWhKX8iKiBRMZtaNAQ8Am9z9G2OeehS4NVi/FfjFmPa/DWbfvAvoHh3iOR/qsgp6EZGxEpPY5xrgk8AGM3s5aLsT+Bqwysw+A+wG/k3w3OPAh4FtwDHg00WteALJeIwazbwRETlhwqB392cZf9wd4Lpx9nfgjmnWNS11mQS7dBsEEREgYlfGjqqvSvLGwf6wyxARmREiGfSNVSm6egfpH9TtikVEIhn0DVWFL2R36KxeRCSaQd9YlQIU9CIiENGgb8jqjF5EZFQkgz4Rj1GfTSroRUSIaNBD4Z43b3T1hV2GiEjoIhv0DdnCFEvdxVJEKl10g74qSe9AjsO6i6WIVLjIBr1m3oiIFEQ26Efn0usKWRGpdJEN+rpMkriZzuhFpOJFNuhjMaOhKsm2Ts28EZHKFtmgB2isTrFlf2/YZYiIhCrSQT+7OkX74WMMDI+EXYqISGgiHfQfHHmaZ1KfI333bLjncli/KuySRETOu8n8wlRZuqTz17y/6+ukYoOFhu52+OXnCutLbw6vMBGR8yyyZ/Tv2f0dUj54cuPwcVj9lXAKEhEJSWSDvnbwwPhPdHec30JEREIW2aDvTbeM/0R92/ktREQkZJEN+mcvuJ3hWObkxmQWrvtyOAWJiIQksl/GbpmzEoB37vjfNA53kaudR+oDd+mLWBGpOJENeiiE/R+r3scPX9jNPTcu46alGrYRkcoT2aGbUQ1VKeJmbNmvWyGISGWKfNDHY8bsmhSb9/eEXYqISCgiH/QAs2tSvLqnO+wyRERCURFB31yT5mDfEJ29A2GXIiJy3lVG0NemAdi0T3eyFJHKUxlBX1MI+tf2avhGRCrPhEFvZg+aWaeZvTqm7S4z22NmLwePD4957ktmts3MtpjZB0tV+LlIJ+M0ZJNs3KsvZEWk8kzmjP77wIfGab/H3a8IHo8DmNmlwC3AZcFrvmNm8WIVOx2za1K8pqAXkQo0YdC7+9PA4Uke7wbgJ+4+6O47gG3AimnUVzTNNWl2HuynfzAXdikiIufVdMboP2tm64OhncagbT7QPmafjqDtNGZ2m5mtNbO1XV1d0yhjcppr0ziwWT8tKCIVZqpBfy9wEXAFsA/4etBu4+zr4x3A3e9z9+Xuvry5uXmKZUze6Mybjfs0fCMilWVKQe/uB9x9xN3zwP28OTzTASwYs2sbsHd6JRZHTTpBVSrOho6jYZciInJeTSnozax1zOZNwOiMnEeBW8wsbWaLgSXAmumVWBxmxpzaNC+3K+hFpLJMePdKM/sxcC3QZGYdwD8C15rZFRSGZXYCfwfg7q+Z2SpgI5AD7nD3kdKUfu5a6jKs2XGYvsEcNelI37hTROSECdPO3T8xTvMDZ9n/buDu6RRVKnPrMjjw6p5u3nXh7LDLERE5LyriythRLXWFX5x6RcM3IlJBKiros6k4jVVJjdOLSEWpqKCHwjTLPynoRaSCVFzQz63LsL97gM4e3bJYRCpDxQX96Di9hm9EpFJUXNDPqU0TMwW9iFSOigv6RDzGnNoML+6c7H3aRETKW8UFPcC8hgwvtx9lYHjGXMslIlIyFRr0WYZHnPUd+sUpEYm+ig16QMM3IlIRKjLos8k4TTUp1uxQ0ItI9FVk0APMrc+wbtcRRvLj3i5fRCQyKjbo5zdk6RvMsUk/RCIiEVexQa9xehGpFBUb9HWZJPXZJM9tPxR2KSIiJVWxQQ/Q1pjlj9sPkRvJh12KiEjJVHTQXzCrir7BHBv2aD69iERXRQd9W2NhnP7ZrQdDrkREpHQqOuirUgnm1KV5dpuCXkSiq6KDHmBBQxXrdh3h2FAu7FJEREpCQT8rSy7vvKCrZEUkoio+6Oc3ZEnETOP0IhJZFR/0iXiM+Q1ZntzcGXYpIiIlUfFBD7CoqZodB/vZcbA/7FJERIpOQQ9c2FQNwOpNB0KuRESk+BT0QF02SXNNmtWbNHwjItGjoA8snF3Fmp2H6T4+HHYpIiJFpaAPLG6qZiTvPP16V9iliIgUlYI+MLc+Q1UqrnF6EYmcCYPezB40s04ze3VM2ywze8LMtgbLxqDdzOybZrbNzNab2VWlLL6YYmYsml3NE5sOMJgbCbscEZGimcwZ/feBD53S9kVgtbsvAVYH2wArgSXB4zbg3uKUeX4saamhf3CEZ17XxVMiEh0TBr27Pw2cen+AG4CHgvWHgBvHtP/AC54HGsystVjFltqCxiqyyTiPbdgXdikiIkUz1TH6FnffBxAs5wTt84H2Mft1BG2nMbPbzGytma3t6poZX4DGY8bipmqe2HiAgWEN34hINBT7y1gbp83H29Hd73P35e6+vLm5uchlTN3FLTX0DeZ4Rve+EZGImGrQHxgdkgmWo1cadQALxuzXBuydennnX1tjFR9P/ZGrH/lzuKsB7rkc1q8KuywRkSmbatA/CtwarN8K/GJM+98Gs2/eBXSPDvGUi0sP/oavxu5nVu4A4NDdDr/8nMJeRMrWZKZX/hh4DrjEzDrM7DPA14DrzWwrcH2wDfA48AawDbgfuL0kVZfQe3Z/hwyDJzcOH4fVXwmnIBGRaUpMtIO7f+IMT103zr4O3DHdosJUO3iGC6a6O85vISIiRaIrY0/Rm24Z/4n6tvNbiIhIkSjoT/HsBbczHMuc3JjMwnVfDqcgEZFpmnDoptJsmbMSKIzV1wweoNOaaPnYf8WW3hxyZSIiU6OgH8eWOSvZMmclm/b18LuNB3i46p1cE3ZRIiJTpKGbs1gyp4aqVJwfPLcz7FJERKZMQX8WiXiMt7XW8cTGA+w9ejzsckREpkRBP4Gl8+txhx+9sDvsUkREpkRBP4G6bJLFTdX8aM1u3adeRMqSgn4SlrbVc7h/iF+9UlZ3cxARART0k3LBrCqaalL88x+2k8+PezNOEZEZS0E/CWbG1Rc0srWzj6e2dE78AhGRGURBP0lLWmqpzya59/fbwy5FROScKOgnKR4zrljQwNpdR3hx56m/rCgiMnMp6M/BZfPqqErF+dbqrWGXIiIyaQr6c5CMx7jqgkae3nqQNTt0Vi8i5UFBf46WttVTk07wT7/dTOH2+yIiM5uC/hwl4zGWL2rkxZ1HeFo/IC4iZUBBPwWXz6unPpvkv/9ms+bVi8iMp6CfgnjMeOfiWby2t4dH/rQn7HJERM5KQT9Fb51bS2t9hq/9ehN9g7mwyxEROSMF/RSZGe9d0szBviG+9aSmW4rIzKWgn4a59Rkuba3lgWd2sONgf9jliIiMS0E/TX92URPxmPGln63XdEsRmZEU9NNUnU7wnrc08fyOw/zkxfawyxEROY2Cvggum1fHgsYsdz+2if3dA2GXIyJyEgV9EZgZ73vrHAZzI9z58w0awhGRGUVBXyQNVSnefeFsntzcyQ/1+7IiMoMo6IvoigUNLJpdxVd/tZEt+3vDLkdEBFDQF5WZ8f63tZCIG3//45cYGNaPiYtI+KYV9Ga208w2mNnLZrY2aJtlZk+Y2dZg2VicUstDdTrB9W9r4fUDffznf3lV4/UiErpinNH/pbtf4e7Lg+0vAqvdfQmwOtiuKAtnV7Ni8Sx+uq6DHzy3K+xyRKTClWLo5gbgoWD9IeDGErzHjPeuxbO4sKmar/xyI89tPxR2OSJSwaYb9A78zszWmdltQVuLu+8DCJZzpvkeZcnM+MBlLTRUJbn94XW6RYKIhGa6QX+Nu18FrATuMLP3TvaFZnabma01s7VdXV3TLGNmSififGRpK4O5PJ984AU6e3UxlYicf9MKenffGyw7gZ8DK4ADZtYKECw7z/Da+9x9ubsvb25unk4ZM1pjVYqPLZtHZ88gtz64ht6B4bBLEpEKM+WgN7NqM6sdXQc+ALwKPArcGux2K/CL6RZZ7ubWZfjw2+eyZX8v//6htRwb0v3rReT8mc4ZfQvwrJm9AqwBHnP33wBfA643s63A9cF2xVs4u5oPXDqXNTsP86kHX6RfP1YiIudJYqovdPc3gGXjtB8CrptOUVF1ydxaAH67cT+f+t4avv/pFVSnp/yvQERkUnRl7Hl2ydxaPnTZXNbtOsK/vf95DvUNhl2SiEScgj4EF7fU8uG3t/La3h7++jt/ZNchTb0UkdJR0IfkouYa/vqq+XT1DXLjt/+VHU99D+65HO5qKCzXrwq7RBGJCA0Qh6i1PsvHr2pj+OX/S8vv/xlsqPBEdzv88nOF9aU3h1egiESCzuhD1lid4oupVVSNhvyo4eOw+ivhFCUikaKgnwHqhg6M/0R3x/ktREQiSUE/A/SmW8Zt78vM1W2ORWTaFPQzwLMX3M5wLHNS2wBp7uy5iU9970XaDx8LqTIRiQIF/QywZc5KnrjoTnrSc3GMnvRcVi/5T/S85Sae236I6+/5A//nD9sZHsmHXaqIlCHNupkhtsxZyZY5K09qWwZc2FzNH17v4r/9ejOP/GkPd33sMt590exwihSRsqQz+hmuNpPko0vn8dGlrew7epxP3P88n3noRbZ16sfHRWRydEZfJi5qrmHhrCpebj/Ks1sP8sHNz3DzOxZwx19eRFtjVdjlicgMpqAvI4l4jOWLZnHpvDrW7DjMqhfbWbW2nb+5aj63X/sWFjVVh12iiMxACvoyVJVKcO0lc7h6YSPrdh3hkZf28NN1HXx06Tw+fc0irrygMewSRWQGUdCXsdpMkmsvmcM7Fs3ipd1H+O1r+3n0lb0snV/Pp65ZxEeWtpJOxMMuU0RCZjPhgpzly5f72rVrp/TaJzcf4JX27iJXVJ6Gcnk27eth/Z5uDvcP0ViV5KYr2/j41W1cOq8u7PJEpMjMbJ27L59oP53RR0gqEWPZggaWttWz+/AxXt3bw0PP7eTBf93BW+fW8vGr2/irZfOYU5eZ8FgiEh0K+ggyMxbOrmbh7GqOD4/w+v5eNu/v5auPbeLuxzZx1cJGVl4+lw9eNpcFszRjRyTqFPQRl03GWbaggWULGjjcP8S2zj62d/Xx1cc28dXHNnH5vDre97YW/uLiZpa11ZOI69IKkahR0FeQWdUpViyexYrFszh6bIjtXf1s7+rjW09u5Zurt1KbSfDnS5r4i4ub+bOLmmhrzGJmYZctItOkoK9QDVUprl6Y4uqFjQwMj7D78DF2HTrGM68f5PEN+wGYW5dhxeJZvGPxLFYsmsWSOTXEYgp+kXKjoBcyyTgXt9RycUst7s6h/iH2HDnOnqPHeWpzJ4++sheA+mySZW31LG1r4O1t9Sxtq2duXUZn/SIznIJeTmJmNNWkaapJs2xBA+5Oz0COPUePs/focTbv7+XZbQfJB7NyZ1WnWNZWz+Xz61nSUsslLbUsbqomldBYv8hMoaCXszIz6rNJ6rNJLm0tzMXPjeTp6huks2eQA70DbNjTzR9e7zoR/vGYsbipmkuCvxLeMqeGRU1VLJxdTU1a/8mJnG/6v07OWSIeo7U+S2t99kRbbiTPkWPDHOof5HD/EIf6hvjj9oM8vmEfYy/Jm12dYnFTYernotlVLGyqZuGsKlobMjRVp/UdgEgJKOilKBLxGM21aZpr0ye1D4/kOXpsmKPHhjh6fJju48Ps7xng9QO99AzkTto3GTda67PMa8gwryHLvPos8xqytDZkaK3P0FyTprEqpc5A5Bwp6KWkkmfoAKDQCXQH4d83kKN3MEfvwDDth4+xcW8PfYO5E8NBo+JmzK5JnThmc01h2TRm2VidpLEqRUNVUvf6EUFBLyFKxmMnvvgdTz7v9A/l6B3I0TeY49jQCMeGcvQPjtA/mONQ3xAvDR0Zt0MYlU3GaagqBH9jdZKGqhSNwXZ9trBdm0kUHukktZkENcG2OgmJCgW9zFixmFGbSVKbSZ51P3dnYDhf6ASGRhgYHn3kT6wfG8px5NgQg7leBoZHOD40wkS380vFY1Sn49RmktRlT+4IatIJsqk41akEVak42VS8sEwWtt9sG/N8Mq4rjyUUJQt6M/sQ8L+AOPBdd/9aqd5LKpuZkQ3CdLK/puvuDOYKHcFQLs9gLs/QSJ6hXP7N7VyewZHC830DOQ73DzGc8zf3G8kzcqY/Jc4gGTeyyUKt6UScdCJGJhknm4yTTsZIJ2In2gvbwXoiRjo5Zj1x+v7JRIxEzEjGY6TGW0/ESMUL6/GY6fqHMKxfBau/At0dUN8G130Zlt5c8rctSdCbWRz4NnA90AG8aGaPuvvGUryfyLkyMzLJOJnk9IZnRvJOLp9neMQZHsmTC5aFh5MLlsP509ty+UJH0TMwzJFjQ+TdGcl7cExnZKSwHD1+MRmFobNEvNAZJONGIhYjmbBCZxAvdAqjncToMh6LEY9BIhYjHjMSMSMWLE/ffrNTiY95/uT12ATHKNQVixW+n4nFjJgZMYOYjXZYhSm9Y9tPPGKMux43w4LtuBWOMXq8mFGaTnD9Kvjl52D4eGG7u72wDSUP+1Kd0a8Atrn7GwBm9hPgBkBBL5FSCK04pb48wN3JOyc6h9xohzDiJzqbvBc6nrw7+bwz4k4+T7D0U5Zvtp/oYIL3yOed4VyeweGRE/vmvbCfO4UHwb7BMdwhz+n75oNjlqOxHcBJHcaJtjc7hbi9uR4zTnRIxpttP+y9kxY/fvKbDB8vnOGXadDPB9rHbHcA7yzFG82uTrNYv5UqMmONdlLuflrHMdqxjAQdw9jOZ+x+I2M6mRMdDn7K9pj1Me95cnuwzuT2OdPxTnpfd/LB8cZ2hKM1jb6m2Q+O/w+ou6Pk/w5KFfTj/d1zUr9uZrcBtwFccMEFU36j0VvwiojMaPe0FYZrTlXfVvK3LtUUgA5gwZjtNmDv2B3c/T53X+7uy5ubm0tUhojIDHHdlyGZPbktmS20l1ipgv5FYImZLTazFHAL8GiJ3ktEZOZbejN87JtQvwCwwvJj3yzfWTfunjOzzwK/pTC98kF3f60U7yUiUjaW3nxegv1UJZsr4O6PA4+X6vgiIjI5ukxPRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYgz9/BvLWdmXcCuKb68CTjD3YIiI+qfUZ+vvOnzhWehu094D5kZEfTTYWZr3X152HWUUtQ/oz5fedPnm/k0dCMiEnEKehGRiItC0N8XdgHnQdQ/oz5fedPnm+HKfoxeRETOLgpn9CIichZlG/Rm9qCZdZrZq2HXUgpmtsDMnjKzTWb2mpl9PuyaisnMMma2xsxeCT7ffwm7plIws7iZ/cnMfhV2LaVgZjvNbIOZvWxma8Oup9jMrMHMfmpmm4P/F98ddk1TUbZDN2b2XqAP+IG7Xx52PcVmZq1Aq7u/ZGa1wDrgRnffGHJpRWFmBlS7e5+ZJYFngc+7+/Mhl1ZUZvYfgOVAnbt/NOx6is3MdgLL3c/0y9flzcweAp5x9+8Gv5ZX5e5Hw67rXJXtGb27Pw0cDruOUnH3fe7+UrDeC2wC5odbVfF4QV+wmQwe5XnWcQZm1gZ8BPhu2LXIuTOzOuC9wAMA7j5UjiEPZRz0lcTMFgFXAi+EW0lxBcMaLwOdwBPuHqnPB/xP4D8C+bALKSEHfmdm68zstrCLKbILgS7ge8Hw23fNrDrsoqZCQT/DmVkN8DPgC+7eE3Y9xeTuI+5+BdAGrDCzyAzBmdlHgU53Xxd2LSV2jbtfBawE7giGVKMiAVwF3OvuVwL9wBfDLWlqFPQzWDB2/TPgYXd/JOx6SiX4c/j3wIdCLqWYrgH+KhjD/gnwPjP7YbglFZ+77w2WncDPgRXhVlRUHUDHmL80f0oh+MuOgn6GCr6sfADY5O7fCLueYjOzZjNrCNazwPuBzeFWVTzu/iV3b3P3RcAtwJPu/u9CLquozKw6mChAMKTxASAys+DcfT/QbmaXBE3XAWU5GSIRdgFTZWY/Bq4FmsysA/hHd38g3KqK6hrgk8CGYBwb4E53fzzEmoqpFXjIzOIUTjhWuXskpyBGWAvw88I5CQngR+7+m3BLKrq/Bx4OZty8AXw65HqmpGynV4qIyORo6EZEJOIU9CIiEaegFxGJOAW9iEjEKehFRCJOQS8iEnEKehGRiFPQi4hE3P8H1DStq24uP4EAAAAASUVORK5CYII=\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1499744606513134, 1.3970589103224236, 1.9124681884207781, 6.361686926992798 \n", + "Times:\t\t168.62498827393395, 94.04820895845543, 36.664092522353194, 1.0000000000000004" + ] + }, + "execution_count": 105, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Interpolating the original function using two of the points\n", + "\n", + "The code below reverse engineers the original curve using only 2 of the estimated points" + ] + }, + { + "cell_type": "code", + "execution_count": 106, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD8CAYAAAB5Pm/hAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHW9JREFUeJzt3Xl0XGeZ5/HvU6WSVLKszVpiSbblJMZZyGLjOAHTaUgAk8CQhKUn0ECGgTZNBw6cYTJDaOYA5wwzORMCPX2gM52QNMlAJxMghEAHTAiBEAix5Wze4tiJN8mbbEebtZbqmT/qypZt2ZalKt/Srd/nHJ1776t7q57K8ruv3nrvvebuiIhIdMXCLkBERHJLQS8iEnEKehGRiFPQi4hEnIJeRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQirijsAgBqa2u9paUl7DJERKaVNWvW7Hf3ulPtlxdB39LSQmtra9hliIhMK2a2fSL7aehGRCTiFPQiIhGnoBcRiTgFvYhIxCnoRUQiTkEvIhJxCnoRkYib1kG/aU8Pt698mc6+obBLERHJW9M66LcfOMR3n3yVnQf7wy5FRCRvTeugb6goBWBv90DIlYiI5K9TBr2ZzTGzJ81so5mtN7PPB+1fM7N2M3sh+Ll2zDG3mtkWM9tkZstzVfzhoO9R0IuInMhE7nWTAr7o7s+Z2UxgjZk9Hvzu2+7+zbE7m9kFwI3AhUAj8Bsze4O7j2SzcIDa8mLMYG/3YLZfWkQkMk7Zo3f33e7+XLDeA2wEmk5yyHXAg+4+6O5bgS3A0mwUe6yieIza8hL2aehGROSETmuM3sxagEXAs0HTZ83sJTO718yqg7YmYOeYw9oY58RgZivMrNXMWjs6Ok678FENFSUaoxcROYkJB72ZlQM/Ab7g7t3AncA5wKXAbuCO0V3HOdyPa3C/y92XuPuSurpT3k75hBpmlmroRkTkJCYU9GaWIBPyP3T3hwHcfa+7j7h7GribI8MzbcCcMYc3A7uyV/LR6itK2acvY0VETmgis24MuAfY6O7fGtM+e8xuNwDrgvVHgRvNrMTM5gMLgFXZK/loDRUl7O8dYiiVztVbiIhMaxOZdbMM+Biw1sxeCNq+DHzYzC4lMyyzDfg0gLuvN7OHgA1kZuzcnIsZN6NGp1h29A7SVJXM1duIiExbpwx6d3+a8cfdHzvJMd8AvjGFuibsrDEXTSnoRUSON62vjAWorygB0BRLEZETmPZBf+Q2CJp5IyIynmkf9DVlxRTFTHPpRUROYNoHfSxm1M8sUY9eROQEpn3Qg+bSi4icTCSCXrdBEBE5sYgEfSl7uhT0IiLjiUzQdw+k6BtKhV2KiEjeiUTQj14otatTvXoRkWNFIugbDwe9nh0rInKsiAR95qIpBb2IyPEiEfQNFaWYwS59ISsicpxIBH0iHqNhZql69CIi44hE0ENm+EZBLyJyvMgE/eyqJLs1dCMicpzIBH1TVZL2zn7cj3s8rYhIQYtM0DdWljKUSnPg0FDYpYiI5JXoBH0wl363LpoSETlK5IK+XV/IiogcJXJBr5k3IiJHi0zQV5clKE3E2N2loBcRGSsyQW9mNFYmdWMzEZFjRCboITN806ahGxGRo0Qq6JuqkrS/rqAXERkrUkE/d1YZ+3sH6R8aCbsUEZG8Eamgn1NTBsDO1/tCrkREJH9EK+irM1MsdxxQ0IuIjIpU0M9Vj15E5DiRCvqaGcWUFcfZcVBBLyIy6pRBb2ZzzOxJM9toZuvN7PNBe42ZPW5mm4NlddBuZvaPZrbFzF4ys8W5/hBjamVuTRk7D2rmjYjIqIn06FPAF939fOAK4GYzuwD4EvCEuy8Angi2Aa4BFgQ/K4A7s171STRXl7FTPXoRkcNOGfTuvtvdnwvWe4CNQBNwHXBfsNt9wPXB+nXA/Z7xZ6DKzGZnvfITmFtTxo6DfbovvYhI4LTG6M2sBVgEPAs0uPtuyJwMgPpgtyZg55jD2oK2Y19rhZm1mllrR0fH6Vd+AnNrkvQPj+i+9CIigQkHvZmVAz8BvuDu3SfbdZy247rX7n6Xuy9x9yV1dXUTLeOURufS6wtZEZGMCQW9mSXIhPwP3f3hoHnv6JBMsNwXtLcBc8Yc3gzsyk65p3Z4iqWCXkQEmNisGwPuATa6+7fG/OpR4KZg/SbgZ2PaPx7MvrkC6Bod4jkTmqsV9CIiYxVNYJ9lwMeAtWb2QtD2ZeA24CEz+ySwA/hQ8LvHgGuBLUAf8ImsVnwKyeI4dTNLNMVSRCRwyqB396cZf9wd4Opx9nfg5inWNSVza8rYduBQmCWIiOSNSF0ZO2p+7QwFvYhIILJBv7d7kEODqbBLEREJXSSD/uzaGQBs3a9evYhIJIN+fp2CXkRkVCSDvmWWgl5EZFQkg740EaepKqmgFxEhokEPmS9kX1PQi4hEO+i3dvTqLpYiUvAiHfTdAykO6i6WIlLgohv0mnkjIgJEOOhH59JrnF5ECl1kg76pKkkiburRi0jBi2zQF8VjzJs1gy37esMuRUQkVJENeoAF9eUKehEpeNEO+oaZbD9wiIHhkbBLEREJTaSDvqtviLTD+f/tVyy77bc88nx72CWJiJxxkQ36R55v58HVO4HMk8nbO/u59eG1CnsRKTiRDfrbV25iMJU+qq1/eITbV24KqSIRkXBENuh3dY7/zNgTtYuIRFVkg76xKnla7SIiURXZoL9l+UKSifhRbclEnFuWLwypIhGRcBSFXUCuXL+oCYCv/3w9r/cNUzezhL+/9vzD7SIihSKyPXrIhP1Dn34zAF++9jyFvIgUpEgHPUBL7QwSceOVvbpCVkQKU+SDPhGPcU5dORt3d4ddiohIKCIf9AAXNFawYZeCXkQKU0EE/YWNlezrGaSjZzDsUkREzriCCPoLZlcAsEHDNyJSgAor6DV8IyIF6JRBb2b3mtk+M1s3pu1rZtZuZi8EP9eO+d2tZrbFzDaZ2fJcFX46KssSNFcn1aMXkYI0kR7994F3j9P+bXe/NPh5DMDMLgBuBC4MjvknM4uPc+wZd8HsCtbv6gq7DBGRM+6UQe/uTwEHJ/h61wEPuvugu28FtgBLp1Bf1lzQWMHW/YfoG0qFXYqIyBk1lTH6z5rZS8HQTnXQ1gTsHLNPW9B2HDNbYWatZtba0dExhTIm5sLGStzh5T09OX8vEZF8MtmgvxM4B7gU2A3cEbTbOPv6eC/g7ne5+xJ3X1JXVzfJMibugsbMF7Lr9YWsiBSYSQW9u+919xF3TwN3c2R4pg2YM2bXZmDX1ErMjsbKUqrLEqxr0zi9iBSWSQW9mc0es3kDMDoj51HgRjMrMbP5wAJg1dRKzA4z45I5VbzY1hl2KSIiZ9Qpb1NsZg8AbwNqzawN+CrwNjO7lMywzDbg0wDuvt7MHgI2ACngZncfyU3pp++S5iqeemUzhwZTzCiJ7B2aRUSOcsq0c/cPj9N8z0n2/wbwjakUlSuXzqki7bC2vYsrzp4VdjkiImdEQVwZO+ri5koAXtyp4RsRKRwFFfSzykuYW1OmcXoRKSgFFfRA5gvZnZp5IyKFo/CCvrmS9s5+9vUMhF2KiMgZUXBBv2huFYB69SJSMAou6C9srKQoZjy/4/WwSxEROSMKLuhLE3EubKygdZuCXkQKQ8EFPcBlLTW80NbJYCpvruUSEcmZwgz6+TUMpdK8pPveiEgBKMygb6kBYNXWid5mX0Rk+irIoK+ZUcy59eWs3qagF5HoK8igh0yvfs221xlJj3u7fBGRyCjYoF86v5qewRQv79GDSEQk2go26EfH6VdrnF5EIq5gg765uoymqiTPvHYg7FJERHKqYIMe4K3n1vKnVw9onF5EIq2gg37Zglp6BlKsbdd8ehGJrsIO+nMyT5l6enNHyJWIiOROQQf9rPISLphdwdNb9oddiohIzhR00AP8xYJantveSd9QKuxSRERyouCDftm5tQyNpHU7BBGJrIIP+staaiiOx3h6s4ZvRCSaCj7ok8VxLj+7hic37Qu7FBGRnCj4oAe46rx6Xu04xLb9h8IuRUQk6xT0wNXnNQDwxMvq1YtI9CjogbmzylhQX84TG/eGXYqISNYp6ANXn9/Aqq0H6R4YDrsUEZGsUtAH3nF+Pam089QrukpWRKJFQR9YNLea6rIET2zUOL2IRMspg97M7jWzfWa2bkxbjZk9bmabg2V10G5m9o9mtsXMXjKzxbksPpviMeOq8xr4zca9DKZGwi5HRCRrJtKj/z7w7mPavgQ84e4LgCeCbYBrgAXBzwrgzuyUeWa89+LZ9Ayk+KPufSMiEXLKoHf3p4Bj7w9wHXBfsH4fcP2Y9vs9489AlZnNzlaxubbs3FoqSov4xUu7wy5FRCRrJjtG3+DuuwGCZX3Q3gTsHLNfW9B2HDNbYWatZtba0ZEfX4AWF8VYfuFZPL5ewzciEh3Z/jLWxmkb9/FN7n6Xuy9x9yV1dXVZLmPyrr14Nj2DKf7wioZvRCQaiiZ53F4zm+3uu4OhmdGpKm3AnDH7NQO7plLgmbbsnFqSiRife+B5BoZHaKxKcsvyhVy/aNw/TERE8t5ke/SPAjcF6zcBPxvT/vFg9s0VQNfoEM908dja3QyNOP3DIzjQ3tnPrQ+v5ZHn28MuTURkUiYyvfIB4BlgoZm1mdkngduAd5rZZuCdwTbAY8BrwBbgbuDvclJ1Dt2+ctNxDwvvHx7h9pWbQqpIRGRqTjl04+4fPsGvrh5nXwdunmpRYdrV2X9a7SIi+U5Xxh6jsSp5Wu0iIvlOQX+MW5YvJJmIH9WWTMS5ZfnCkCoSEZmayc66iazR2TW3r9xEe2c/8ZjxP254o2bdiMi0pR79OK5f1MQfv3QVd3zoEkbSTkNladgliYhMmoL+JN5z8WyqyhL832e2h12KiMikKehPojQR598vmcOvN+xld5dm3YjI9KSgP4WPXjGPtDsPPLsj7FJERCZFQX8Kc2rKuGphPf+6aidDqXTY5YiInDYF/QR87M3z2N87yL+tnVa37RERART0E3LlgjoW1Jfzz79/jczFvyIi04eCfgJiMeNv//IcXt7Tw+825ce980VEJkpBP0Hvu7SRxspS7vz9q2GXIiJyWhT0E5SIx/jUX5zNqq0HWbP99bDLERGZMAX9abhx6RyqyhJ857ebwy5FRGTCFPSnoay4iBVXns2TmzpYs/3Y56WLiOQnBf1p+g9vaaG2vIT/9atNmoEjItOCgv40lRUX8dm3n8OzWw/y9BY9QFxE8p+CfhI+fPlcmqqSfHPlJtJp9epFJL8p6CehpCjOF96xgBfbuvjZi3pouIjkNwX9JH1gcTMXN1dy2y9f5tBgKuxyREROSEE/SbGY8dV/dyF7uwf5p99tCbscEZETUtBPwZvmVXPDoibufmor2w8cCrscEZFxKein6EvXnEdxUYy//+k6TbcUkbykoJ+ihopS/us15/H0lv38aE1b2OWIiBxHQZ8Ff710Lktbavjvv9jAvu6BsMsRETmKgj4LYjHjtg9cxEAqzVce0RCOiOQXBX2WnF1Xzn9+1xv49Ya9PLh6Z9jliIgcpqDPok+99Wzeem4tX//5erbs6wm7HBERQEGfVbGY8a2/uoSy4iI+98ALDAyPhF2SiMjUgt7MtpnZWjN7wcxag7YaM3vczDYHy+rslDo91FeU8s0PXczG3d18/efrwy5HRCQrPfq3u/ul7r4k2P4S8IS7LwCeCLYLylXnNfB3bzuHB1bt5IfPbg+7HBEpcLkYurkOuC9Yvw+4Pgfvkfe++K6FvH1hHV97dD2rt+khJSISnqkGvQO/NrM1ZrYiaGtw990AwbJ+iu8xLcVjxj/cuIjm6jI+84M17DjQF3ZJIlKgphr0y9x9MXANcLOZXTnRA81shZm1mllrR0fHFMvIT5XJBHd/fAmptPPxe59lf+9g2CWJSAGaUtC7+65guQ/4KbAU2GtmswGC5b4THHuXuy9x9yV1dXVTKSOvnVtfzj03Xcae7gH+4/dX65bGInLGTTrozWyGmc0cXQfeBawDHgVuCna7CfjZVIuc7t40r5rvfmQx63d18zf3t9I/pGmXInLmTKVH3wA8bWYvAquAf3P3XwG3Ae80s83AO4Ptgnf1+Q3c/sGLeea1A3zyvtUKexE5Y4ome6C7vwZcMk77AeDqqRQVVe9f3AzAF3/0Ip+8bzX33HQZyeJ4yFWJSNTpytgz7P2Lm/nWX13Cn187wEfveZbXDw2FXZKIRJyCPgQ3LGrmOx9ZzNr2Lj7wf/7EzoOaeikiuTPpoRuZmmsvmk1teQmfum8177/zT3z8ink8uHonuzr7aaxKcsvyhVy/qCnsMkUkAtSjD9HS+TX85DNvYTiV5o7HX6G9sx8H2jv7ufXhtTzyfHvYJYpIBCjoQ7agYSal43wh2z88wu0rN4VQkYhEjYI+D+ztGv/xg7s6+89wJSISRQr6PNBYlRy3vTKZ0GMJRWTKFPR54JblC0kmjh6+iRl09g/zN/e3srtLPXsRmTwFfR64flET//P9F9FUlcSApqokd3zwEr7ynvN5est+3nHH7/mXP25lJK3evYicPsuHoYElS5Z4a2tr2GXkpZ0H+/jKI+v4/SsdXNRUydfedwFvmlcTdlkikgfMbM2Yhz6dkHr0eW5OTRnf/8RlfOcji9jbPcAH7nyGz/xgDdv2Hwq7NBGZJnTB1DRgZrz34kauOq+eu5/ayj8/9Sq/2biXv758Hn/7l+dwVmVp2CWKSB7T0M00tK97gG//5hUeam0jbsaHljTzmbedQ3N1WdilicgZNNGhGwX9NLbjQB93/v5VfrxmJ+5w3aVNfGJZC29sqgy7NBE5AxT0BWRXZz93PfUa/2/1TvqHR1gyr5qb3tLCu994Fom4voYRiSoFfQHq6h/mR607uf+Z7ew42Ef9zBJuWNzEBxc3s6BhZtjliUiWKegL2Eja+d2mfTywagdPbupgJO1cMqeKDy5u4j0XN1IzozjsEkUkCxT0AkBHzyA/e6GdH69p4+U9PcQMLp8/i2suOovlF55FQ4Vm7IhMVwp6OYq7s2F3N79cu4dfrtvNqx2ZefhvmlfNVefVc+WCOi5srCAWs5ArFZGJUtDLSW3e28Ov1u1h5YY9rGvvBmDWjGL+YkEtV76hjmXn1qq3L5LnFPQyYR09g/xhcwdPvdLBU5v3czB4ju3cmjKWzq9haUsNl82voWVWGWbq8YvkCwW9TEo67azf1c2zWw+wautBWre/fjj4a8tLuHROFRc3V3JRcyUXNVVSW14ScsUihUtBL1nh7rza0cuqra/Tuu0gL7Z18tr+Q4z+Z9NYWcpFzZVc2FjJGxpmsvCsmcytKSOusX6RnJto0OteN3JSZsa59TM5t34mH7l8LgA9A8Os39XNuvYuXmrr4qW2Tlau33v4mJKiGOfUlbPwrJksaChnQf1M5teW0VxdRmni+McmikhuKejltM0sTXDF2bO44uxZh9sODabYsq+XTXt72Ly3h017e3nm1QP8dMwDzs2gsTLJvFllzJs1g5ZgObemjKaqJBXJIn0HIJIDCnrJihklRVwyp4pL5lQd1d7VP8yrHb3sONDHtgOH2B4sV67fc3js//BrFMeZXZWksSpJY2UpjVVJZleW0lSV5KzKUupmllBeopOByOlS0EtOVSYTLJ5bzeK51cf9rqt/mB0H+thxsI/dXf20d/azu3OAXV39bNjVzf7eweOOKU3EqC0voW5mCXXB8vB2sF5dlqC6rJiKZELfFYigoJcQVSYTmdk7zePfbXNgeIQ9XZng39M1wP7eQTp6BtnfO0RHzyDbD/QdNSvoWGaZ96guKw6WmfWqsmKqyxJUzSimKpmgvLSIitIiZpYmKC8pYmZpETOKi3TxmESGgl7yVmkiTkvtDFpqZ5x0v+GRNAcPZcK/o3eQzr4hXj80nFn2DfN63xBd/cN09A7yyt5eOvuGODQ0ctLXNIPy4iLKSzPBP/YkMHoiKCuOkzy8jFMW/CQTRUfWi+OUBfuUFMU07CShyFnQm9m7gf8NxIHvufttuXovKWyJeIyGitLTupJ3MDVCV98wXf3DdA+k6B1M0TMwTO9Aip6BzHrPYGa9dyBFz2DmhLHzYB/dAykODaboHz75yeJYMYNk4sjJoTQRo6QocwIoGbteFKM0Mdp+pK2kKB7sd2Tf0f2Ki2Ik4jGK4kZxfPz1RLCu4azwPPJ8O7ev3MSuzn4aq5Lcsnwh1y9qyvn75iTozSwOfBd4J9AGrDazR919Qy7eT+R0lRTFqa+IUz+F2zyk085AaoS+oRH6hzLLvqHUkfXhEfqHUkH7kX36hzNtQ6k0A8MjDKbSDAyn6eofZnA4zWAqzWAq0z44nGYgNUI2L3eJGYdDPzHmBDC6XhSPURw3io75fVHMKIob8VhmPWZGUcyIx4NlzIjb2O3YkfbYkX2KYkYsNv4+R+8XIxaDoljm5BQzgqVhY9ZjRrDMHGfB9lHrwe8txpH1Y14j139tPfJ8O7c+vPZwB6G9s59bH14LkPOwz1WPfimwxd1fAzCzB4HrAAW9REYsZsGwTG5HQN2dVNoPnxQyJ4Dj11PpNEMpZ3gksz6ccoZG0qRG0gyPOMNB2/BI+qj1Y48bSnnm+JHMPr2pFEOpNCNpP/yTOrxMM5KGkXT6cNvYfaYTO3zCOHLiiFnm3/PoiWP0hBAfc3IYbY+ZBfuCceQkE4tltl/e083wyNH/TPqHR7h95aZpG/RNwM4x223A5Tl6L5FIM7PDPevp9PgYdyftkEqnSacJTgo+7gnh2BPF4RPJiJP2zEnIHdLBa46kHXdnJNh2z+yfHt1n7Pp422Ne47j18Y457vUy6yPuh+vKvP+R7cxy9J+DHxfyo3Z19uf830Wugn68v4GO+pRmtgJYATB37twclSEiYcn0fCEeG70aurCvil52229pHyfUG6uSOX/vXD1QtA2YM2a7Gdg1dgd3v8vdl7j7krq6uhyVISKSH25ZvpDkMbcASSbi3LJ8Yc7fO1c9+tXAAjObD7QDNwIfydF7iYjkvdFx+MjMunH3lJl9FlhJ5u+1e919fS7eS0Rkurh+UdMZCfZj5Wy6gLs/BjyWq9cXEZGJydUYvYiI5AkFvYhIxCnoRUQiTkEvIhJxCnoRkYhT0IuIRJyCXkQk4syzef/TyRZh1gFsn+ThtcD+LJaTj6L+GfX5pjd9vvDMc/dT3kMmL4J+Ksys1d2XhF1HLkX9M+rzTW/6fPlPQzciIhGnoBcRibgoBP1dYRdwBkT9M+rzTW/6fHlu2o/Ri4jIyUWhRy8iIicxbYPezO41s31mti7sWnLBzOaY2ZNmttHM1pvZ58OuKZvMrNTMVpnZi8Hn+3rYNeWCmcXN7Hkz+0XYteSCmW0zs7Vm9oKZtYZdT7aZWZWZ/djMXg7+X3xz2DVNxrQdujGzK4Fe4H53f2PY9WSbmc0GZrv7c2Y2E1gDXO/uG0IuLSvMzIAZ7t5rZgngaeDz7v7nkEvLKjP7T8ASoMLd3xt2PdlmZtuAJe6er/PMp8TM7gP+4O7fM7NioMzdO8Ou63RN2x69uz8FHAy7jlxx993u/lyw3gNsBM78o2lyxDN6g81E8DM9ex0nYGbNwHuA74Vdi5w+M6sArgTuAXD3oekY8jCNg76QmFkLsAh4NtxKsisY1ngB2Ac87u6R+nzAPwD/BUiHXUgOOfBrM1tjZivCLibLzgY6gH8Jht++Z2Yzwi5qMhT0ec7MyoGfAF9w9+6w68kmdx9x90uBZmCpmUVmCM7M3gvsc/c1YdeSY8vcfTFwDXBzMKQaFUXAYuBOd18EHAK+FG5Jk6Ogz2PB2PVPgB+6+8Nh15MrwZ/DvwPeHXIp2bQMeF8whv0gcJWZ/SDckrLP3XcFy33AT4Gl4VaUVW1A25i/NH9MJvinHQV9ngq+rLwH2Oju3wq7nmwzszozqwrWk8A7gJfDrSp73P1Wd2929xbgRuC37v7RkMvKKjObEUwUIBjSeBcQmVlw7r4H2GlmC4Omq4FpORmiKOwCJsvMHgDeBtSaWRvwVXe/J9yqsmoZ8DFgbTCODfBld38sxJqyaTZwn5nFyXQ4HnL3SE5BjLAG4KeZPglFwL+6+6/CLSnrPgf8MJhx8xrwiZDrmZRpO71SREQmRkM3IiIRp6AXEYk4Bb2ISMQp6EVEIk5BLyIScQp6EZGIU9CLiEScgl5EJOL+P7IeihVMYkBBAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "(array([168.62498827, 94.03830275, 36.64656385, 0.97773399]),\n", + " array([168.62498827, 94.04820896, 36.66409252, 1. ]))" + ] + }, + "execution_count": 106, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "x1, x2 = pred.low_bucket.feerate**ALPHA, pred.normal_bucket.feerate**ALPHA\n", + "y1, y2 = pred.low_bucket.estimated_seconds, pred.normal_bucket.estimated_seconds\n", + "b2 = (y1 - y2*x2/x1) / (1 - x1/x2)\n", + "b1 = (y1 - b2) * x1\n", + "def p(ff):\n", + " return b1/ff**ALPHA + b2\n", + "\n", + "plt.figure()\n", + "plt.plot(x, p(x))\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "p(pred.feerates()), pred.times()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Challenge: outliers\n", + "\n", + "The segment below illustrates a challenge in the current approach. It is sufficient to add a single outlier \n", + "to the total weight (with `feerate=100`), and the `feerate_to_time` function is notably influenced. In truth, this tx should not affect our prediction because it only captures the first slot of each block, however because we sample with repetition it has a significant impact on the function. The following figure shows the `feerate_to_time` function with such an outlier " + ] + }, + { + "cell_type": "code", + "execution_count": 107, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAG1xJREFUeJzt3X1wHPWd5/H3t+dJD7bkJ/lRfiIxEEwIJF7wHrm7HA6PSWGqNslxt0tce9RRdWGX7N3ebUJSBXck2SN3uZBNEXJFYgdnNxXiImQhVLjgNbAbskAwT+bBGBubYPlRfpJlW5Ilzff+6JY8lmakkZGmx9OfV5Vqun/9m57vyJY++vWve9rcHRERSZ4g7gJERCQeCgARkYRSAIiIJJQCQEQkoRQAIiIJpQAQEUkoBYCISEIpAEREEkoBICKSUOm4CxjJjBkzfNGiRXGXISJyVnnxxRcPuHvLaP2qOgAWLVrExo0b4y5DROSsYma/L6efDgGJiCSUAkBEJKEUACIiCaUAEBFJKAWAiEhCKQBERBJKASAiklA1GQB7Orr49hNb2N5+LO5SRESqVk0GwIHOk3z3yW2803487lJERKpWTQZAXSZ8W929/TFXIiJSvWo0AFIAdCkARERKqskAqM+GAaARgIhIaTUZAAMjAAWAiEhptRkA6fBtdZ3Mx1yJiEj1qskASKcCMinTHICIyAhqMgAAcumUDgGJiIyg7AAws5SZvWxmj0Xri83seTPbamY/M7Ns1J6L1rdF2xcV7OP2qH2LmV093m+mUC4dKABEREYwlhHAF4HNBevfBO5x9yXAYeDmqP1m4LC7fxC4J+qHmV0A3AgsBa4B7jOz1PsrvzQFgIjIyMoKADNrBT4F/DBaN+AK4KGoy1rghmh5ZbROtH1F1H8l8KC797j7DmAbcOl4vIlisulAcwAiIiModwTwHeCvgIHTaqYDR9y9L1pvA+ZFy/OAnQDR9o6o/2B7keeMuzAAdBaQiEgpowaAmX0a2O/uLxY2F+nqo2wb6TmFr3eLmW00s43t7e2jlVdSNqVDQCIiIylnBHA5cL2ZvQs8SHjo5zvAFDNLR31agd3RchswHyDa3gwcKmwv8pxB7n6/uy9z92UtLS1jfkMDMumArpMKABGRUkYNAHe/3d1b3X0R4STuk+7+x8BTwGeibquAR6LlR6N1ou1PurtH7TdGZwktBpYAvxu3dzJETnMAIiIjSo/epaQvAQ+a2deBl4HVUftq4G/NbBvhX/43Arj7G2a2DngT6ANudfcJ+w2tQ0AiIiMbUwC4+9PA09HydoqcxePu3cBnSzz/G8A3xlrkmcjpEJCIyIhq9krgrK4EFhEZUQ0HQEB3n04DFREppWYDIJcO6M87vf0KARGRYmo6AEB3BRMRKaVmAyCb1n2BRURGUrMBMDAC6NZNYUREiqrZAMimdWN4EZGR1HAA6BCQiMhIajYANAksIjIyBYCISELVbABkU+Fb61EAiIgUVbMBoBGAiMjIajYABiaBT+gD4UREiqrZAKjLRKeBKgBERIqq+QA43qMAEBEppmYDIBUY6cA40ds3emcRkQSq2QCAcB7ghEYAIiJF1XYApAKOn9QIQESkmJoOgExKIwARkVJqOgDSKdMIQESkhJoPAF0HICJSXE0HQCYION6jEYCISDG1HQBpBYCISCm1HQAp47gOAYmIFFXTAZBNBZzQJLCISFE1HQCZVEB3b5583uMuRUSk6tR8AIA+ElpEpJgaDwAD0LUAIiJF1HQADNwVTFcDi4gMV9MBkIluCqMRgIjIcLUdACndFUxEpJQaD4BoDkAXg4mIDFPjAaARgIhIKTUdAAOTwBoBiIgMV9MBoBGAiEhpNR4Aug5ARKSUmg6AVGAEpkNAIiLFjBoAZlZnZr8zs1fN7A0z+x9R+2Ize97MtprZz8wsG7XnovVt0fZFBfu6PWrfYmZXT9SbKng9cukUx7oVACIiQ5UzAugBrnD3jwAXA9eY2XLgm8A97r4EOAzcHPW/GTjs7h8E7on6YWYXADcCS4FrgPvMLDWeb6aYXDqgUwEgIjLMqAHgoWPRaib6cuAK4KGofS1wQ7S8Mlon2r7CzCxqf9Dde9x9B7ANuHRc3sUIMumATh0CEhEZpqw5ADNLmdkrwH5gPfAOcMTdB36ztgHzouV5wE6AaHsHML2wvchzJkw2FdDZ3TvRLyMictYpKwDcvd/dLwZaCf9q/1CxbtGjldhWqv00ZnaLmW00s43t7e3llDeibDrgaJdGACIiQ43pLCB3PwI8DSwHpphZOtrUCuyOltuA+QDR9mbgUGF7kecUvsb97r7M3Ze1tLSMpbyisqmAzh6NAEREhirnLKAWM5sSLdcDnwQ2A08Bn4m6rQIeiZYfjdaJtj/p7h613xidJbQYWAL8brzeSCnZdKCzgEREikiP3oU5wNrojJ0AWOfuj5nZm8CDZvZ14GVgddR/NfC3ZraN8C//GwHc/Q0zWwe8CfQBt7r7hF+iO3AWkLsTzkWLiAiUEQDuvgm4pEj7doqcxePu3cBnS+zrG8A3xl7mmcumA/ryTk9fnrrMhJ91KiJy1qjpK4EhDABA1wKIiAxR8wGQGwwATQSLiBSq+QD4w+NP8kz2NhZ/rxXuuRA2rYu7JBGRqlDOJPBZ67z9j/PJfd8iG/SEDR074Ze3hcsXfS6+wkREqkBNjwA+/t59ZL3n9MbeLthwVzwFiYhUkZoOgMk9+4pv6GirbCEiIlWopgOgMzer+Ibm1soWIiJShWo6AJ5Z8AV6g7rTGzP1sOKOeAoSEakiNT0JvGXmtQB85O3vMtcOYs2t4S9/TQCLiNR2AEAYAl/aej7XXTiHb37morjLERGpGjV9CGiAPhFURGS4RARALh3Q0aUAEBEplIwAyKQ4fFwBICJSKBEBUJcOOHLiZNxliIhUlUQEQC6T0iEgEZEhEhEAdZmA4yf76e3Px12KiEjVSEYApMMbwWgUICJySjICILoT2JETCgARkQEJCYDwbXZ0aSJYRGRAIgIgpxGAiMgwiQiAuui2kAoAEZFTkhEAAyMATQKLiAxKRADk0gEGdOhiMBGRQYkIADOjLpPSCEBEpEAiAgCgPpPSHICISIHEBEAuE2gEICJSIDEBkE0FHDmuOQARkQGJCYBcJuCQJoFFRAYlJgAasmkOawQgIjIoMQFQn01x/GQ/3b39cZciIlIVEhMADdHFYAc1ChARAZIUANkoAI71xFyJiEh1SEwA1Gc1AhARKZScABg4BHRMASAiAgkKgIZsGtAhIBGRAYkJgEzKSAemQ0AiIpHEBICZ0ZhLc0AjABERoIwAMLP5ZvaUmW02szfM7ItR+zQzW29mW6PHqVG7mdl3zWybmW0ys48W7GtV1H+rma2auLdVXH0mpTkAEZFIOSOAPuAv3f1DwHLgVjO7APgysMHdlwAbonWAa4El0dctwPchDAzgTuAy4FLgzoHQqJS6TKARgIhIZNQAcPc97v5StNwJbAbmASuBtVG3tcAN0fJK4Mceeg6YYmZzgKuB9e5+yN0PA+uBa8b13YyiPptSAIiIRMY0B2Bmi4BLgOeBWe6+B8KQAGZG3eYBOwue1ha1lWof+hq3mNlGM9vY3t4+lvJG1ZBJc+j4Sdx9XPcrInI2KjsAzGwS8HPgL9z96Ehdi7T5CO2nN7jf7+7L3H1ZS0tLueWVpT6borff6ezpG9f9ioicjcoKADPLEP7y/4m7Pxw174sO7RA97o/a24D5BU9vBXaP0F4xjbnwYrD9R7sr+bIiIlWpnLOADFgNbHb3bxdsehQYOJNnFfBIQfvno7OBlgMd0SGiXwNXmdnUaPL3qqitYiblwovB9h3VPICISLqMPpcDNwGvmdkrUdtXgLuBdWZ2M/Ae8Nlo26+A64BtwAngTwHc/ZCZfQ14Iep3l7sfGpd3UabGKAD2dmgEICIyagC4+zMUP34PsKJIfwduLbGvNcCasRQ4ngZHAJ0KABGRxFwJDJBJBdRlAvZpBCAikqwAgHAUoDkAEZEEBkBDNs1enQUkIpK8AGjMpRQAIiIkMQCyado7e8jndTWwiCRb4gJgUi5Nf951XwARSbzkBUDdwMVgOgwkIsmWuABozOpiMBERSGAATI5GALuOdMVciYhIvBIXAA3ZFOnAaDt8Iu5SRERilbgAMDOa6zO0HdYIQESSLXEBAOFE8E6NAEQk4RIZAE11GdoOaQQgIsmW0ABIc6Srl2O6M5iIJFgiA2BF7z/yTPY2Gv/nDLjnQti0Lu6SREQqrpwbwtSU8/Y/zifb/w/ZIPpE0I6d8MvbwuWLPhdfYSIiFZa4EcDH37uPrA/5OOjeLthwVzwFiYjEJHEBMLlnX/ENHW2VLUREJGaJC4DO3KziG5pbK1uIiEjMEhcAzyz4Ar1B3emNmXpYcUc8BYmIxCRxk8BbZl4LwLJ37mVGfzs2ZR624k5NAItI4iQuACAMgV/0Xc76zft46qZPsHhGY9wliYhUXOIOAQ2Y0pABYMeBYzFXIiISj8QGwNTGLADb24/HXImISDwSGwD1mRT1mRTbDygARCSZEhsAEB4G2qERgIgkVOIDYOv+zrjLEBGJRaIDYMakHAeOneTgsZ7RO4uI1JjEBwDAlr0aBYhI8iQ8AMIzgTYrAEQkgRIdAA3ZNJNyad7aczTuUkREKi7RAQAwrTHLZgWAiCRQ4gNgxqQsW/cfo68/H3cpIiIVpQCYlKOnL8+7B3U9gIgkS+IDoGVyeCbQpraOmCsREamsxAfAtMYs2XTAKzuPxF2KiEhFJT4AAjNmTc7x8nsKABFJllEDwMzWmNl+M3u9oG2ama03s63R49So3czsu2a2zcw2mdlHC56zKuq/1cxWTczbOTMzm+rYvOco3b39cZciIlIx5YwAHgCuGdL2ZWCDuy8BNkTrANcCS6KvW4DvQxgYwJ3AZcClwJ0DoVENZjfV0Zd33tit00FFJDlGDQB3/yfg0JDmlcDaaHktcENB+4899BwwxczmAFcD6939kLsfBtYzPFRiM7s5vEfwq5oHEJEEOdM5gFnuvgcgepwZtc8Ddhb0a4vaSrUPY2a3mNlGM9vY3t5+huWNzaRcmqa6NC/+/nBFXk9EpBqM9ySwFWnzEdqHN7rf7+7L3H1ZS0vLuBY3krlT6vnndw7gXrQsEZGac6YBsC86tEP0uD9qbwPmF/RrBXaP0F41WqfWc/hEL2/v0z2CRSQZzjQAHgUGzuRZBTxS0P756Gyg5UBHdIjo18BVZjY1mvy9KmqrGvOnNgDw7DsHYq5ERKQyyjkN9KfAs8B5ZtZmZjcDdwNXmtlW4MpoHeBXwHZgG/AD4AsA7n4I+BrwQvR1V9RWNZrqMzTXZ3h2+8G4SxERqYj0aB3c/d+V2LSiSF8Hbi2xnzXAmjFVV2HzptTz7DsH6c87qaDYtIWISO1I/JXAhRZOb+Bodx8vvaezgUSk9ikACiyc3kBg8A+b98VdiojIhFMAFMilU7RObeAf3lQAiEjtUwAMsXhGI++0H+fdA7o/gIjUNgXAEItnNALw6zf2xlyJiMjEUgAM0VyfYXZTHX//yq64SxERmVAKgCLOmz2ZzXs62bqvM+5SREQmjAKgiCUzJxEYGgWISE1TABTRmEszf1oDD7+0i/68PhxORGqTAqCEpXOa2NPRzVNv7R+9s4jIWUgBUMI5LZOYXJdm7bPvxl2KiMiEUACUkAqMC+c285utB9jero+IFpHaowAYwdK5TaQC4we/2R53KSIi404BMILGXJqlc5pYt7GNtsMn4i5HRGRcKQBGsWzRVADue/qdmCsRERlfCoBRTK7LcMGcJta9sJMd+nwgEakhCoAyXLZ4GqnA+Npjb8ZdiojIuFEAlKExl+YPFk3jybf28/QWXRcgIrVBAVCmi+dPYVpjlq/+4nU6u3vjLkdE5H1TAJQpFRgrzp/J7o4uvv7Y5rjLERF53xQAYzB3Sj0fWzCVn23cyeOv7Ym7HBGR90UBMEbLz5nOnOY6/su6V9myVx8XLSJnLwXAGKUC47oL55AKjP/4440cONYTd0kiImdEAXAGJtWlue7Ds9nT0cVNq5+no0uTwiJy9lEAnKE5zfV86sNzeHvfMT6/5nmOnDgZd0kiImOiAHgfFk5v5NoLZ/P6rqP80ff/mV1HuuIuSUSkbAqA9+kDLZO44eK57Drcxcp7n+G57QfjLklEpCwKgHHQOrWBz3yslbzDv//Bc9z75Fb6+vNxlyUiMiIFwDiZPinHv102nyUzJ/GtJ97mhu/9ltd3dcRdlohISQqAcZRNB1y9dDbXXTibHQePc/29z/CVX7zG3o7uuEsTERkmHXcBtcbMWDJrMvOnNfDc9oP87IWd/PzFNv5k+UL+9PJFtE5tiLtEERFAATBh6jIpPnHeTC5ZMJXntx/kR7/dwY9+u4Orl87mpuULWX7OdILA4i5TRBJMATDBmuszXLV0Nss/MJ1NbR08vaWdx1/fy6ymHNd/ZC6fvmguH57XrDAQkYpTAFRIU12Gj39wBpctnsaOA8fZsreTNb99lx/8ZgfTGrN84rwWPnHeTJYvnsbMprq4yxWRBFAAVFgmFXDurMmcO2syXb39/P7Acd49eILHX9vLwy/tAmDelHqWLZrKxxZOZencJs6dNZnJdZnRd75pHWy4CzraoLkVVtwBF31ugt+RiJytFAAxqs+kOH9OE+fPaSLvzv6jPezu6GJPRzcbNu/nkVd2D/adO6WOC+Y0sWTWZBZOa2DB9AYWTGtgTnM9qcDCX/6/vA16o6uRO3aG66AQEJGiKh4AZnYN8DdACvihu99d6RqqUWDG7OY6ZjeHh3/cnc7uPg4c6+HA8ZMc7OzhlZ1HePKt/eT91PPSgTFvaj0PdX+Vlv4hH0XR20XvE/+dYx+4gab6TBgUIiKRigaAmaWA7wFXAm3AC2b2qLvrbutDmBlN9Rma6jOc03KqPZ93Onv66OjqHfw62tXL9P72ovtJde7mkq+tB2ByXZop9RmmNmaZ0pBlSn2G5voMjbk0jdkUDbk0k3IpGrJpGqPHSbk0DdkUjbk0uXRALp0ilw40aS0ynmI6fFvpEcClwDZ33w5gZg8CKwEFQJmCwGiOfnEXOrZxFk09e4f1P5xp4V8vbKG7tz/86svT0dVLe2cP3b399PTlOdmXp69wWFGGdGDk0gHZgVDIBNSlU2TTAXWZU0GRSQWkU0Y6MNKpgEzKSAVGOggG28JHI5MKom02ZFv4PDMjZUZg4fchKLKcsqhfEK4XLod9jCA41S8wou0W7SfsZwbGwGO4n/AxbMcouc2ibBxYH6hjsI8pPKVAjIdvKx0A84CdBettwGUVrqEmPbPgC1z5zl+TyZ+66rg3qOP5xX/GxTOnjPr8/rzT25+ntz8MhN7+gvX+PL19Tl8+T3/e6cv7aY/hcp6+fJ7u7n4OnzjVnncn7+Ehrf684w79HrXno+W8M7b4qQ3DAoXSITIQNhQJplP9TgXN4GsMyZrTVodsLPW80/c4dNuphbL6FWkofN5I9RYG59D9lVvvSO2l9j+8pvLqLedFDVh9+CvMzA8/fMuGu2ouAIp9f0772TezW4BbABYsWHDGL7Rs0TQ+tnDqGT//7LMEXpuDFwwj0yvu4NoPf5Zr4y6tDPkoUMIgcfr6nb7+aDkKJ3fIFwTJwPJgwERhkx8MnjBk8gXbfOhzouXB/gXPdwdn4BHcgdPWfbB9YJ3B9VP98vni7YXrlNjf4OuOsM0JGwtrCCs9XcGmItuKR/DQ5sKoPm1/w/qV3reXWBn6Z0Cpesutafhrjf17M1Ltw75lJb6HpV5zQMvBA8U3dLSNuL/xUOkAaAPmF6y3ArsLO7j7/cD9AMuWLXtffxgmbqh90efO2jN+UikjlYIcqbhLEamse1rDwz5DNbdO+EtX+sPgXgCWmNliM8sCNwKPVrgGEZHqseIOyNSf3papD9snWEVHAO7eZ2Z/Bvya8DTQNe7+RiVrEBGpKgOj9gScBYS7/wr4VaVfV0SkasV0+Fb3AxARSSgFgIhIQikAREQSSgEgIpJQCgARkYRSAIiIJJQCQEQkoRQAIiIJZaU+BKoamFk78Pu464jMAEp8alNVUH1nrpprg+qur5prg+qubyJrW+juLaN1quoAqCZmttHdl8VdRymq78xVc21Q3fVVc21Q3fVVQ206BCQiklAKABGRhFIAlO/+uAsYheo7c9VcG1R3fdVcG1R3fbHXpjkAEZGE0ghARCShFACjMLP5ZvaUmW02szfM7Itx1zSUmaXM7GUzeyzuWoYysylm9pCZvRV9D/8w7poKmdl/jv5dXzezn5pZXYy1rDGz/Wb2ekHbNDNbb2Zbo8fYbnRdor7/Hf3bbjKzX5jZlGqqr2DbfzUzN7MZ1VSbmf25mW2J/g/+r0rXpQAYXR/wl+7+IWA5cKuZXRBzTUN9EdgcdxEl/A3w/9z9fOAjVFGdZjYPuA1Y5u4XEt6l7sYYS3oAuGZI25eBDe6+BNgQrcflAYbXtx640N0vAt4Gbq90UQUeYHh9mNl84ErgvUoXVOABhtRmZv8GWAlc5O5LgW9VuigFwCjcfY+7vxQtdxL+ApsXb1WnmFkr8Cngh3HXMpSZNQH/ClgN4O4n3f1IvFUNkwbqzSwNNAC74yrE3f8JODSkeSWwNlpeC9xQ0aIKFKvP3Z9w975o9Tlg4u9kXkKJ7x/APcBfAbFNeJao7T8Bd7t7T9Rnf6XrUgCMgZktAi4Bno+3ktN8h/A/dz7uQoo4B2gHfhQdovqhmTXGXdQAd99F+FfXe8AeoMPdn4i3qmFmufseCP8YAWbGXM9I/gPweNxFFDKz64Fd7v5q3LUUcS7wL83seTP7RzP7g0oXoAAok5lNAn4O/IW7H427HgAz+zSw391fjLuWEtLAR4Hvu/slwHHiPYRxmuh4+kpgMTAXaDSzP4m3qrOTmX2V8HDpT+KuZYCZNQBfBe6Iu5YS0sBUwkPL/w1YZ2ZWyQIUAGUwswzhL/+fuPvDcddT4HLgejN7F3gQuMLM/i7ekk7TBrS5+8CI6SHCQKgWnwR2uHu7u/cCDwP/IuaahtpnZnMAoseKHyYYjZmtAj4N/LFX13nlHyAM91ejn5FW4CUzmx1rVae0AQ976HeEo/iKTlIrAEYRJfJqYLO7fzvuegq5++3u3uruiwgnL59096r5C9bd9wI7zey8qGkF8GaMJQ31HrDczBqif+cVVNEkdeRRYFW0vAp4JMZahjGza4AvAde7+4m46ynk7q+5+0x3XxT9jLQBH43+X1aDvweuADCzc4EsFf7gOgXA6C4HbiL86/qV6Ou6uIs6i/w58BMz2wRcDPx1zPUMikYmDwEvAa8R/jzEdnWmmf0UeBY4z8zazOxm4G7gSjPbSngmy91VVt+9wGRgffSz8X+rrL6qUKK2NcA50amhDwKrKj2C0pXAIiIJpRGAiEhCKQBERBJKASAiklAKABGRhFIAiIgklAJARCShFAAiIgmlABARSaj/D6Gz7+yGqKSPAAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1539704395225572, 1.4115360845240776, 4.139754128892224, 16.2278954349457 \n", + "Times:\t\t2769.889957638353, 1513.4606486459202, 60.00000000000002, 1.0000000000000007" + ] + }, + "execution_count": 107, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "estimator = FeerateEstimator(total_weight=total_weight + 100**ALPHA, \n", + " inclusion_interval=avg_mass/network_mass_rate)\n", + "\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Outliers: solution\n", + "\n", + "Compute the estimator conditioned on the event the the top most transaction captures the first slot. This decreases `total_weight` on the one hand (thus increasing `p`), while increasing `inclusion_interval` on the other, by capturing a block slot. If this estimator gives lower prediction times we switch to it, and then repeat the process with the next highest transaction. The process convegres when the estimator is no longer improving or if all block slots are captured. " + ] + }, + { + "cell_type": "code", + "execution_count": 108, + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAD8CAYAAAB+UHOxAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAHmVJREFUeJzt3X2QXXWd5/H39z72U/op6TzQnZggEVREwR7Ah3IdIw5RxzCzYuG4Gl1qU7MwPoyzpehUyYxTM+PUuAO6A9REQeMui7DImjgLg5mAhSgBwlMMhJAQIOkkJB066Tx2p/ve7/5xTie307e7k7597+m+5/OqunXP+Z3fOed7fehPfufR3B0REYmfRNQFiIhINBQAIiIxpQAQEYkpBYCISEwpAEREYkoBICISUwoAEZGYGjcAzOwOM9tnZpuKLPtvZuZmNiucNzP7vpltM7ONZnZJQd/lZrY1/Cyf3J8hIiJn60xGAD8Grjy90czmA1cAOwqalwKLw88K4LawbytwI3AZcClwo5m1lFK4iIiUJjVeB3d/xMwWFll0E/A1YHVB2zLgJx7cXrzezJrNbB7wQWCtu/cAmNlaglC5a6x9z5o1yxcuLLZrEREZzVNPPbXf3dvG6zduABRjZp8Adrn7c2ZWuKgd2Fkw3xW2jdY+poULF7Jhw4aJlCgiEltm9tqZ9DvrADCzOuAvgY8UW1ykzcdoL7b9FQSHj1iwYMHZliciImdoIlcBvRlYBDxnZq8CHcDTZjaX4F/28wv6dgC7x2gfwd1Xununu3e2tY07ghERkQk66wBw99+5+2x3X+juCwn+uF/i7q8Da4DPhVcDXQ70uvse4EHgI2bWEp78/UjYJiIiETmTy0DvAh4DzjezLjO7dozu9wPbgW3AD4DrAMKTv38DPBl+vj10QlhERKJhU/l9AJ2dna6TwCIiZ8fMnnL3zvH66U5gEZGYUgCIiMRUVQbA4b4Bblr7Es/uPBh1KSIiU1ZVBkAu73xv3Vaefu1A1KWIiExZVRkA9dng/rYj/YMRVyIiMnVVZQCkkwlq0gkFgIjIGKoyACAYBRzuUwCIiIymegMgk+Jw30DUZYiITFlVGwC1maQOAYmIjKFqA6Auk+SIDgGJiIyqqgPgkA4BiYiMqooDQCeBRUTGUsUBoHMAIiJjqeoAONo/yFR+2qmISJSqNgDqsynyDsdO5KIuRURkSqraAKjLJAE9DkJEZDRVGwD1meB5QDoRLCJSXNUGgEYAIiJjq9oAGHoiqB4HISJSXNUGwMkRgA4BiYgUNW4AmNkdZrbPzDYVtP2jmb1oZhvN7P+aWXPBsm+Y2TYz22Jmf1DQfmXYts3Mbpj8nzJc3dA5AB0CEhEp6kxGAD8GrjytbS1wobtfBLwEfAPAzN4GXAO8PVznVjNLmlkSuAVYCrwN+HTYt2zqssEIQCeBRUSKGzcA3P0RoOe0tl+6+9Bf1vVARzi9DPipu/e7+yvANuDS8LPN3be7+wngp2Hfshm6CkiHgEREipuMcwD/GXggnG4HdhYs6wrbRmsvm2TCSCeNI/06CSwiUkxJAWBmfwkMAncONRXp5mO0F9vmCjPbYGYburu7SymPbErPAxIRGc2EA8DMlgMfBz7jpx640wXML+jWAeweo30Ed1/p7p3u3tnW1jbR8gDIpBIc0iEgEZGiJhQAZnYl8HXgE+5+rGDRGuAaM8ua2SJgMfAE8CSw2MwWmVmG4ETxmtJKH186aToHICIyitR4HczsLuCDwCwz6wJuJLjqJwusNTOA9e7+p+7+vJndA7xAcGjoenfPhdv5M+BBIAnc4e7Pl+H3DJNJJjh0XOcARESKGTcA3P3TRZpvH6P/3wJ/W6T9fuD+s6quRNl0kl4FgIhIUVV7JzBATSqhABARGUVVB0A2HbwXWC+FEREZqboDIJVgIOf0DeSjLkVEZMqp6gCoSQWPg9BhIBGRkao6ALLp4OcpAERERqruAEgpAERERlPVAVCT1iEgEZHRVHUAaAQgIjK6qg4AjQBEREZX1QGQ0QhARGRUVR0ACTNq0noekIhIMVUdABC8E0ABICIyUgwCQM8DEhEpRgEgIhJTVR8AmVSCg8cUACIip6v6AKjROwFERIqq+gDIphIc6lMAiIicrvoDIJ2kfzBP30Au6lJERKaUqg+AmvBmMF0KKiIyXNUHQG34OIgDOhEsIjLMuAFgZneY2T4z21TQ1mpma81sa/jdErabmX3fzLaZ2UYzu6RgneVh/61mtrw8P2ekoecB9Rw9UaldiohMC2cyAvgxcOVpbTcA69x9MbAunAdYCiwOPyuA2yAIDOBG4DLgUuDGodAot9rM0AhAASAiUmjcAHD3R4Ce05qXAavC6VXAVQXtP/HAeqDZzOYBfwCsdfcedz8ArGVkqJRFrUYAIiJFTfQcwBx33wMQfs8O29uBnQX9usK20drLbugQ0AEFgIjIMJN9EtiKtPkY7SM3YLbCzDaY2Ybu7u6SC0omjGwqQY8OAYmIDDPRANgbHtoh/N4XtncB8wv6dQC7x2gfwd1Xununu3e2tbVNsLzh6jJJjQBERE4z0QBYAwxdybMcWF3Q/rnwaqDLgd7wENGDwEfMrCU8+fuRsK0iatJJenQZqIjIMKnxOpjZXcAHgVlm1kVwNc93gHvM7FpgB3B12P1+4KPANuAY8AUAd+8xs78Bngz7fdvdTz+xXDbZVIKeo/2V2p2IyLQwbgC4+6dHWbSkSF8Hrh9lO3cAd5xVdZOkNp3UVUAiIqep+juBIbgX4MBRHQISESkUiwCoSSc5PpDTA+FERArEIgB0M5iIyEjxCICMAkBE5HSxCICTdwPrZjARkZNiEQA6BCQiMlKsAkB3A4uInBKLAMimExgaAYiIFIpFACTMqMsm6T6iABARGRKLAACoy6TYf0SPgxARGRKLADh/3wP8YvBP+ZftH4abLoSN90RdkohI5MZ9FtB0d/6+B7ji5b8j7X1BQ+9O+MWXgumLPhVdYSIiEav6EcD7d9xKOt83vHHgOKz7djQFiYhMEVUfADP69xZf0NtV2UJERKaYqg+Aw9k5xRc0dVS2EBGRKabqA+DRBdcxkKgZ3piuhSXfiqYgEZEpoupPAm+ZvRSA97x6C00n9tFXN4+6pX+tE8AiEntVPwKAIARue9dqzu2/k7vff7/++IuIEJMAAKhJJ0gYdB/WzWAiIhCjADAzGrK6G1hEZEhJAWBmf25mz5vZJjO7y8xqzGyRmT1uZlvN7G4zy4R9s+H8tnD5wsn4AWejNpPUCEBEJDThADCzduBLQKe7XwgkgWuAfwBucvfFwAHg2nCVa4ED7n4ecFPYr6Jq00n2KQBERIDSDwGlgFozSwF1wB7gQ8C94fJVwFXh9LJwnnD5EjOzEvd/VuoyKQWAiEhowgHg7ruA7wI7CP7w9wJPAQfdfTDs1gW0h9PtwM5w3cGw/8yJ7n8iGrIpeo6cYDCXr+RuRUSmpFIOAbUQ/Kt+EXAOUA8sLdLVh1YZY1nhdleY2QYz29Dd3T3R8oqqzybJubNf7wUQESnpENCHgVfcvdvdB4D7gPcCzeEhIYAOYHc43QXMBwiXNwE9p2/U3Ve6e6e7d7a1tZVQ3kgNNUFZe3qPT+p2RUSmo1ICYAdwuZnVhcfylwAvAA8Dnwz7LAdWh9NrwnnC5Q+5+4gRQDnNyKYBeL23b5yeIiLVr5RzAI8TnMx9GvhduK2VwNeBr5rZNoJj/LeHq9wOzAzbvwrcUELdE9KQHRoBKABEREp6FpC73wjceFrzduDSIn37gKtL2V+patIJUglj7yEFgIhIbO4EhuBu4Bk1KY0ARESIWQAA1GdTOgksIkJsA0AjABGR2AVAQzbF3kN95PMVvQBJRGTKiWUADOScnmO6GUxE4i2WAQC6F0BEJH4BUKN7AUREIIYBMCMcAew+qCuBRCTeYhcAdZkkqYTRdeBY1KWIiEQqdgFgZjTVptnZoxGAiMRb7AIAgvMAO3o0AhCReItlADTWpNmpQ0AiEnOxDICm2jSH+wbpPT4QdSkiIpGJZQA0hpeC6kSwiMRZPAOgNngxjE4Ei0icxToANAIQkTiLZQDUpBJkUwm6DmgEICLxFcsAMLPgSiBdCioiMRbLAACYoXsBRCTmYhsATXVpdvQc03sBRCS2SgoAM2s2s3vN7EUz22xm7zGzVjNba2Zbw++WsK+Z2ffNbJuZbTSzSybnJ0xMS22G/sE8e/SCeBGJqVJHAN8D/s3dLwDeCWwGbgDWuftiYF04D7AUWBx+VgC3lbjvkjTXBVcCvdJ9NMoyREQiM+EAMLNG4APA7QDufsLdDwLLgFVht1XAVeH0MuAnHlgPNJvZvAlXXqKW+gwAr+w/ElUJIiKRKmUEcC7QDfzIzJ4xsx+aWT0wx933AITfs8P+7cDOgvW7wrZI1GeSZJIJXtYIQERiqpQASAGXALe5+8XAUU4d7inGirSNOANrZivMbIOZbeju7i6hvLGZGS31aV7ZrwAQkXgqJQC6gC53fzycv5cgEPYOHdoJv/cV9J9fsH4HsPv0jbr7SnfvdPfOtra2EsobX1NNmu3dOgQkIvE04QBw99eBnWZ2fti0BHgBWAMsD9uWA6vD6TXA58KrgS4HeocOFUWluT7DroPH6R/MRVmGiEgkUiWu/0XgTjPLANuBLxCEyj1mdi2wA7g67Hs/8FFgG3As7Buplro0eYedPcc4b/aMqMsREamokgLA3Z8FOossWlKkrwPXl7K/ydZcF1wJ9HL3UQWAiMRObO8EBmgNA2DbPp0HEJH4iXUAZFIJmmrTvPj64ahLERGpuFgHAEBrfYYX9xyKugwRkYqLfQDMrM+wff9RTgzmoy5FRKSiFAANGXJ5Z7seCSEiMRP7AJjVkAVgi84DiEjMxD4AWuoyJEwBICLxE/sASCaM1vqMAkBEYif2AQDBlUAv6EogEYkZBQCwzH7D/zm+Av+rZrjpQth4T9QliYiUXanPApr2zt/3AB/uvZlMoj9o6N0Jv/hSMH3Rp6IrTESkzGI/Anj/jlvJeP/wxoHjsO7b0RQkIlIhsQ+AGf17iy/o7apsISIiFRb7ADicnVN8QVNHZQsREamw2AfAowuuYyBRM7wxXQtLvhVNQSIiFRL7k8BbZi8F4PJXbqF5YB/99fOovfKvdQJYRKpe7EcAEITAynev5s39d/Ivl6zWH38RiQUFQCibSjKrIcNTrx2IuhQRkYpQABSY11TLU68dYDCnR0OLSPVTABQ4p7mWYydyekOYiMRCyQFgZkkze8bM/jWcX2Rmj5vZVjO728wyYXs2nN8WLl9Y6r4n2znNwdVAT77aE3ElIiLlNxkjgC8Dmwvm/wG4yd0XAweAa8P2a4ED7n4ecFPYb0qZUZOmqTbNhld1HkBEql9JAWBmHcDHgB+G8wZ8CLg37LIKuCqcXhbOEy5fEvafUuY21vDEKz24e9SliIiUVakjgJuBrwFDZ01nAgfdfTCc7wLaw+l2YCdAuLw37D+lnNNcQ/eRfl5941jUpYiIlNWEA8DMPg7sc/enCpuLdPUzWFa43RVmtsHMNnR3d0+0vAmb31oHwKNbK79vEZFKKmUE8D7gE2b2KvBTgkM/NwPNZjZ0h3EHsDuc7gLmA4TLm4ARZ1vdfaW7d7p7Z1tbWwnlTUxzbXAe4JGt+yu+bxGRSppwALj7N9y9w90XAtcAD7n7Z4CHgU+G3ZYDq8PpNeE84fKHfAoeaDcz5rfU8tjLbzCg+wFEpIqV4z6ArwNfNbNtBMf4bw/bbwdmhu1fBW4ow74nxYLWOo70D/LczoNRlyIiUjaT8jA4d/8V8KtwejtwaZE+fcDVk7G/cpvfWocZPLJ1P50LW6MuR0SkLHQncBE16SRzG2v41ZZ9UZciIlI2CoBRLJxZz8auXl7v7Yu6FBGRslAAjOLNbfUArN08yisjRUSmOQXAKFrrM7TWpXlw0+tRlyIiUhYKgFGYGYvaGnhs+xv0Hh+IuhwRkUmnABjDm9vqyeWdh17UYSARqT4KgDHMbayhsSbF6md2j99ZRGSaUQCMwcx4y5wZ/HrrfroP90ddjojIpFIAjOOCuTPIufOL5zQKEJHqogAYx8yGLHMas9z3dFfUpYiITCoFwBl4y5wZbNp9iC16V7CIVBEFwBl469xGUgnjf65/NepSREQmjQLgDNRmkiye3cB9T+/icJ/uCRCR6qAAOEMXdTRz7ESO+57eFXUpIiKTQgFwhuY21TC3sYZVv32VfH7KvcdGROSsKQDOwjvnN7F9/1E9IE5EqoIC4Cy8ZfYMmuvS/PND25iCb7MUETkrCoCzkEgY717Qwu929fJrvTReRKY5BcBZumDeDGbUpLj531/SKEBEpjUFwFlKJRL83sJWnt5xkAef17kAEZm+JhwAZjbfzB42s81m9ryZfTlsbzWztWa2NfxuCdvNzL5vZtvMbKOZXTJZP6LS3j6vkZn1Gf7+gc0M5PJRlyMiMiGljAAGgb9w97cClwPXm9nbgBuAde6+GFgXzgMsBRaHnxXAbSXsO1KJhPHe82by2hvHuHP9a1GXIyIyIRMOAHff4+5Ph9OHgc1AO7AMWBV2WwVcFU4vA37igfVAs5nNm3DlEVs0s54FrXV895cvsfeQXhwvItPPpJwDMLOFwMXA48Acd98DQUgAs8Nu7cDOgtW6wrZpycz4/fPb6BvI8Vdrno+6HBGRs1ZyAJhZA/Az4CvufmisrkXaRlxGY2YrzGyDmW3o7u4utbyyaq7LcOmiVh7Y9DoPPq+Xx4vI9FJSAJhZmuCP/53ufl/YvHfo0E74vS9s7wLmF6zeAYx4y4q7r3T3TnfvbGtrK6W8irhkQQttM7Lc8LON7NOhIBGZRkq5CsiA24HN7v5PBYvWAMvD6eXA6oL2z4VXA10O9A4dKprOkgnjyrfP5Uj/IF+5+1k9J0hEpo1SRgDvAz4LfMjMng0/HwW+A1xhZluBK8J5gPuB7cA24AfAdSXse0pprc/wgcVt/PblN7jl4W1RlyMickZSE13R3R+l+HF9gCVF+jtw/UT3N9W9/ZxGdh08zn9f+xKL5zRw5YXT9gInEYkJ3Qk8ScyMJRfMZl5TDV+5+1k27eqNuiQRkTEpACZRKpngY++YRyaV4HN3PMHL3UeiLklEZFQKgElWn01x1Tvb6R/I8Sc/WM+ON45FXZKISFEKgDJoqc9w1cXtHDo+yKdWPsa2fYejLklEZAQFQJnMasjyRxe3c7hvgP9422M89dqBqEsSERlGAVBGbTOyXP3u+SQM/uQH6/n5M3qhvIhMHQqAMmuqTfPJd3cwqyHLV+5+lhtXb+LEoB4hLSLRUwBUQF0mxR9d3M7FC5pZ9dhr/PGtv2HL6zovICLRUgBUSDJhfGBxGx97xzy27z/Kx//Hr7nl4W16oYyIREYBUGHnzW7gM5ctYOHMev7xwS1cefMj/GrLvvFXFBGZZAqACNRlUnz0HfP4w4vm0XP0BJ//0ZN8/o4n2Nh1MOrSRCRGJvwsICnduW0NLJhZx3M7e1n/yht84p9/w++f38YXlyzmkgUtUZcnIlVOARCxVCLBu9/UwoXtjTzX1cvjr/Tw8K2/5Z0dTXz2PQv5+EXzqEkng84b74F134beLmjqgCXfgos+Fe0PEJFpy4KHdE5NnZ2dvmHDhgmt+9Lew/y/jdPvdQMnBvO8sOcQm3b18sbREzTVpln2rnP4/IwnWPTYN7GB46c6p2vhD7+vEBCRYczsKXfvHK+fzgFMMZlUgnfNb+Yzly3gjy9uZ3Zjlv/9+A4yv/rb4X/8AQaOByMCEZEJ0CGgKcrMmN9ax/zWOvoHc7Q//kbRft7bxe6Dx2lvrq1whSIy3SkApoFsKsnh7Bwa+0e+eH5Xfibv/85DzGuq4bJFrfzeolYuWdDCebMbSCc1wBOR0SkApolHF1zHFS//Hen8qRfPDyRqeHT+f+U/WBu7Dh7n3zfv4+fP7gYgk0xw/twZXNjexNvPaeSt8xo5r62Bprp0VD9BRKYYBcA0sWX2UgDev+NWZvTv5XB2Do8uuI49s5fyLuBd85txdw4eH2DvoT66D/fTfaSfnz+zi7ue2HFyOy11ac6b3cCb24LPm2bW0d5SS0dzHY21KcxGe8uniFQbBcA0smX20pNBUIyZ0VKXoaUuwwVzgzZ353DfIPuP9HPg2AAHjp1gz8E+Xth9iKMncsPWr8skaW+upaOllnOaa5nXVMOshixtM4LPrIbgk0np0JJINah4AJjZlcD3gCTwQ3f/TqVriBMzo7E2TWPtyEM/fQM5eo8PcKhvgMN9gxw+Psjh/gFe2HOI9dt7OD6QK7JFaKxJ0TYjy8yGLE21aZpr0zSFn+a6YF/NdZmTbTNqUtRnUtSkExphiEwhFQ0AM0sCtwBXAF3Ak2a2xt1fqGQdEqhJJ6lJJ5nTWFN0+WAuz7ETufAzyLETOY6G38f6c+w6cJxX9h+lfyDH8YEcA7mx7ylJGNRmktRnUtRnU9RnkzRkg3Coy6ZoyCapy6SoTSfJphLUpJNk0wlqUsF3NjV8/uR3Qf900kgnEiQSChqZRiK6ybPSI4BLgW3uvh3AzH4KLAMUAFNQKpmgsTZRdPRQTC7v9A3k6B/MD/s+kcszMJhnIOfBdC7PicE8R/oGOXB0gMF8MD+Q8/A7T6m3JybNSCWDTzqRIJ1MBNPJRBASyUTR6VRBWyqRIJmA5NC3GYmEkUoE30kzkonwEy5LDi0Plw31P7U83J4F04mwzoSd+piBWbDMgETCCPIs+B7qc7IvRiJxqr9Z0G/oOxGOuoa2Y5xaPrSdoXYr2E6xWoZGcMF08J+1RnUl2ngP/OJLwX09AL07g3koewhUOgDagZ0F813AZRWuQcokmbDwX/albcfdyTsM5vPk8s5gzoPvvDOYz4+czzu5XDCfcyefd/Lu5PMMm88VtJ0YzHN8IHdyX0GfsP/JdSDvjo/yPTQ9de+lr6yhGBgKpaEGK2yjMDiGlg8F2akNDbWdWt+Gbf9k15PBdGpbBbs+uV87vZaCjZxe2+lBR5HlheueKttGto23HLjj4DeZnR/lJs8qC4Bi/1QY9v8fM1sBrABYsGDBhHd07qx6/ssHzp3w+iJnKl8QPCdDJp8n56eW5fLBJ18wXRhIubzjhSHDqSA8FTxBe9BWGEQAwwPr5LrDtuMj1g0C7PT2YBmc2v7QsmBPhdOE00GDe2Hb8L6EtQxfHsz4yeUjt0/YVrj9wr4Mq+X0bZ3amY9Y3yko/eR+C9dnxG899WMK/3AVe6JOsb7D/rMIv9t69o9cGYLDQWVW6QDoAuYXzHcAuws7uPtKYCUEzwKa6I5SyQQNuhFKRKa6mzqCwz6na+oo+64r/RfySWCxmS0yswxwDbCmwjWIiEwdS74VPNixULo2aC+zio4A3H3QzP4MeJDgMtA73P35StYgIjKlDB3nj8FVQLj7/cD9ld6viMiUddGnInmsuw6Si4jElAJARCSmFAAiIjGlABARiSkFgIhITCkARERiSgEgIhJTCgARkZgyL/YUoynCzLqB16KuYwJmAaM84anqxOm3gn5vNaum3/omd28br9OUDoDpysw2uHtn1HVUQpx+K+j3VrM4/dYhOgQkIhJTCgARkZhSAJTHyqgLqKA4/VbQ761mcfqtgM4BiIjElkYAIiIxpQCYJGY238weNrPNZva8mX056prKzcySZvaMmf1r1LWUm5k1m9m9ZvZi+N/xe6KuqZzM7M/D/x1vMrO7zKwm6pomk5ndYWb7zGxTQVurma01s63hd0uUNVaCAmDyDAJ/4e5vBS4Hrjezt0VcU7l9GdgcdREV8j3g39z9AuCdVPHvNrN24EtAp7tfSPD2vmuirWrS/Ri48rS2G4B17r4YWBfOVzUFwCRx9z3u/nQ4fZjgD0R7tFWVj5l1AB8Dfhh1LeVmZo3AB4DbAdz9hLsfjLaqsksBtWaWAuqA3RHXM6nc/RGg57TmZcCqcHoVcFVFi4qAAqAMzGwhcDHweLSVlNXNwNeAfNSFVMC5QDfwo/CQ1w/NrD7qosrF3XcB3wV2AHuAXnf/ZbRVVcQcd98DwT/ogNkR11N2CoBJZmYNwM+Ar7j7oajrKQcz+ziwz92firqWCkkBlwC3ufvFwFGq+PBAeOx7GbAIOAeoN7P/FG1VUg4KgElkZmmCP/53uvt9UddTRu8DPmFmrwI/BT5kZv8r2pLKqgvocvehEd29BIFQrT4MvOLu3e4+ANwHvDfimiphr5nNAwi/90VcT9kpACaJmRnBMeLN7v5PUddTTu7+DXfvcPeFBCcHH3L3qv0Xoru/Duw0s/PDpiXACxGWVG47gMvNrC783/USqvikd4E1wPJwejmwOsJaKiIVdQFV5H3AZ4HfmdmzYds33f3+CGuSyfNF4E4zywDbgS9EXE/ZuPvjZnYv8DTB1W3PUGV3yZrZXcAHgVlm1gXcCHwHuMfMriUIwaujq7AydCewiEhM6RCQiEhMKQBERGJKASAiElMKABGRmFIAiIjElAJARCSmFAAiIjGlABARian/D+hfAG5Faoo0AAAAAElFTkSuQmCC\n", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "data": { + "text/plain": [ + "Feerates:\t1.1531420689155165, 1.4085104512204296, 2.816548045571761, 11.10120050773006 \n", + "Times:\t\t874.010579873836, 479.615551452334, 60.00000000000001, 1.0000000000000004" + ] + }, + "execution_count": 108, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def build_estimator():\n", + " _feerates = [1.0]*10 + [1.1]*10 + [1.2]*10 + [1.5]*3000 + [2]*3000\\\n", + "+ [2.1]*3000 + [3]*10 + [4]*10 + [5]*10 + [6] + [7] + [10] + [100] + [200]*200\n", + " _total_weight = sum(np.array(_feerates)**ALPHA)\n", + " _network_mass_rate = bps * block_mass_limit\n", + " estimator = FeerateEstimator(total_weight=_total_weight, \n", + " inclusion_interval=avg_mass/_network_mass_rate)\n", + " \n", + " nr = _network_mass_rate\n", + " for i in range(len(_feerates)-1, -1, -1):\n", + " tw = sum(np.array(_feerates[:i])**ALPHA)\n", + " nr -= avg_mass\n", + " if nr <= 0:\n", + " print(\"net mass rate {}\", nr)\n", + " break\n", + " e = FeerateEstimator(total_weight=tw, \n", + " inclusion_interval=avg_mass/nr)\n", + " if e.feerate_to_time(1.0) < estimator.feerate_to_time(1.0):\n", + " # print(\"removing {}\".format(_feerates[i]))\n", + " estimator = e\n", + " else:\n", + " break\n", + " \n", + " return estimator\n", + "\n", + "estimator = build_estimator()\n", + "pred = estimator.calc_estimations()\n", + "x = np.linspace(1, pred.priority_bucket.feerate, 100000)\n", + "y = estimator.feerate_to_time(x)\n", + "plt.figure()\n", + "plt.plot(x, y)\n", + "plt.fill_between(x, estimator.inclusion_interval, y2=y, alpha=0.5)\n", + "plt.scatter(pred.feerates(), pred.times(), zorder=100)\n", + "plt.show()\n", + "pred" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python [conda env:gr]", + "language": "python", + "name": "conda-env-gr-py" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.5" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/mining/src/feerate/mod.rs b/mining/src/feerate/mod.rs new file mode 100644 index 0000000..5ef3579 --- /dev/null +++ b/mining/src/feerate/mod.rs @@ -0,0 +1,231 @@ +//! See the accompanying fee_estimation.ipynb Jupyter Notebook which details the reasoning +//! behind this fee estimator. + +use crate::block_template::selector::ALPHA; +use itertools::Itertools; +use std::fmt::Display; + +/// A type representing fee/mass of a transaction in `sompi/gram` units. +/// Given a feerate value recommendation, calculate the required fee by +/// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +pub type Feerate = f64; + +#[derive(Clone, Copy, Debug)] +pub struct FeerateBucket { + pub feerate: f64, + pub estimated_seconds: f64, +} + +impl Display for FeerateBucket { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "({:.4}, {:.4}s)", self.feerate, self.estimated_seconds) + } +} + +#[derive(Clone, Debug)] +pub struct FeerateEstimations { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: FeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl FeerateEstimations { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +impl Display for FeerateEstimations { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "(fee/mass, secs) priority: {}, ", self.priority_bucket)?; + write!(f, "normal: {}, ", self.normal_buckets.iter().format(", "))?; + write!(f, "low: {}", self.low_buckets.iter().format(", ")) + } +} + +pub struct FeerateEstimatorArgs { + pub network_blocks_per_second: u64, + pub maximum_mass_per_block: u64, +} + +impl FeerateEstimatorArgs { + pub fn new(network_blocks_per_second: u64, maximum_mass_per_block: u64) -> Self { + Self { network_blocks_per_second, maximum_mass_per_block } + } + + pub fn network_mass_per_second(&self) -> u64 { + self.network_blocks_per_second * self.maximum_mass_per_block + } +} + +#[derive(Debug, Clone)] +pub struct FeerateEstimator { + /// The total probability weight of current mempool ready transactions, i.e., `Σ_{tx in mempool}(tx.fee/tx.mass)^alpha`. + /// Note that some estimators might consider a reduced weight which excludes outliers. See [`Frontier::build_feerate_estimator`] + total_weight: f64, + + /// The amortized time **in seconds** between transactions, given the current transaction masses present in the mempool. Or in + /// other words, the inverse of the transaction inclusion rate. For instance, if the average transaction mass is 2500 grams, + /// the block mass limit is 500,000 and the network has 10 BPS, then this number would be 1/2000 seconds. + inclusion_interval: f64, +} + +impl FeerateEstimator { + pub fn new(total_weight: f64, inclusion_interval: f64) -> Self { + assert!(total_weight >= 0.0); + assert!((0f64..1f64).contains(&inclusion_interval)); + Self { total_weight, inclusion_interval } + } + + pub(crate) fn feerate_to_time(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / feerate.powi(ALPHA) + c1 + } + + fn time_to_feerate(&self, time: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + assert!(c1 < time, "{c1}, {time}"); + ((c1 * c2 / time) / (1f64 - c1 / time)).powf(1f64 / ALPHA as f64) + } + + /// The antiderivative function of [`feerate_to_time`] excluding the constant shift `+ c1` + #[inline] + fn feerate_to_time_antiderivative(&self, feerate: f64) -> f64 { + let (c1, c2) = (self.inclusion_interval, self.total_weight); + c1 * c2 / (-2f64 * feerate.powi(ALPHA - 1)) + } + + /// Returns the feerate value for which the integral area is `frac` of the total area between `lower` and `upper`. + fn quantile(&self, lower: f64, upper: f64, frac: f64) -> f64 { + assert!((0f64..=1f64).contains(&frac)); + assert!(0.0 < lower && lower <= upper, "{lower}, {upper}"); + let (c1, c2) = (self.inclusion_interval, self.total_weight); + if c1 == 0.0 || c2 == 0.0 { + // if c1 · c2 == 0.0, the integral area is empty, so we simply return `lower` + return lower; + } + let z1 = self.feerate_to_time_antiderivative(lower); + let z2 = self.feerate_to_time_antiderivative(upper); + // Get the total area corresponding to `frac` of the integral area between `lower` and `upper` + // which can be expressed as z1 + frac * (z2 - z1) + let z = frac * z2 + (1f64 - frac) * z1; + // Calc the x value (feerate) corresponding to said area + ((c1 * c2) / (-2f64 * z)).powf(1f64 / (ALPHA - 1) as f64) + } + + pub fn calc_estimations(&self, minimum_standard_feerate: f64) -> FeerateEstimations { + let min = minimum_standard_feerate; + // Choose `high` such that it provides sub-second waiting time + let high = self.time_to_feerate(1f64).max(min); + // Choose `low` feerate such that it provides sub-hour waiting time AND it covers (at least) the 0.25 quantile + let low = self.time_to_feerate(3600f64).max(self.quantile(min, high, 0.25)); + // Choose `normal` feerate such that it provides sub-minute waiting time AND it covers (at least) the 0.66 quantile between low and high. + let normal = self.time_to_feerate(60f64).max(self.quantile(low, high, 0.66)); + // Choose an additional point between normal and low + let mid = self.time_to_feerate(1800f64).max(self.quantile(min, high, 0.5)); + /* Intuition for the above: + 1. The quantile calculations make sure that we return interesting points on the `feerate_to_time` curve. + 2. They also ensure that the times don't diminish too high if small increments to feerate would suffice + to cover large fractions of the integral area (reflecting the position within the waiting-time distribution) + */ + FeerateEstimations { + priority_bucket: FeerateBucket { feerate: high, estimated_seconds: self.feerate_to_time(high) }, + normal_buckets: vec![ + FeerateBucket { feerate: normal, estimated_seconds: self.feerate_to_time(normal) }, + FeerateBucket { feerate: mid, estimated_seconds: self.feerate_to_time(mid) }, + ], + low_buckets: vec![FeerateBucket { feerate: low, estimated_seconds: self.feerate_to_time(low) }], + } + } +} + +#[derive(Clone, Debug)] +pub struct FeeEstimateVerbose { + pub estimations: FeerateEstimations, + + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} + +#[cfg(test)] +mod tests { + use super::*; + use itertools::Itertools; + + #[test] + fn test_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 1002283.659, inclusion_interval: 0.004f64 }; + let estimations = estimator.calc_estimations(1.0); + let buckets = estimations.ordered_buckets(); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + } + dbg!(estimations); + } + + #[test] + fn test_min_feerate_estimations() { + let estimator = FeerateEstimator { total_weight: 0.00659, inclusion_interval: 0.004f64 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + println!("{estimations}"); + let buckets = estimations.ordered_buckets(); + assert!(buckets.last().unwrap().feerate >= minimum_feerate); + for (i, j) in buckets.into_iter().tuple_windows() { + assert!(i.feerate >= j.feerate); + assert!(i.estimated_seconds <= j.estimated_seconds); + } + } + + #[test] + fn test_zero_values() { + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.0, inclusion_interval: 0.1 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(estimator.inclusion_interval, bucket.estimated_seconds); + } + + let estimator = FeerateEstimator { total_weight: 0.1, inclusion_interval: 0.0 }; + let minimum_feerate = 0.755; + let estimations = estimator.calc_estimations(minimum_feerate); + let buckets = estimations.ordered_buckets(); + for bucket in buckets { + assert_eq!(minimum_feerate, bucket.feerate); + assert_eq!(0.0, bucket.estimated_seconds); + } + } +} diff --git a/mining/src/lib.rs b/mining/src/lib.rs index 2986577..745fb63 100644 --- a/mining/src/lib.rs +++ b/mining/src/lib.rs @@ -8,12 +8,17 @@ use mempool::tx::Priority; mod block_template; pub(crate) mod cache; pub mod errors; +pub mod feerate; pub mod manager; mod manager_tests; pub mod mempool; pub mod model; pub mod monitor; +// Exposed for benchmarks +pub use block_template::{policy::Policy, selector::RebalancingWeightedTransactionSelector}; +pub use mempool::model::frontier::{feerate_key::FeerateTransactionKey, search_tree::SearchTree, Frontier}; + #[cfg(test)] pub mod testutils; diff --git a/mining/src/manager.rs b/mining/src/manager.rs index 64de72c..c4f398a 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -2,6 +2,7 @@ use crate::{ block_template::{builder::BlockTemplateBuilder, errors::BuilderError}, cache::BlockTemplateCache, errors::MiningManagerResult, + feerate::{FeeEstimateVerbose, FeerateEstimations, FeerateEstimatorArgs}, mempool::{ config::Config, model::tx::{MempoolTransaction, TransactionPostValidation, TransactionPreValidation, TxRemovalReason}, @@ -12,7 +13,6 @@ use crate::{ Mempool, }, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, topological_sort::IntoIterTopologically, tx_insert::TransactionInsertion, @@ -27,7 +27,7 @@ use spectre_consensus_core::{ args::{TransactionValidationArgs, TransactionValidationBatchArgs}, ConsensusApi, }, - block::{BlockTemplate, TemplateBuildMode}, + block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, errors::{block::RuleError as BlockRuleError, tx::TxRuleError}, tx::{MutableTransaction, Transaction, TransactionId, TransactionOutput}, @@ -107,14 +107,14 @@ impl MiningManager { loop { attempts += 1; - let transactions = self.block_candidate_transactions(); - let block_template_builder = BlockTemplateBuilder::new(self.config.maximum_mass_per_block); + let selector = self.build_selector(); + let block_template_builder = BlockTemplateBuilder::new(); let build_mode = if attempts < self.config.maximum_build_block_template_attempts { TemplateBuildMode::Standard } else { TemplateBuildMode::Infallible }; - match block_template_builder.build_block_template(consensus, miner_data, transactions, build_mode) { + match block_template_builder.build_block_template(consensus, miner_data, selector, build_mode) { Ok(block_template) => { let block_template = cache_lock.set_immutable_cached_template(block_template); match attempts { @@ -197,8 +197,37 @@ impl MiningManager { } } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - self.mempool.read().block_candidate_transactions() + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.mempool.read().build_selector() + } + + /// Returns realtime feerate estimations based on internal mempool state + pub(crate) fn get_realtime_feerate_estimations(&self) -> FeerateEstimations { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let estimator = self.mempool.read().build_feerate_estimator(args); + estimator.calc_estimations(self.config.minimum_feerate()) + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub(crate) fn get_realtime_feerate_estimations_verbose(&self) -> FeeEstimateVerbose { + let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); + let network_mass_per_second = args.network_mass_per_second(); + let mempool_read = self.mempool.read(); + let estimator = mempool_read.build_feerate_estimator(args); + let ready_transactions_count = mempool_read.ready_transaction_count(); + let ready_transaction_total_mass = mempool_read.ready_transaction_total_mass(); + drop(mempool_read); + FeeEstimateVerbose { + estimations: estimator.calc_estimations(self.config.minimum_feerate()), + network_mass_per_second, + mempool_ready_transactions_count: ready_transactions_count as u64, + mempool_ready_transactions_total_mass: ready_transaction_total_mass, + // TODO: Next PR + next_block_template_feerate_min: -1.0, + next_block_template_feerate_median: -1.0, + next_block_template_feerate_max: -1.0, + } } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. @@ -209,7 +238,7 @@ impl MiningManager { #[cfg(test)] pub(crate) fn block_template_builder(&self) -> BlockTemplateBuilder { - BlockTemplateBuilder::new(self.config.maximum_mass_per_block) + BlockTemplateBuilder::new() } /// validate_and_insert_transaction validates the given transaction, and @@ -799,6 +828,16 @@ impl MiningManagerProxy { consensus.clone().spawn_blocking(move |c| self.inner.get_block_template(c, &miner_data)).await } + /// Returns realtime feerate estimations based on internal mempool state + pub async fn get_realtime_feerate_estimations(self) -> FeerateEstimations { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations()).await.unwrap() + } + + /// Returns realtime feerate estimations based on internal mempool state with additional verbose data + pub async fn get_realtime_feerate_estimations_verbose(self) -> FeeEstimateVerbose { + spawn_blocking(move || self.inner.get_realtime_feerate_estimations_verbose()).await.unwrap() + } + /// Validates a transaction and adds it to the set of known transactions that have not yet been /// added to any block. /// diff --git a/mining/src/manager_tests.rs b/mining/src/manager_tests.rs index 272f6fc..24c12de 100644 --- a/mining/src/manager_tests.rs +++ b/mining/src/manager_tests.rs @@ -7,9 +7,10 @@ mod tests { mempool::{ config::{Config, DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE}, errors::RuleError, + model::frontier::selectors::TakeAllSelector, tx::{Orphan, Priority, RbfPolicy}, }, - model::{candidate_tx::CandidateTransaction, tx_query::TransactionQuery}, + model::tx_query::TransactionQuery, testutils::consensus_mock::ConsensusMock, MiningCounters, }; @@ -1095,7 +1096,7 @@ mod tests { // Collect all parent transactions for the next block template. // They are ready since they have no parents in the mempool. - let transactions = mining_manager.block_candidate_transactions(); + let transactions = mining_manager.build_selector().select_transactions(); assert_eq!( TX_PAIRS_COUNT, transactions.len(), @@ -1103,7 +1104,7 @@ mod tests { ); parent_txs.iter().for_each(|x| { assert!( - transactions.iter().any(|tx| tx.tx.id() == x.id()), + transactions.iter().any(|tx| tx.id() == x.id()), "the parent transaction {} should be candidate for the next block template", x.id() ); @@ -1119,8 +1120,9 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec, ) { + let transactions = transactions.into_iter().map(Arc::new).collect::>(); for _ in 0..4 { // Run a few times to get more randomness compare_modified_template_to_built( @@ -1187,7 +1189,7 @@ mod tests { consensus: &dyn ConsensusApi, address_prefix: Prefix, mining_manager: &MiningManager, - transactions: Vec, + transactions: Vec>, first_op: OpType, second_op: OpType, ) { @@ -1196,7 +1198,12 @@ mod tests { // Build a fresh template for coinbase2 as a reference let builder = mining_manager.block_template_builder(); - let result = builder.build_block_template(consensus, &miner_data_2, transactions, TemplateBuildMode::Standard); + let result = builder.build_block_template( + consensus, + &miner_data_2, + Box::new(TakeAllSelector::new(transactions)), + TemplateBuildMode::Standard, + ); assert!(result.is_ok(), "build block template failed for miner data 2"); let expected_template = result.unwrap(); diff --git a/mining/src/mempool/config.rs b/mining/src/mempool/config.rs index a8ac1a1..d17afb8 100644 --- a/mining/src/mempool/config.rs +++ b/mining/src/mempool/config.rs @@ -1,7 +1,6 @@ use spectre_consensus_core::constants::TX_VERSION; -pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u64 = 1_000_000; -pub(crate) const DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT: u64 = 50_000; +pub(crate) const DEFAULT_MAXIMUM_TRANSACTION_COUNT: u32 = 1_000_000; pub(crate) const DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS: u64 = 5; pub(crate) const DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS: u64 = 60; @@ -29,8 +28,7 @@ pub(crate) const DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION: u16 = TX_VERSION; #[derive(Clone, Debug)] pub struct Config { - pub maximum_transaction_count: u64, - pub maximum_ready_transaction_count: u64, + pub maximum_transaction_count: u32, pub maximum_build_block_template_attempts: u64, pub transaction_expire_interval_daa_score: u64, pub transaction_expire_scan_interval_daa_score: u64, @@ -47,13 +45,13 @@ pub struct Config { pub minimum_relay_transaction_fee: u64, pub minimum_standard_transaction_version: u16, pub maximum_standard_transaction_version: u16, + pub network_blocks_per_second: u64, } impl Config { #[allow(clippy::too_many_arguments)] pub fn new( - maximum_transaction_count: u64, - maximum_ready_transaction_count: u64, + maximum_transaction_count: u32, maximum_build_block_template_attempts: u64, transaction_expire_interval_daa_score: u64, transaction_expire_scan_interval_daa_score: u64, @@ -70,10 +68,10 @@ impl Config { minimum_relay_transaction_fee: u64, minimum_standard_transaction_version: u16, maximum_standard_transaction_version: u16, + network_blocks_per_second: u64, ) -> Self { Self { maximum_transaction_count, - maximum_ready_transaction_count, maximum_build_block_template_attempts, transaction_expire_interval_daa_score, transaction_expire_scan_interval_daa_score, @@ -90,6 +88,7 @@ impl Config { minimum_relay_transaction_fee, minimum_standard_transaction_version, maximum_standard_transaction_version, + network_blocks_per_second, } } @@ -98,7 +97,6 @@ impl Config { pub const fn build_default(target_milliseconds_per_block: u64, relay_non_std_transactions: bool, max_block_mass: u64) -> Self { Self { maximum_transaction_count: DEFAULT_MAXIMUM_TRANSACTION_COUNT, - maximum_ready_transaction_count: DEFAULT_MAXIMUM_READY_TRANSACTION_COUNT, maximum_build_block_template_attempts: DEFAULT_MAXIMUM_BUILD_BLOCK_TEMPLATE_ATTEMPTS, transaction_expire_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_INTERVAL_SECONDS * 1000 / target_milliseconds_per_block, transaction_expire_scan_interval_daa_score: DEFAULT_TRANSACTION_EXPIRE_SCAN_INTERVAL_SECONDS * 1000 @@ -118,11 +116,18 @@ impl Config { minimum_relay_transaction_fee: DEFAULT_MINIMUM_RELAY_TRANSACTION_FEE, minimum_standard_transaction_version: DEFAULT_MINIMUM_STANDARD_TRANSACTION_VERSION, maximum_standard_transaction_version: DEFAULT_MAXIMUM_STANDARD_TRANSACTION_VERSION, + network_blocks_per_second: 1000 / target_milliseconds_per_block, } } pub fn apply_ram_scale(mut self, ram_scale: f64) -> Self { - self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u64; // Allow only scaling down + self.maximum_transaction_count = (self.maximum_transaction_count as f64 * ram_scale.min(1.0)) as u32; // Allow only scaling down self } + + /// Returns the minimum standard fee/mass ratio currently required by the mempool + pub(crate) fn minimum_feerate(&self) -> f64 { + // The parameter minimum_relay_transaction_fee is in sompi/kg units so divide by 1000 to get sompi/gram + self.minimum_relay_transaction_fee as f64 / 1000.0 + } } diff --git a/mining/src/mempool/mod.rs b/mining/src/mempool/mod.rs index 8c6afa2..5668942 100644 --- a/mining/src/mempool/mod.rs +++ b/mining/src/mempool/mod.rs @@ -1,6 +1,6 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, model::{ - candidate_tx::CandidateTransaction, owner_txs::{GroupedOwnerTransactions, ScriptPublicKeySet}, tx_query::TransactionQuery, }, @@ -12,7 +12,10 @@ use self::{ model::{accepted_transactions::AcceptedTransactions, orphan_pool::OrphanPool, pool::Pool, transactions_pool::TransactionsPool}, tx::Priority, }; -use spectre_consensus_core::tx::{MutableTransaction, TransactionId}; +use spectre_consensus_core::{ + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId}, +}; use spectre_core::time::Stopwatch; use std::sync::Arc; @@ -112,9 +115,23 @@ impl Mempool { count } - pub(crate) fn block_candidate_transactions(&self) -> Vec { - let _sw = Stopwatch::<10>::with_threshold("block_candidate_transactions op"); - self.transaction_pool.all_ready_transactions() + pub(crate) fn ready_transaction_count(&self) -> usize { + self.transaction_pool.ready_transaction_count() + } + + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.transaction_pool.ready_transaction_total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + let _sw = Stopwatch::<10>::with_threshold("build_selector op"); + self.transaction_pool.build_selector() + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.transaction_pool.build_feerate_estimator(args) } pub(crate) fn all_transaction_ids_with_priority(&self, priority: Priority) -> Vec { diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/feerate_key.rs new file mode 100644 index 0000000..c0e9629 --- /dev/null +++ b/mining/src/mempool/model/feerate_key.rs @@ -0,0 +1,79 @@ +use crate::block_template::selector::ALPHA; + +use super::tx::MempoolTransaction; +use spectre_consensus_core::tx::Transaction; +use std::sync::Arc; + +#[derive(Clone, Debug)] +pub struct FeerateTransactionKey { + pub fee: u64, + pub mass: u64, + weight: f64, + pub tx: Arc, +} + +impl Eq for FeerateTransactionKey {} + +impl PartialEq for FeerateTransactionKey { + fn eq(&self, other: &Self) -> bool { + self.tx.id() == other.tx.id() + } +} + +impl FeerateTransactionKey { + pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } + } + + pub fn feerate(&self) -> f64 { + self.fee as f64 / self.mass as f64 + } + + pub fn weight(&self) -> f64 { + self.weight + } +} + +impl std::hash::Hash for FeerateTransactionKey { + fn hash(&self, state: &mut H) { + // Transaction id is a sufficient identifier for this key + self.tx.id().hash(state); + } +} + +impl PartialOrd for FeerateTransactionKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for FeerateTransactionKey { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Our first priority is the feerate + match self.feerate().total_cmp(&other.feerate()) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // If feerates are equal, prefer the higher fee in absolute value + match self.fee.cmp(&other.fee) { + core::cmp::Ordering::Equal => {} + ord => return ord, + } + + // + // At this point we don't compare the mass fields since if both feerate + // and fee are equal, mass must be equal as well + // + + // Finally, we compare transaction ids in order to allow multiple transactions with + // the same fee and mass to exist within the same sorted container + self.tx.id().cmp(&other.tx.id()) + } +} + +impl From<&MempoolTransaction> for FeerateTransactionKey { + fn from(tx: &MempoolTransaction) -> Self { + Self::new(tx.mtx.calculated_fee.unwrap(), tx.mtx.tx.mass(), tx.mtx.tx.clone()) + } +} diff --git a/mining/src/mempool/model/frontier.rs b/mining/src/mempool/model/frontier.rs new file mode 100644 index 0000000..9073d93 --- /dev/null +++ b/mining/src/mempool/model/frontier.rs @@ -0,0 +1,454 @@ +use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, + model::candidate_tx::CandidateTransaction, + Policy, RebalancingWeightedTransactionSelector, +}; + +use feerate_key::FeerateTransactionKey; +use rand::{distributions::Uniform, prelude::Distribution, Rng}; +use search_tree::SearchTree; +use selectors::{SequenceSelector, SequenceSelectorInput, TakeAllSelector}; +use spectre_consensus_core::block::TemplateTransactionSelector; +use spectre_core::trace; +use std::collections::HashSet; + +pub(crate) mod feerate_key; +pub(crate) mod search_tree; +pub(crate) mod selectors; + +/// If the frontier contains less than 4x the block mass limit, we consider +/// inplace sampling to be less efficient (due to collisions) and thus use +/// the rebalancing selector +const COLLISION_FACTOR: u64 = 4; + +/// Multiplication factor for in-place sampling. We sample 20% more than the +/// hard limit in order to allow the SequenceSelector to compensate for consensus rejections. +const MASS_LIMIT_FACTOR: f64 = 1.2; + +/// A rough estimation for the average transaction mass. The usage is a non-important edge case +/// hence we just throw this here (as oppose to performing an accurate estimation) +const TYPICAL_TX_MASS: f64 = 2000.0; + +/// Management of the transaction pool frontier, that is, the set of transactions in +/// the transaction pool which have no mempool ancestors and are essentially ready +/// to enter the next block template. +#[derive(Default)] +pub struct Frontier { + /// Frontier transactions sorted by feerate order and searchable for weight sampling + search_tree: SearchTree, + + /// Total masses: Σ_{tx in frontier} tx.mass + total_mass: u64, +} + +impl Frontier { + pub fn total_weight(&self) -> f64 { + self.search_tree.total_weight() + } + + pub fn total_mass(&self) -> u64 { + self.total_mass + } + + pub fn len(&self) -> usize { + self.search_tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn insert(&mut self, key: FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.insert(key) { + self.total_mass += mass; + true + } else { + false + } + } + + pub fn remove(&mut self, key: &FeerateTransactionKey) -> bool { + let mass = key.mass; + if self.search_tree.remove(key) { + self.total_mass -= mass; + true + } else { + false + } + } + + /// Samples the frontier in-place based on the provided policy and returns a SequenceSelector. + /// + /// This sampling algorithm should be used when frontier total mass is high enough compared to + /// policy mass limit so that the probability of sampling collisions remains low. + /// + /// Convergence analysis: + /// 1. Based on the above we can safely assume that `k << n`, where `n` is the total number of frontier items + /// and `k` is the number of actual samples (since `desired_mass << total_mass` and mass per item is bounded) + /// 2. Indeed, if the weight distribution is not too spread (i.e., `max(weights) = O(min(weights))`), `k << n` means + /// that the probability of collisions is low enough and the sampling process will converge in `O(k log(n))` w.h.p. + /// 3. It remains to deal with the case where the weight distribution is highly biased. The process implemented below + /// keeps track of the top-weight element. If the distribution is highly biased, this element will be sampled with + /// sufficient probability (in constant time). Following each sampling collision we search for a consecutive range of + /// top elements which were already sampled and narrow the sampling space to exclude them all. We do this by computing + /// the prefix weight up to the top most item which wasn't sampled yet (inclusive) and then continue the sampling process + /// over the narrowed space. This process is repeated until acquiring the desired mass. + /// 4. Numerical stability. Naively, one would simply subtract `total_weight -= top.weight` in order to narrow the sampling + /// space. However, if `top.weight` is much larger than the remaining weight, the above f64 subtraction will yield a number + /// close or equal to zero. We fix this by implementing a `log(n)` prefix weight operation. + /// 5. Q. Why not just use u64 weights? + /// A. The current weight calculation is `feerate^alpha` with `alpha=3`. Using u64 would mean that the feerate space + /// is limited to a range of size `(2^64)^(1/3) = ~2^21 = ~2M`. Already with current usages, the feerate can vary + /// from `~1/50` (2000 sompi for a transaction with 100K storage mass), to `5M` (100 SPR fee for a transaction with + /// 2000 mass = 100·100_000_000/2000), resulting in a range of size 250M (`5M/(1/50)`). + /// By using floating point arithmetics we gain the adjustment of the probability space to the accuracy level required for + /// current samples. And if the space is highly biased, the repeated elimination of top items and the prefix weight computation + /// will readjust it. + pub fn sample_inplace(&self, rng: &mut R, policy: &Policy, _collisions: &mut u64) -> SequenceSelectorInput + where + R: Rng + ?Sized, + { + debug_assert!(!self.search_tree.is_empty(), "expected to be called only if not empty"); + + // Sample 20% more than the hard limit in order to allow the SequenceSelector to + // compensate for consensus rejections. + // Note: this is a soft limit which is why the loop below might pass it if the + // next sampled transaction happens to cross the bound + let desired_mass = (policy.max_block_mass as f64 * MASS_LIMIT_FACTOR) as u64; + + let mut distr = Uniform::new(0f64, self.total_weight()); + let mut down_iter = self.search_tree.descending_iter(); + let mut top = down_iter.next().unwrap(); + let mut cache = HashSet::new(); + let mut sequence = SequenceSelectorInput::default(); + let mut total_selected_mass: u64 = 0; + let mut collisions = 0; + + // The sampling process is converging so the cache will eventually hold all entries, which guarantees loop exit + 'outer: while cache.len() < self.search_tree.len() && total_selected_mass <= desired_mass { + let query = distr.sample(rng); + let item = { + let mut item = self.search_tree.search(query); + while !cache.insert(item.tx.id()) { + collisions += 1; + // Try to narrow the sampling space in order to reduce further sampling collisions + if cache.contains(&top.tx.id()) { + loop { + match down_iter.next() { + Some(next) => top = next, + None => break 'outer, + } + // Loop until finding a top item which was not sampled yet + if !cache.contains(&top.tx.id()) { + break; + } + } + let remaining_weight = self.search_tree.prefix_weight(top); + distr = Uniform::new(0f64, remaining_weight); + } + let query = distr.sample(rng); + item = self.search_tree.search(query); + } + item + }; + sequence.push(item.tx.clone(), item.mass); + total_selected_mass += item.mass; // Max standard mass + Mempool capacity bound imply this will not overflow + } + trace!("[mempool frontier sample inplace] collisions: {collisions}, cache: {}", cache.len()); + *_collisions += collisions; + sequence + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier. + /// + /// The logic is divided into three cases: + /// 1. The frontier is small and can fit entirely into a block: perform no sampling and return + /// a TakeAllSelector + /// 2. The frontier has at least ~4x the capacity of a block: expected collision rate is low, perform + /// in-place k*log(n) sampling and return a SequenceSelector + /// 3. The frontier has 1-4x capacity of a block. In this case we expect a high collision rate while + /// the number of overall transactions is still low, so we take all of the transactions and use the + /// rebalancing weighted selector (performing the actual sampling out of the mempool lock) + /// + /// The above thresholds were selected based on benchmarks. Overall, this dynamic selection provides + /// full transaction selection in less than 150 µs even if the frontier has 1M entries (!!). See mining/benches + /// for more details. + pub fn build_selector(&self, policy: &Policy) -> Box { + if self.total_mass <= policy.max_block_mass { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } else if self.total_mass > policy.max_block_mass * COLLISION_FACTOR { + let mut rng = rand::thread_rng(); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, policy, &mut 0), policy.clone())) + } else { + Box::new(RebalancingWeightedTransactionSelector::new( + policy.clone(), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + } + + /// Exposed for benchmarking purposes + pub fn build_selector_sample_inplace(&self, _collisions: &mut u64) -> Box { + let mut rng = rand::thread_rng(); + let policy = Policy::new(500_000); + Box::new(SequenceSelector::new(self.sample_inplace(&mut rng, &policy, _collisions), policy)) + } + + /// Exposed for benchmarking purposes + pub fn build_selector_take_all(&self) -> Box { + Box::new(TakeAllSelector::new(self.search_tree.ascending_iter().map(|k| k.tx.clone()).collect())) + } + + /// Exposed for benchmarking purposes + pub fn build_rebalancing_selector(&self) -> Box { + Box::new(RebalancingWeightedTransactionSelector::new( + Policy::new(500_000), + self.search_tree.ascending_iter().cloned().map(CandidateTransaction::from_key).collect(), + )) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + let average_transaction_mass = match self.len() { + 0 => TYPICAL_TX_MASS, + n => self.total_mass() as f64 / n as f64, + }; + let bps = args.network_blocks_per_second as f64; + let mut mass_per_block = args.maximum_mass_per_block as f64; + let mut inclusion_interval = average_transaction_mass / (mass_per_block * bps); + let mut estimator = FeerateEstimator::new(self.total_weight(), inclusion_interval); + + // Search for better estimators by possibly removing extremely high outliers + let mut down_iter = self.search_tree.descending_iter().skip(1); + loop { + // Update values for the next iteration. In order to remove the outlier from the + // total weight, we must compensate by capturing a block slot. + mass_per_block -= average_transaction_mass; + if mass_per_block <= average_transaction_mass { + // Out of block slots, break + break; + } + + // Re-calc the inclusion interval based on the new block "capacity". + // Note that inclusion_interval < 1.0 as required by the estimator, since mass_per_block > average_transaction_mass (by condition above) and bps >= 1 + inclusion_interval = average_transaction_mass / (mass_per_block * bps); + + // Compute the weight up to, and including, current key (or use zero weight if next is none) + let next = down_iter.next(); + let prefix_weight = next.map(|key| self.search_tree.prefix_weight(key)).unwrap_or_default(); + let pending_estimator = FeerateEstimator::new(prefix_weight, inclusion_interval); + + // Test the pending estimator vs. the current one + if pending_estimator.feerate_to_time(1.0) < estimator.feerate_to_time(1.0) { + estimator = pending_estimator; + } else { + // The pending estimator is no better, break. Indicates that the reduction in + // network mass per second is more significant than the removed weight + break; + } + + if next.is_none() { + break; + } + } + estimator + } +} + +#[cfg(test)] +mod tests { + use super::*; + use feerate_key::tests::build_feerate_key; + use itertools::Itertools; + use rand::thread_rng; + use std::collections::HashMap; + + #[test] + pub fn test_highly_irregular_sampling() { + let mut rng = thread_rng(); + let cap = 1000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + if i == 0 { + // Add an extremely large fee in order to create extremely high variance + fee = 100_000_000 * 1_000_000; // 1M SPR + } + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let _sample = frontier.sample_inplace(&mut rng, &Policy::new(500_000), &mut 0); + } + + #[test] + pub fn test_mempool_sampling_small() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let mut frontier = Frontier::default(); + for item in map.values().cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_rebalancing_selector(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_sample_inplace(&mut 0); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector_take_all(); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + + let mut selector = frontier.build_selector(&Policy::new(500_000)); + selector.select_transactions().iter().map(|k| k.gas).sum::(); + } + + #[test] + pub fn test_total_mass_tracking() { + let mut rng = thread_rng(); + let cap = 10000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let fee: u64 = if i % (cap as u64 / 100) == 0 { 1000000 } else { rng.gen_range(1..10000) }; + let mass: u64 = rng.gen_range(1..100000); // Use distinct mass values to challenge the test + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + let len = cap / 2; + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let prev_total_mass = frontier.total_mass(); + // Assert the total mass + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Add a bunch of duplicates and make sure the total mass remains the same + let mut dup_items = frontier.search_tree.ascending_iter().take(len / 2).cloned().collect_vec(); + for dup in dup_items.iter().cloned() { + (!frontier.insert(dup)).then_some(()).unwrap(); + } + assert_eq!(prev_total_mass, frontier.total_mass()); + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + + // Remove a few elements from the map in order to randomize the iterator + dup_items.iter().take(10).for_each(|k| { + map.remove(&k.tx.id()); + }); + + // Add and remove random elements some of which will be duplicate insertions and some missing removals + for item in map.values().step_by(2) { + frontier.remove(item); + if let Some(item2) = dup_items.pop() { + frontier.insert(item2); + } + } + assert_eq!(frontier.total_mass(), frontier.search_tree.ascending_iter().map(|k| k.mass).sum::()); + } + + #[test] + fn test_feerate_estimator() { + let mut rng = thread_rng(); + let cap = 2000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mut fee: u64 = rng.gen_range(1..1000000); + let mass: u64 = 1650; + // 304 (~500,000/1650) extreme outliers is an edge case where the build estimator logic should be tested at + if i <= 303 { + // Add an extremely large fee in order to create extremely high variance + fee = i * 10_000_000 * 1_000_000; + } + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(1.0); + + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } + + #[test] + fn test_constant_feerate_estimator() { + const MIN_FEERATE: f64 = 1.0; + let cap = 20_000; + let mut map = HashMap::with_capacity(cap); + for i in 0..cap as u64 { + let mass: u64 = 1650; + let fee = (mass as f64 * MIN_FEERATE) as u64; + let key = build_feerate_key(fee, mass, i); + map.insert(key.tx.id(), key); + } + + for len in [0, 1, 10, 100, 200, 300, 500, 750, cap / 2, (cap * 2) / 3, (cap * 4) / 5, (cap * 5) / 6, cap] { + println!(); + println!("Testing a frontier with {} txs...", len.min(cap)); + let mut frontier = Frontier::default(); + for item in map.values().take(len).cloned() { + frontier.insert(item).then_some(()).unwrap(); + } + + let args = FeerateEstimatorArgs { network_blocks_per_second: 1, maximum_mass_per_block: 500_000 }; + // We are testing that the build function actually returns and is not looping indefinitely + let estimator = frontier.build_feerate_estimator(args); + let estimations = estimator.calc_estimations(MIN_FEERATE); + let buckets = estimations.ordered_buckets(); + // Test for the absence of NaN, infinite or zero values in buckets + for b in buckets.iter() { + assert!( + b.feerate.is_normal() && b.feerate >= 1.0, + "bucket feerate must be a finite number greater or equal to the minimum standard feerate" + ); + assert!( + b.estimated_seconds.is_normal() && b.estimated_seconds > 0.0, + "bucket estimated seconds must be a finite number greater than zero" + ); + } + dbg!(len, estimator); + dbg!(estimations); + } + } +} diff --git a/mining/src/mempool/model/frontier/feerate_weight.rs b/mining/src/mempool/model/frontier/feerate_weight.rs new file mode 100644 index 0000000..8421c7b --- /dev/null +++ b/mining/src/mempool/model/frontier/feerate_weight.rs @@ -0,0 +1,265 @@ +use super::feerate_key::FeerateTransactionKey; +use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; +use sweep_bptree::tree::{Argument, SearchArgument}; +use sweep_bptree::{BPlusTree, NodeStoreVec}; + +type FeerateKey = FeerateTransactionKey; + +#[derive(Clone, Copy, Debug, Default)] +struct FeerateWeight(f64); + +impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } +} + +impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.weight()).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } +} + +impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.weight(); + sum += w; + if query < sum { + return Some(i); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } + } +} + +struct PrefixWeightVisitor<'a> { + key: &'a FeerateKey, + accumulated_weight: f64, +} + +impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // The idx is the place where a matching element could be inserted while maintaining + // sorted order, go to left child + idx + } + Ok(idx) => { + // Exact match, go to right child. + idx + 1 + } + } + } +} + +impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + let idx = self.search_in_keys(keys); + // trace!("[visit_leaf] {}, {}", keys.len(), idx); + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + Some(self.accumulated_weight) + } +} + +type InnerTree = BPlusTree>; + +pub struct SearchTree { + tree: InnerTree, +} + +impl Default for SearchTree { + fn default() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } +} + +impl SearchTree { + pub fn new() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + pub fn insert(&mut self, key: FeerateKey) -> bool { + self.tree.insert(key, ()).is_none() + } + + pub fn remove(&mut self, key: &FeerateKey) -> bool { + self.tree.remove(key).is_some() + } + + pub fn search(&self, query: f64) -> &FeerateKey { + self.tree.get_by_argument(query).expect("clamped").0 + } + + pub fn total_weight(&self) -> f64 { + self.tree.root_argument().weight() + } + + pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { + self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() + } + + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + self.tree.iter().rev().map(|(key, ())| key) + } + + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { + self.tree.iter().map(|(key, ())| key) + } + + pub fn first(&self) -> Option<&FeerateKey> { + self.tree.first().map(|(k, ())| k) + } + + pub fn last(&self) -> Option<&FeerateKey> { + self.tree.last().map(|(k, ())| k) + } +} + +#[cfg(test)] +mod tests { + use super::super::feerate_key::tests::build_feerate_key; + use super::*; + use itertools::Itertools; + use std::collections::HashSet; + + #[test] + fn test_feerate_weight_queries() { + let mut btree = SearchTree::new(); + let mass = 2000; + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + s.insert(key.clone()); + btree.insert(key); + } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + btree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); + v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(btree.descending_iter()) { + assert_eq!(&expected, &item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly + let eps: f64 = 0.001; + let mut sum = 0.0; + for expected in v { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = btree.search(sample); + assert_eq!(&expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well + } + sum += weight; + } + + println!("{}, {}", sum, btree.total_weight()); + + // Test clamped search bounds + assert_eq!(btree.first(), Some(btree.search(f64::NEG_INFINITY))); + assert_eq!(btree.first(), Some(btree.search(-1.0))); + assert_eq!(btree.first(), Some(btree.search(-eps))); + assert_eq!(btree.first(), Some(btree.search(0.0))); + assert_eq!(btree.last(), Some(btree.search(sum))); + assert_eq!(btree.last(), Some(btree.search(sum + eps))); + assert_eq!(btree.last(), Some(btree.search(sum + 1.0))); + assert_eq!(btree.last(), Some(btree.search(1.0 / 0.0))); + assert_eq!(btree.last(), Some(btree.search(f64::INFINITY))); + let _ = btree.search(f64::NAN); + } + + #[test] + fn test_btree_rev_iter() { + let mut btree = SearchTree::new(); + let mass = 2000; + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + v.push(key.clone()); + btree.insert(key); + } + v.sort(); + + for (expected, item) in v.into_iter().rev().zip(btree.descending_iter()) { + assert_eq!(&expected, item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + } +} diff --git a/mining/src/mempool/model/frontier/search_tree.rs b/mining/src/mempool/model/frontier/search_tree.rs new file mode 100644 index 0000000..fc18b21 --- /dev/null +++ b/mining/src/mempool/model/frontier/search_tree.rs @@ -0,0 +1,335 @@ +use super::feerate_key::FeerateTransactionKey; +use std::iter::FusedIterator; +use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; +use sweep_bptree::tree::{Argument, SearchArgument}; +use sweep_bptree::{BPlusTree, NodeStoreVec}; + +type FeerateKey = FeerateTransactionKey; + +/// A struct for implementing "weight space" search using the SearchArgument customization. +/// The weight space is the range `[0, total_weight)` and each key has a "logical" interval allocation +/// within this space according to its tree position and weight. +/// +/// We implement the search efficiently by maintaining subtree weights which are updated with each +/// element insertion/removal. Given a search query `p ∈ [0, total_weight)` we then find the corresponding +/// element in log time by walking down from the root and adjusting the query according to subtree weights. +/// For instance if the query point is `123.56` and the top 3 subtrees have weights `120, 10.5 ,100` then we +/// recursively query the middle subtree with the point `123.56 - 120 = 3.56`. +/// +/// See SearchArgument implementation below for more details. +#[derive(Clone, Copy, Debug, Default)] +struct FeerateWeight(f64); + +impl FeerateWeight { + /// Returns the weight value + pub fn weight(&self) -> f64 { + self.0 + } +} + +impl Argument for FeerateWeight { + fn from_leaf(keys: &[FeerateKey]) -> Self { + Self(keys.iter().map(|k| k.weight()).sum()) + } + + fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { + Self(arguments.iter().map(|a| a.0).sum()) + } +} + +impl SearchArgument for FeerateWeight { + type Query = f64; + + fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { + let mut sum = 0.0; + for (i, k) in keys.iter().enumerate() { + let w = k.weight(); + sum += w; + if query < sum { + return Some(i); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last leaf if the query + // value is out of bounds + match keys.len() { + 0 => None, + n => Some(n - 1), + } + } + + fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { + // Search algorithm: Locate the next subtree to visit by iterating through `arguments` + // and subtracting the query until the correct range is found + for (i, a) in arguments.iter().enumerate() { + if query >= a.0 { + query -= a.0; + } else { + return Some((i, query)); + } + } + // In order to avoid sensitivity to floating number arithmetics, + // we logically "clamp" the search, returning the last subtree if the query + // value is out of bounds. Eventually this will lead to the return of the + // last leaf (see locate_in_leaf as well) + match arguments.len() { + 0 => None, + n => Some((n - 1, arguments[n - 1].0)), + } + } +} + +/// Visitor struct which accumulates the prefix weight up to a provided key (inclusive) in log time. +/// +/// The basic idea is to use the subtree weights stored in the tree for walking down from the root +/// to the leaf (corresponding to the searched key), and accumulating all weights proceeding the walk-down path +struct PrefixWeightVisitor<'a> { + /// The key to search up to + key: &'a FeerateKey, + /// This field accumulates the prefix weight during the visit process + accumulated_weight: f64, +} + +impl<'a> PrefixWeightVisitor<'a> { + pub fn new(key: &'a FeerateKey) -> Self { + Self { key, accumulated_weight: Default::default() } + } + + /// Returns the index of the first `key ∈ keys` such that `key > self.key`. If no such key + /// exists, the returned index will be the length of `keys`. + fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { + match keys.binary_search(self.key) { + Err(idx) => { + // self.key is not in keys, idx is the index of the following key + idx + } + Ok(idx) => { + // Exact match, return the following index + idx + 1 + } + } + } +} + +impl<'a> DescendVisit for PrefixWeightVisitor<'a> { + type Result = f64; + + fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { + let idx = self.search_in_keys(keys); + // Invariants: + // a. arguments.len() == keys.len() + 1 (n inner node keys are the separators between n+1 subtrees) + // b. idx <= keys.len() (hence idx < arguments.len()) + + // Based on the invariants, we first accumulate all the subtree weights up to idx + for argument in arguments.iter().take(idx) { + self.accumulated_weight += argument.weight(); + } + + // ..and then go down to the idx'th subtree + DescendVisitResult::GoDown(idx) + } + + fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { + // idx is the index of the key following self.key + let idx = self.search_in_keys(keys); + // Accumulate all key weights up to idx (which is inclusive if self.key ∈ tree) + for key in keys.iter().take(idx) { + self.accumulated_weight += key.weight(); + } + // ..and return the final result + Some(self.accumulated_weight) + } +} + +type InnerTree = BPlusTree>; + +/// A transaction search tree sorted by feerate order and searchable for probabilistic weighted sampling. +/// +/// All `log(n)` expressions below are in base 64 (based on constants chosen within the sweep_bptree crate). +/// +/// The tree has the following properties: +/// 1. Linear time ordered access (ascending / descending) +/// 2. Insertions/removals in log(n) time +/// 3. Search for a weight point `p ∈ [0, total_weight)` in log(n) time +/// 4. Compute the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) +/// according to key order, in log(n) time +/// 5. Access the total weight in O(1) time. The total weight has numerical stability since it +/// is recomputed from subtree weights for each item insertion/removal +/// +/// Computing the prefix weight is a crucial operation if the tree is used for random sampling and +/// the tree is highly imbalanced in terms of weight variance. See [`Frontier::sample_inplace`] for +/// more details. +pub struct SearchTree { + tree: InnerTree, +} + +impl Default for SearchTree { + fn default() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } +} + +impl SearchTree { + pub fn new() -> Self { + Self { tree: InnerTree::new(Default::default()) } + } + + pub fn len(&self) -> usize { + self.tree.len() + } + + pub fn is_empty(&self) -> bool { + self.len() == 0 + } + + /// Inserts a key into the tree in log(n) time. Returns `false` if the key was already in the tree. + pub fn insert(&mut self, key: FeerateKey) -> bool { + self.tree.insert(key, ()).is_none() + } + + /// Remove a key from the tree in log(n) time. Returns `false` if the key was not in the tree. + pub fn remove(&mut self, key: &FeerateKey) -> bool { + self.tree.remove(key).is_some() + } + + /// Search for a weight point `query ∈ [0, total_weight)` in log(n) time + pub fn search(&self, query: f64) -> &FeerateKey { + self.tree.get_by_argument(query).expect("clamped").0 + } + + /// Access the total weight in O(1) time + pub fn total_weight(&self) -> f64 { + self.tree.root_argument().weight() + } + + /// Computes the prefix weight of a key, i.e., the sum of weights up to that key (inclusive) + /// according to key order, in log(n) time + pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { + self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() + } + + /// Iterate the tree in descending key order (going down from the + /// highest key). Linear in the number of keys *actually* iterated. + pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().rev().map(|(key, ())| key) + } + + /// Iterate the tree in ascending key order (going up from the + /// lowest key). Linear in the number of keys *actually* iterated. + pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator + FusedIterator { + self.tree.iter().map(|(key, ())| key) + } + + /// The lowest key in the tree (by key order) + pub fn first(&self) -> Option<&FeerateKey> { + self.tree.first().map(|(k, ())| k) + } + + /// The highest key in the tree (by key order) + pub fn last(&self) -> Option<&FeerateKey> { + self.tree.last().map(|(k, ())| k) + } +} + +#[cfg(test)] +mod tests { + use super::super::feerate_key::tests::build_feerate_key; + use super::*; + use itertools::Itertools; + use std::collections::HashSet; + use std::ops::Sub; + + #[test] + fn test_feerate_weight_queries() { + let mut tree = SearchTree::new(); + let mass = 2000; + // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than + // 64^2 keys in order to trigger at least a few intermediate tree nodes + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + + #[allow(clippy::mutable_key_type)] + let mut s = HashSet::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + s.insert(key.clone()); + tree.insert(key); + } + + // Randomly remove 1/6 of the items + let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); + for r in remove { + s.remove(&r); + tree.remove(&r); + } + + // Collect to vec and sort for reference + let mut v = s.into_iter().collect_vec(); + v.sort(); + + // Test reverse iteration + for (expected, item) in v.iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, &item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + + // Sweep through the tree and verify that weight search queries are handled correctly + let eps: f64 = 0.001; + let mut sum = 0.0; + for expected in v.iter() { + let weight = expected.weight(); + let eps = eps.min(weight / 3.0); + let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; + for sample in samples { + let key = tree.search(sample); + assert_eq!(expected, key); + assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well + } + sum += weight; + } + + println!("{}, {}", sum, tree.total_weight()); + + // Test clamped search bounds + assert_eq!(tree.first(), Some(tree.search(f64::NEG_INFINITY))); + assert_eq!(tree.first(), Some(tree.search(-1.0))); + assert_eq!(tree.first(), Some(tree.search(-eps))); + assert_eq!(tree.first(), Some(tree.search(0.0))); + assert_eq!(tree.last(), Some(tree.search(sum))); + assert_eq!(tree.last(), Some(tree.search(sum + eps))); + assert_eq!(tree.last(), Some(tree.search(sum + 1.0))); + assert_eq!(tree.last(), Some(tree.search(1.0 / 0.0))); + assert_eq!(tree.last(), Some(tree.search(f64::INFINITY))); + let _ = tree.search(f64::NAN); + + // Assert prefix weights + let mut prefix = Vec::with_capacity(v.len()); + prefix.push(v[0].weight()); + for i in 1..v.len() { + prefix.push(prefix[i - 1] + v[i].weight()); + } + let eps = v.iter().map(|k| k.weight()).min_by(f64::total_cmp).unwrap() * 1e-4; + for (expected_prefix, key) in prefix.into_iter().zip(v) { + let prefix = tree.prefix_weight(&key); + assert!(expected_prefix.sub(prefix).abs() < eps); + } + } + + #[test] + fn test_tree_rev_iter() { + let mut tree = SearchTree::new(); + let mass = 2000; + let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); + let mut v = Vec::with_capacity(fees.len()); + for (i, fee) in fees.iter().copied().enumerate() { + let key = build_feerate_key(fee, mass, i as u64); + v.push(key.clone()); + tree.insert(key); + } + v.sort(); + + for (expected, item) in v.into_iter().rev().zip(tree.descending_iter()) { + assert_eq!(&expected, item); + assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well + } + } +} diff --git a/mining/src/mempool/model/frontier/selectors.rs b/mining/src/mempool/model/frontier/selectors.rs new file mode 100644 index 0000000..c537149 --- /dev/null +++ b/mining/src/mempool/model/frontier/selectors.rs @@ -0,0 +1,162 @@ +use crate::Policy; +use spectre_consensus_core::{ + block::TemplateTransactionSelector, + tx::{Transaction, TransactionId}, +}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, +}; + +pub struct SequenceSelectorTransaction { + pub tx: Arc, + pub mass: u64, +} + +impl SequenceSelectorTransaction { + pub fn new(tx: Arc, mass: u64) -> Self { + Self { tx, mass } + } +} + +type SequencePriorityIndex = u32; + +/// The input sequence for the [`SequenceSelector`] transaction selector +#[derive(Default)] +pub struct SequenceSelectorInput { + /// We use the btree map ordered by insertion order in order to follow + /// the initial sequence order while allowing for efficient removal of previous selections + inner: BTreeMap, +} + +impl FromIterator for SequenceSelectorInput { + fn from_iter>(iter: T) -> Self { + Self { inner: BTreeMap::from_iter(iter.into_iter().enumerate().map(|(i, v)| (i as SequencePriorityIndex, v))) } + } +} + +impl SequenceSelectorInput { + pub fn push(&mut self, tx: Arc, mass: u64) { + let idx = self.inner.len() as SequencePriorityIndex; + self.inner.insert(idx, SequenceSelectorTransaction::new(tx, mass)); + } + + pub fn iter(&self) -> impl Iterator { + self.inner.values() + } +} + +/// Helper struct for storing data related to previous selections +struct SequenceSelectorSelection { + tx_id: TransactionId, + mass: u64, + priority_index: SequencePriorityIndex, +} + +/// A selector which selects transactions in the order they are provided. The selector assumes +/// that the transactions were already selected via weighted sampling and simply tries them one +/// after the other until the block mass limit is reached. +pub struct SequenceSelector { + input_sequence: SequenceSelectorInput, + selected_vec: Vec, + /// Maps from selected tx ids to tx mass so that the total used mass can be subtracted on tx reject + selected_map: Option>, + total_selected_mass: u64, + overall_candidates: usize, + overall_rejections: usize, + policy: Policy, +} + +impl SequenceSelector { + pub fn new(input_sequence: SequenceSelectorInput, policy: Policy) -> Self { + Self { + overall_candidates: input_sequence.inner.len(), + selected_vec: Vec::with_capacity(input_sequence.inner.len()), + input_sequence, + selected_map: Default::default(), + total_selected_mass: Default::default(), + overall_rejections: Default::default(), + policy, + } + } + + #[inline] + fn reset_selection(&mut self) { + self.selected_vec.clear(); + self.selected_map = None; + } +} + +impl TemplateTransactionSelector for SequenceSelector { + fn select_transactions(&mut self) -> Vec { + // Remove selections from the previous round if any + for selection in self.selected_vec.drain(..) { + self.input_sequence.inner.remove(&selection.priority_index); + } + // Reset selection data structures + self.reset_selection(); + let mut transactions = Vec::with_capacity(self.input_sequence.inner.len()); + + // Iterate the input sequence in order + for (&priority_index, tx) in self.input_sequence.inner.iter() { + if self.total_selected_mass.saturating_add(tx.mass) > self.policy.max_block_mass { + // We assume the sequence is relatively small, hence we keep on searching + // for transactions with lower mass which might fit into the remaining gap + continue; + } + self.total_selected_mass += tx.mass; + self.selected_vec.push(SequenceSelectorSelection { tx_id: tx.tx.id(), mass: tx.mass, priority_index }); + transactions.push(tx.tx.as_ref().clone()) + } + transactions + } + + fn reject_selection(&mut self, tx_id: TransactionId) { + // Lazy-create the map only when there are actual rejections + let selected_map = self.selected_map.get_or_insert_with(|| self.selected_vec.iter().map(|tx| (tx.tx_id, tx.mass)).collect()); + let mass = selected_map.remove(&tx_id).expect("only previously selected txs can be rejected (and only once)"); + // Selections must be counted in total selected mass, so this subtraction cannot underflow + self.total_selected_mass -= mass; + self.overall_rejections += 1; + } + + fn is_successful(&self) -> bool { + const SUFFICIENT_MASS_THRESHOLD: f64 = 0.8; + const LOW_REJECTION_FRACTION: f64 = 0.2; + + // We consider the operation successful if either mass occupation is above 80% or rejection rate is below 20% + self.overall_rejections == 0 + || (self.total_selected_mass as f64) > self.policy.max_block_mass as f64 * SUFFICIENT_MASS_THRESHOLD + || (self.overall_rejections as f64) < self.overall_candidates as f64 * LOW_REJECTION_FRACTION + } +} + +/// A selector that selects all the transactions it holds and is always considered successful. +/// If all mempool transactions have combined mass which is <= block mass limit, this selector +/// should be called and provided with all the transactions. +pub struct TakeAllSelector { + txs: Vec>, +} + +impl TakeAllSelector { + pub fn new(txs: Vec>) -> Self { + Self { txs } + } +} + +impl TemplateTransactionSelector for TakeAllSelector { + fn select_transactions(&mut self) -> Vec { + // Drain on the first call so that subsequent calls return nothing + self.txs.drain(..).map(|tx| tx.as_ref().clone()).collect() + } + + fn reject_selection(&mut self, _tx_id: TransactionId) { + // No need to track rejections (for reduced mass), since there's nothing else to select + } + + fn is_successful(&self) -> bool { + // Considered successful because we provided all mempool transactions to this + // selector, so there's no point in retries + true + } +} diff --git a/mining/src/mempool/model/mod.rs b/mining/src/mempool/model/mod.rs index 88997e4..bfe6222 100644 --- a/mining/src/mempool/model/mod.rs +++ b/mining/src/mempool/model/mod.rs @@ -1,4 +1,5 @@ pub(crate) mod accepted_transactions; +pub(crate) mod frontier; pub(crate) mod map; pub(crate) mod orphan_pool; pub(crate) mod pool; diff --git a/mining/src/mempool/model/transactions_pool.rs b/mining/src/mempool/model/transactions_pool.rs index bf17928..422631c 100644 --- a/mining/src/mempool/model/transactions_pool.rs +++ b/mining/src/mempool/model/transactions_pool.rs @@ -1,4 +1,5 @@ use crate::{ + feerate::{FeerateEstimator, FeerateEstimatorArgs}, mempool::{ config::Config, errors::{RuleError, RuleResult}, @@ -10,18 +11,21 @@ use crate::{ }, tx::Priority, }, - model::{candidate_tx::CandidateTransaction, topological_index::TopologicalIndex}, + model::topological_index::TopologicalIndex, + Policy, }; use spectre_consensus_core::{ - tx::TransactionId, - tx::{MutableTransaction, TransactionOutpoint}, + block::TemplateTransactionSelector, + tx::{MutableTransaction, TransactionId, TransactionOutpoint}, }; use spectre_core::{time::unix_now, trace, warn}; use std::{ - collections::{hash_map::Keys, hash_set::Iter, HashSet}, + collections::{hash_map::Keys, hash_set::Iter}, sync::Arc, }; +use super::frontier::Frontier; + /// Pool of transactions to be included in a block template /// /// ### Rust rewrite notes @@ -54,7 +58,7 @@ pub(crate) struct TransactionsPool { /// Transactions dependencies formed by outputs present in pool - successor relations. chained_transactions: TransactionsEdges, /// Transactions with no parents in the mempool -- ready to be inserted into a block template - ready_transactions: HashSet, + ready_transactions: Frontier, last_expire_scan_daa_score: u64, /// last expire scan time in milliseconds @@ -105,7 +109,7 @@ impl TransactionsPool { let parents = self.get_parent_transaction_ids_in_pool(&transaction.mtx); self.parent_transactions.insert(id, parents.clone()); if parents.is_empty() { - self.ready_transactions.insert(id); + self.ready_transactions.insert((&transaction).into()); } for parent_id in parents { let entry = self.chained_transactions.entry(parent_id).or_default(); @@ -133,18 +137,20 @@ impl TransactionsPool { if let Some(parents) = self.parent_transactions.get_mut(chain) { parents.remove(transaction_id); if parents.is_empty() { - self.ready_transactions.insert(*chain); + let tx = self.all_transactions.get(chain).unwrap(); + self.ready_transactions.insert(tx.into()); } } } } self.parent_transactions.remove(transaction_id); self.chained_transactions.remove(transaction_id); - self.ready_transactions.remove(transaction_id); // Remove the transaction itself let removed_tx = self.all_transactions.remove(transaction_id).ok_or(RuleError::RejectMissingTransaction(*transaction_id))?; + self.ready_transactions.remove(&(&removed_tx).into()); + // TODO: consider using `self.parent_transactions.get(transaction_id)` // The tradeoff to consider is whether it might be possible that a parent tx exists in the pool // however its relation as parent is not registered. This can supposedly happen in rare cases where @@ -161,15 +167,18 @@ impl TransactionsPool { self.ready_transactions.len() } - /// all_ready_transactions returns all fully populated mempool transactions having no parents in the mempool. - /// These transactions are ready for being inserted in a block template. - pub(crate) fn all_ready_transactions(&self) -> Vec { - // The returned transactions are leaving the mempool so they are cloned - self.ready_transactions - .iter() - .take(self.config.maximum_ready_transaction_count as usize) - .map(|id| CandidateTransaction::from_mutable(&self.all_transactions.get(id).unwrap().mtx)) - .collect() + pub(crate) fn ready_transaction_total_mass(&self) -> u64 { + self.ready_transactions.total_mass() + } + + /// Dynamically builds a transaction selector based on the specific state of the ready transactions frontier + pub(crate) fn build_selector(&self) -> Box { + self.ready_transactions.build_selector(&Policy::new(self.config.maximum_mass_per_block)) + } + + /// Builds a feerate estimator based on internal state of the ready transactions frontier + pub(crate) fn build_feerate_estimator(&self, args: FeerateEstimatorArgs) -> FeerateEstimator { + self.ready_transactions.build_feerate_estimator(args) } /// Is the mempool transaction identified by `transaction_id` unchained, thus having no successor? @@ -229,8 +238,8 @@ impl TransactionsPool { // An error is returned if the mempool is filled with high priority and other unremovable transactions. let tx_count = self.len() + free_slots - transactions_to_remove.len(); - if tx_count as u64 > self.config.maximum_transaction_count { - let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count); + if tx_count as u64 > self.config.maximum_transaction_count as u64 { + let err = RuleError::RejectMempoolIsFull(tx_count - free_slots, self.config.maximum_transaction_count as u64); warn!("{}", err.to_string()); return Err(err); } diff --git a/mining/src/mempool/model/tx.rs b/mining/src/mempool/model/tx.rs index f596f78..303ab78 100644 --- a/mining/src/mempool/model/tx.rs +++ b/mining/src/mempool/model/tx.rs @@ -2,7 +2,6 @@ use crate::mempool::tx::{Priority, RbfPolicy}; use spectre_consensus_core::tx::{MutableTransaction, Transaction, TransactionId, TransactionOutpoint}; use spectre_mining_errors::mempool::RuleError; use std::{ - cmp::Ordering, fmt::{Display, Formatter}, sync::Arc, }; @@ -35,26 +34,6 @@ impl MempoolTransaction { } } -impl Ord for MempoolTransaction { - fn cmp(&self, other: &Self) -> Ordering { - self.fee_rate().total_cmp(&other.fee_rate()).then(self.id().cmp(&other.id())) - } -} - -impl Eq for MempoolTransaction {} - -impl PartialOrd for MempoolTransaction { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl PartialEq for MempoolTransaction { - fn eq(&self, other: &Self) -> bool { - self.fee_rate() == other.fee_rate() - } -} - impl RbfPolicy { #[cfg(test)] /// Returns an alternate policy accepting a transaction insertion in case the policy requires a replacement diff --git a/mining/src/model/candidate_tx.rs b/mining/src/model/candidate_tx.rs index 41730de..48ecdf4 100644 --- a/mining/src/model/candidate_tx.rs +++ b/mining/src/model/candidate_tx.rs @@ -1,10 +1,11 @@ -use spectre_consensus_core::tx::{MutableTransaction, Transaction}; +use crate::FeerateTransactionKey; +use spectre_consensus_core::tx::Transaction; use std::sync::Arc; /// Transaction with additional metadata needed in order to be a candidate /// in the transaction selection algorithm #[derive(Clone, Debug, PartialEq, Eq)] -pub(crate) struct CandidateTransaction { +pub struct CandidateTransaction { /// The actual transaction pub tx: Arc, /// Populated fee @@ -14,9 +15,7 @@ pub(crate) struct CandidateTransaction { } impl CandidateTransaction { - pub(crate) fn from_mutable(tx: &MutableTransaction) -> Self { - let mass = tx.tx.mass(); - assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); - Self { tx: tx.tx.clone(), calculated_fee: tx.calculated_fee.expect("fee is expected to be populated"), calculated_mass: mass } + pub fn from_key(key: FeerateTransactionKey) -> Self { + Self { tx: key.tx, calculated_fee: key.fee, calculated_mass: key.mass } } } diff --git a/mining/src/model/mod.rs b/mining/src/model/mod.rs index 0c6caee..1116b6e 100644 --- a/mining/src/model/mod.rs +++ b/mining/src/model/mod.rs @@ -1,7 +1,7 @@ use spectre_consensus_core::tx::TransactionId; use std::collections::HashSet; -pub(crate) mod candidate_tx; +pub mod candidate_tx; pub mod owner_txs; pub mod topological_index; pub mod topological_sort; diff --git a/mining/src/monitor.rs b/mining/src/monitor.rs index e034b79..ab05216 100644 --- a/mining/src/monitor.rs +++ b/mining/src/monitor.rs @@ -1,4 +1,5 @@ use super::MiningCounters; +use crate::manager::MiningManagerProxy; use spectre_core::{ debug, info, task::{ @@ -13,6 +14,8 @@ use std::{sync::Arc, time::Duration}; const MONITOR: &str = "mempool-monitor"; pub struct MiningMonitor { + mining_manager: MiningManagerProxy, + // Counters counters: Arc, @@ -24,11 +27,12 @@ pub struct MiningMonitor { impl MiningMonitor { pub fn new( + mining_manager: MiningManagerProxy, counters: Arc, tx_script_cache_counters: Arc, tick_service: Arc, ) -> MiningMonitor { - MiningMonitor { counters, tx_script_cache_counters, tick_service } + MiningMonitor { mining_manager, counters, tx_script_cache_counters, tick_service } } pub async fn worker(self: &Arc) { @@ -62,6 +66,8 @@ impl MiningMonitor { delta.low_priority_tx_counts, delta.tx_accepted_counts, ); + let feerate_estimations = self.mining_manager.clone().get_realtime_feerate_estimations().await; + debug!("Realtime feerate estimations: {}", feerate_estimations); } if tx_script_cache_snapshot != last_tx_script_cache_snapshot { debug!( diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index e8df3dc..8b26c8c 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -117,6 +117,10 @@ pub enum RpcApiOps { /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy SubmitTransactionReplacement, + + // Fee estimation related commands + GetFeeEstimate, + GetFeeEstimateExperimental, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index f4588bb..0e39073 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -315,6 +315,22 @@ pub trait RpcApi: Sync + Send + AnySync { request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + // Fee estimation API + + async fn get_fee_estimate(&self) -> RpcResult { + Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) + } + async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; + + async fn get_fee_estimate_experimental(&self, verbose: bool) -> RpcResult { + self.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await + } + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult; + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs new file mode 100644 index 0000000..f9de9de --- /dev/null +++ b/rpc/core/src/model/feerate_estimate.rs @@ -0,0 +1,55 @@ +use borsh::{BorshDeserialize, BorshSerialize}; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeerateBucket { + /// The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds + pub feerate: f64, + + /// The estimated inclusion time for a transaction with fee/mass = feerate + pub estimated_seconds: f64, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimate { + /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + /// + /// Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + /// Given a feerate value recommendation, calculate the required fee by + /// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + pub priority_bucket: RpcFeerateBucket, + + /// A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + /// provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + /// times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + /// between them, one can compose a complete feerate function on the client side. The API makes an effort + /// to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + pub normal_buckets: Vec, + + /// A vector of *low* priority feerate values. The first value of this vector is guaranteed to + /// exist and provide an estimation for sub-*hour* DAG inclusion. + pub low_buckets: Vec, +} + +impl RpcFeeEstimate { + pub fn ordered_buckets(&self) -> Vec { + std::iter::once(self.priority_bucket) + .chain(self.normal_buckets.iter().copied()) + .chain(self.low_buckets.iter().copied()) + .collect() + } +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcFeeEstimateVerboseExperimentalData { + pub mempool_ready_transactions_count: u64, + pub mempool_ready_transactions_total_mass: u64, + pub network_mass_per_second: u64, + + pub next_block_template_feerate_min: f64, + pub next_block_template_feerate_median: f64, + pub next_block_template_feerate_max: f64, +} diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index f1cdc66..0adef55 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -850,6 +850,35 @@ impl GetDaaScoreTimestampEstimateResponse { } } +// ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +// Fee rate estimations + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateRequest {} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateResponse { + pub estimate: RpcFeeEstimate, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalRequest { + pub verbose: bool, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetFeeEstimateExperimentalResponse { + /// The usual feerate estimate response + pub estimate: RpcFeeEstimate, + + /// Experimental verbose data + pub verbose: Option, +} + // ---------------------------------------------------------------------------- // Subscriptions & notifications // ---------------------------------------------------------------------------- diff --git a/rpc/core/src/model/mod.rs b/rpc/core/src/model/mod.rs index fd07a10..8950bd1 100644 --- a/rpc/core/src/model/mod.rs +++ b/rpc/core/src/model/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod blue_work; +pub mod feerate_estimate; pub mod hash; pub mod header; pub mod hex_cnv; @@ -15,6 +16,7 @@ pub mod tx; pub use address::*; pub use block::*; pub use blue_work::*; +pub use feerate_estimate::*; pub use hash::*; pub use header::*; pub use hex_cnv::*; diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index 1902e5c..d2d38a8 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -272,6 +272,8 @@ impl RpcApi for GrpcClient { route!(get_mempool_entries_by_addresses_call, GetMempoolEntriesByAddresses); route!(get_coin_supply_call, GetCoinSupply); route!(get_daa_score_timestamp_estimate_call, GetDaaScoreTimestampEstimate); + route!(get_fee_estimate_call, GetFeeEstimate); + route!(get_fee_estimate_experimental_call, GetFeeEstimateExperimental); // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index 21c9332..230c625 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -60,6 +60,8 @@ message SpectredRequest { GetSyncStatusRequestMessage getSyncStatusRequest = 1094; GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; + GetFeeEstimateRequestMessage getFeeEstimateRequest = 1106; + GetFeeEstimateExperimentalRequestMessage getFeeEstimateExperimentalRequest = 1108; } } @@ -120,6 +122,8 @@ message SpectredResponse { GetSyncStatusResponseMessage getSyncStatusResponse = 1095; GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; + GetFeeEstimateResponseMessage getFeeEstimateResponse = 1107; + GetFeeEstimateExperimentalResponseMessage getFeeEstimateExperimentalResponse = 1109; } } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 5c0f8b1..0570dde 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -866,3 +866,59 @@ message GetDaaScoreTimestampEstimateResponseMessage{ repeated uint64 timestamps = 1; RPCError error = 1000; } + +message RpcFeerateBucket { + // Fee/mass of a transaction in `sompi/gram` units + double feerate = 1; + double estimated_seconds = 2; +} + +// Data required for making fee estimates. +// +// Feerate values represent fee/mass of a transaction in `sompi/gram` units. +// Given a feerate value recommendation, calculate the required fee by +// taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` +message RpcFeeEstimate { + // Top-priority feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + RpcFeerateBucket priority_bucket = 1; + + // A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + // provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + // times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + // between them, one can compose a complete feerate function on the client side. The API makes an effort + // to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + repeated RpcFeerateBucket normal_buckets = 2; + + // A vector of *low* priority feerate values. The first value of this vector is guaranteed to + // exist and provide an estimation for sub-*hour* DAG inclusion. + repeated RpcFeerateBucket low_buckets = 3; +} + +message RpcFeeEstimateVerboseExperimentalData { + uint64 mempool_ready_transactions_count = 1; + uint64 mempool_ready_transactions_total_mass = 2; + uint64 network_mass_per_second = 3; + + double next_block_template_feerate_min = 11; + double next_block_template_feerate_median = 12; + double next_block_template_feerate_max = 13; +} + +message GetFeeEstimateRequestMessage { +} + +message GetFeeEstimateResponseMessage { + RpcFeeEstimate estimate = 1; + RPCError error = 1000; +} + +message GetFeeEstimateExperimentalRequestMessage { + bool verbose = 1; +} + +message GetFeeEstimateExperimentalResponseMessage { + RpcFeeEstimate estimate = 1; + RpcFeeEstimateVerboseExperimentalData verbose = 2; + + RPCError error = 1000; +} diff --git a/rpc/grpc/core/src/convert/feerate_estimate.rs b/rpc/grpc/core/src/convert/feerate_estimate.rs new file mode 100644 index 0000000..e85021d --- /dev/null +++ b/rpc/grpc/core/src/convert/feerate_estimate.rs @@ -0,0 +1,66 @@ +use crate::protowire; +use crate::{from, try_from}; +use spectre_rpc_core::RpcError; + +// ---------------------------------------------------------------------------- +// rpc_core to protowire +// ---------------------------------------------------------------------------- + +from!(item: &spectre_rpc_core::RpcFeerateBucket, protowire::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +from!(item: &spectre_rpc_core::RpcFeeEstimate, protowire::RpcFeeEstimate, { + Self { + priority_bucket: Some((&item.priority_bucket).into()), + normal_buckets: item.normal_buckets.iter().map(|b| b.into()).collect(), + low_buckets: item.low_buckets.iter().map(|b| b.into()).collect(), + } +}); + +from!(item: &spectre_rpc_core::RpcFeeEstimateVerboseExperimentalData, protowire::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); + +// ---------------------------------------------------------------------------- +// protowire to rpc_core +// ---------------------------------------------------------------------------- + +try_from!(item: &protowire::RpcFeerateBucket, spectre_rpc_core::RpcFeerateBucket, { + Self { + feerate: item.feerate, + estimated_seconds: item.estimated_seconds, + } +}); + +try_from!(item: &protowire::RpcFeeEstimate, spectre_rpc_core::RpcFeeEstimate, { + Self { + priority_bucket: item.priority_bucket + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("RpcFeeEstimate".to_string(), "priority_bucket".to_string()))? + .try_into()?, + normal_buckets: item.normal_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + low_buckets: item.low_buckets.iter().map(|b| b.try_into()).collect::, _>>()?, + } +}); + +try_from!(item: &protowire::RpcFeeEstimateVerboseExperimentalData, spectre_rpc_core::RpcFeeEstimateVerboseExperimentalData, { + Self { + network_mass_per_second: item.network_mass_per_second, + mempool_ready_transactions_count: item.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: item.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: item.next_block_template_feerate_min, + next_block_template_feerate_median: item.next_block_template_feerate_median, + next_block_template_feerate_max: item.next_block_template_feerate_max, + } +}); diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index 1f737ed..990a421 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -401,6 +401,25 @@ from!(item: RpcResult<&spectre_rpc_core::GetDaaScoreTimestampEstimateResponse>, Self { timestamps: item.timestamps.clone(), error: None } }); +// Fee estimate API + +from!(&spectre_rpc_core::GetFeeEstimateRequest, protowire::GetFeeEstimateRequestMessage); +from!(item: RpcResult<&spectre_rpc_core::GetFeeEstimateResponse>, protowire::GetFeeEstimateResponseMessage, { + Self { estimate: Some((&item.estimate).into()), error: None } +}); +from!(item: &spectre_rpc_core::GetFeeEstimateExperimentalRequest, protowire::GetFeeEstimateExperimentalRequestMessage, { + Self { + verbose: item.verbose + } +}); +from!(item: RpcResult<&spectre_rpc_core::GetFeeEstimateExperimentalResponse>, protowire::GetFeeEstimateExperimentalResponseMessage, { + Self { + estimate: Some((&item.estimate).into()), + verbose: item.verbose.as_ref().map(|x| x.into()), + error: None + } +}); + from!(&spectre_rpc_core::PingRequest, protowire::PingRequestMessage); from!(RpcResult<&spectre_rpc_core::PingResponse>, protowire::PingResponseMessage); @@ -818,6 +837,30 @@ try_from!(item: &protowire::GetDaaScoreTimestampEstimateResponseMessage, RpcResu Self { timestamps: item.timestamps.clone() } }); +try_from!(&protowire::GetFeeEstimateRequestMessage, spectre_rpc_core::GetFeeEstimateRequest); +try_from!(item: &protowire::GetFeeEstimateResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateResponseMessage".to_string(), "estimate".to_string()))? + .try_into()? + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalRequestMessage, spectre_rpc_core::GetFeeEstimateExperimentalRequest, { + Self { + verbose: item.verbose + } +}); +try_from!(item: &protowire::GetFeeEstimateExperimentalResponseMessage, RpcResult, { + Self { + estimate: item.estimate + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("GetFeeEstimateExperimentalResponseMessage".to_string(), "estimate".to_string()))? + .try_into()?, + verbose: item.verbose.as_ref().map(|x| x.try_into()).transpose()? + } +}); + try_from!(&protowire::PingRequestMessage, spectre_rpc_core::PingRequest); try_from!(&protowire::PingResponseMessage, RpcResult); diff --git a/rpc/grpc/core/src/convert/mod.rs b/rpc/grpc/core/src/convert/mod.rs index f7046ea..e27b7f8 100644 --- a/rpc/grpc/core/src/convert/mod.rs +++ b/rpc/grpc/core/src/convert/mod.rs @@ -1,6 +1,7 @@ pub mod address; pub mod block; pub mod error; +pub mod feerate_estimate; pub mod header; pub mod mempool; pub mod message; diff --git a/rpc/grpc/core/src/convert/spectred.rs b/rpc/grpc/core/src/convert/spectred.rs index 195de3d..85e0c5c 100644 --- a/rpc/grpc/core/src/convert/spectred.rs +++ b/rpc/grpc/core/src/convert/spectred.rs @@ -58,6 +58,8 @@ pub mod spectred_request_convert { impl_into_spectred_request!(GetServerInfo); impl_into_spectred_request!(GetSyncStatus); impl_into_spectred_request!(GetDaaScoreTimestampEstimate); + impl_into_spectred_request!(GetFeeEstimate); + impl_into_spectred_request!(GetFeeEstimateExperimental); impl_into_spectred_request!(NotifyBlockAdded); impl_into_spectred_request!(NotifyNewBlockTemplate); @@ -190,6 +192,8 @@ pub mod spectred_response_convert { impl_into_spectred_response!(GetServerInfo); impl_into_spectred_response!(GetSyncStatus); impl_into_spectred_response!(GetDaaScoreTimestampEstimate); + impl_into_spectred_response!(GetFeeEstimate); + impl_into_spectred_response!(GetFeeEstimateExperimental); impl_into_spectred_notify_response!(NotifyBlockAdded); impl_into_spectred_notify_response!(NotifyNewBlockTemplate); diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index 1607645..d4b5bfb 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -82,6 +82,8 @@ pub enum SpectredPayloadOps { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, // Subscription commands for starting/stopping notifications NotifyBlockAdded, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index a2cfd2a..96d4101 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -76,6 +76,8 @@ impl Factory { GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, NotifyBlockAdded, NotifyNewBlockTemplate, NotifyFinalityConflict, diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index 0ab19a0..91efb2d 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -235,6 +235,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API diff --git a/rpc/service/src/converter/feerate_estimate.rs b/rpc/service/src/converter/feerate_estimate.rs new file mode 100644 index 0000000..eb10f94 --- /dev/null +++ b/rpc/service/src/converter/feerate_estimate.rs @@ -0,0 +1,49 @@ +use spectre_mining::feerate::{FeeEstimateVerbose, FeerateBucket, FeerateEstimations}; +use spectre_rpc_core::{ + message::GetFeeEstimateExperimentalResponse as RpcFeeEstimateVerboseResponse, RpcFeeEstimate, + RpcFeeEstimateVerboseExperimentalData as RpcFeeEstimateVerbose, RpcFeerateBucket, +}; + +pub trait FeerateBucketConverter { + fn into_rpc(self) -> RpcFeerateBucket; +} + +impl FeerateBucketConverter for FeerateBucket { + fn into_rpc(self) -> RpcFeerateBucket { + RpcFeerateBucket { feerate: self.feerate, estimated_seconds: self.estimated_seconds } + } +} + +pub trait FeeEstimateConverter { + fn into_rpc(self) -> RpcFeeEstimate; +} + +impl FeeEstimateConverter for FeerateEstimations { + fn into_rpc(self) -> RpcFeeEstimate { + RpcFeeEstimate { + priority_bucket: self.priority_bucket.into_rpc(), + normal_buckets: self.normal_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + low_buckets: self.low_buckets.into_iter().map(FeerateBucketConverter::into_rpc).collect(), + } + } +} + +pub trait FeeEstimateVerboseConverter { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse; +} + +impl FeeEstimateVerboseConverter for FeeEstimateVerbose { + fn into_rpc(self) -> RpcFeeEstimateVerboseResponse { + RpcFeeEstimateVerboseResponse { + estimate: self.estimations.into_rpc(), + verbose: Some(RpcFeeEstimateVerbose { + network_mass_per_second: self.network_mass_per_second, + mempool_ready_transactions_count: self.mempool_ready_transactions_count, + mempool_ready_transactions_total_mass: self.mempool_ready_transactions_total_mass, + next_block_template_feerate_min: self.next_block_template_feerate_min, + next_block_template_feerate_median: self.next_block_template_feerate_median, + next_block_template_feerate_max: self.next_block_template_feerate_max, + }), + } + } +} diff --git a/rpc/service/src/converter/mod.rs b/rpc/service/src/converter/mod.rs index 2e14603..fd167d3 100644 --- a/rpc/service/src/converter/mod.rs +++ b/rpc/service/src/converter/mod.rs @@ -1,3 +1,4 @@ pub mod consensus; +pub mod feerate_estimate; pub mod index; pub mod protocol; diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index 2d9b23f..f947084 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -1,6 +1,7 @@ //! Core server implementation for ClientAPI use super::collector::{CollectorFromConsensus, CollectorFromIndex}; +use crate::converter::feerate_estimate::{FeeEstimateConverter, FeeEstimateVerboseConverter}; use crate::converter::{consensus::ConsensusConverter, index::IndexConverter, protocol::ProtocolConverter}; use crate::service::NetworkType::{Mainnet, Testnet}; use async_trait::async_trait; @@ -61,9 +62,11 @@ use spectre_rpc_core::{ Notification, RpcError, RpcResult, }; use spectre_txscript::{extract_script_pub_key_address, pay_to_address_script}; +use spectre_utils::expiring_cache::ExpiringCache; use spectre_utils::{channel::Channel, triggers::SingleTrigger}; use spectre_utils_tower::counters::TowerConnectionCounters; use spectre_utxoindex::api::UtxoIndexProxy; +use std::time::Duration; use std::{ collections::HashMap, iter::once, @@ -109,6 +112,8 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + fee_estimate_cache: ExpiringCache, + fee_estimate_verbose_cache: ExpiringCache, } const RPC_CORE: &str = "rpc-core"; @@ -208,6 +213,8 @@ impl RpcCoreService { perf_monitor, p2p_tower_counters, grpc_tower_counters, + fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), + fee_estimate_verbose_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } } @@ -663,6 +670,30 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetDaaScoreTimestampEstimateResponse::new(timestamps)) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + let mining_manager = self.mining_manager.clone(); + let estimate = + self.fee_estimate_cache.get(async move { mining_manager.get_realtime_feerate_estimations().await.into_rpc() }).await; + Ok(GetFeeEstimateResponse { estimate }) + } + + async fn get_fee_estimate_experimental_call( + &self, + request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + if request.verbose { + let mining_manager = self.mining_manager.clone(); + let response = self + .fee_estimate_verbose_cache + .get(async move { mining_manager.get_realtime_feerate_estimations_verbose().await.into_rpc() }) + .await; + Ok(response) + } else { + let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; + Ok(GetFeeEstimateExperimentalResponse { estimate, verbose: None }) + } + } + async fn ping_call(&self, _: PingRequest) -> RpcResult { Ok(PingResponse {}) } diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 775c1f9..6df428f 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -596,21 +596,23 @@ impl RpcApi for SpectreRpcClient { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, - GetSyncStatus, + GetSinkBlueScore, GetSubnetwork, + GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index abf806a..f9e5942 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -44,22 +44,24 @@ impl Router { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, - GetDaaScoreTimestampEstimate, - GetServerInfo, GetCurrentNetwork, + GetDaaScoreTimestampEstimate, + GetFeeEstimate, + GetFeeEstimateExperimental, GetHeaders, GetInfo, GetInfo, GetMempoolEntries, GetMempoolEntriesByAddresses, GetMempoolEntry, - GetPeerAddresses, GetMetrics, + GetPeerAddresses, + GetServerInfo, GetSink, + GetSinkBlueScore, GetSubnetwork, GetSyncStatus, GetUtxosByAddresses, - GetSinkBlueScore, GetVirtualChainFromBlock, Ping, ResolveFinalityConflict, diff --git a/spectred/src/args.rs b/spectred/src/args.rs index 88d112c..e7f8622 100644 --- a/spectred/src/args.rs +++ b/spectred/src/args.rs @@ -134,7 +134,7 @@ impl Default for Args { #[cfg(feature = "devnet-prealloc")] prealloc_address: None, #[cfg(feature = "devnet-prealloc")] - prealloc_amount: 1_000_000, + prealloc_amount: 10_000_000_000, disable_upnp: false, disable_dns_seeding: false, diff --git a/spectred/src/daemon.rs b/spectred/src/daemon.rs index 39c9ae0..cc47487 100644 --- a/spectred/src/daemon.rs +++ b/spectred/src/daemon.rs @@ -419,15 +419,16 @@ do you confirm? (answer y/n or pass --yes to the Spectred command line to confir let (address_manager, port_mapping_extender_svc) = AddressManager::new(config.clone(), meta_db, tick_service.clone()); - let mining_monitor = Arc::new(MiningMonitor::new(mining_counters.clone(), tx_script_cache_counters.clone(), tick_service.clone())); let mining_manager = MiningManagerProxy::new(Arc::new(MiningManager::new_with_extended_config( config.target_time_per_block, false, config.max_block_mass, config.ram_scale, config.block_template_cache_lifetime, - mining_counters, + mining_counters.clone(), ))); + let mining_monitor = + Arc::new(MiningMonitor::new(mining_manager.clone(), mining_counters, tx_script_cache_counters.clone(), tick_service.clone())); let flow_context = Arc::new(FlowContext::new( consensus_manager.clone(), diff --git a/testing/integration/src/mempool_benchmarks.rs b/testing/integration/src/mempool_benchmarks.rs index 5b96c5a..3a3a208 100644 --- a/testing/integration/src/mempool_benchmarks.rs +++ b/testing/integration/src/mempool_benchmarks.rs @@ -295,8 +295,8 @@ async fn bench_bbt_latency_2() { const BLOCK_COUNT: usize = usize::MAX; const MEMPOOL_TARGET: u64 = 600_000; - const TX_COUNT: usize = 1_400_000; - const TX_LEVEL_WIDTH: usize = 20_000; + const TX_COUNT: usize = 1_000_000; + const TX_LEVEL_WIDTH: usize = 300_000; const TPS_PRESSURE: u64 = u64::MAX; const SUBMIT_BLOCK_CLIENTS: usize = 20; diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index c2505ea..e8f63b0 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -557,6 +557,33 @@ async fn sanity_test() { }) } + SpectredPayloadOps::GetFeeEstimate => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate().await.unwrap(); + info!("{:?}", response.priority_bucket); + assert!(!response.normal_buckets.is_empty()); + assert!(!response.low_buckets.is_empty()); + for bucket in response.ordered_buckets() { + info!("{:?}", bucket); + } + }) + } + + SpectredPayloadOps::GetFeeEstimateExperimental => { + let rpc_client = client.clone(); + tst!(op, { + let response = rpc_client.get_fee_estimate_experimental(true).await.unwrap(); + assert!(!response.estimate.normal_buckets.is_empty()); + assert!(!response.estimate.low_buckets.is_empty()); + for bucket in response.estimate.ordered_buckets() { + info!("{:?}", bucket); + } + assert!(response.verbose.is_some()); + info!("{:?}", response.verbose); + }) + } + SpectredPayloadOps::NotifyBlockAdded => { let rpc_client = client.clone(); let id = listener_id; diff --git a/testing/integration/src/tasks/tx/sender.rs b/testing/integration/src/tasks/tx/sender.rs index 1962088..705ec4e 100644 --- a/testing/integration/src/tasks/tx/sender.rs +++ b/testing/integration/src/tasks/tx/sender.rs @@ -114,7 +114,7 @@ impl Task for TransactionSenderTask { break; } prev_mempool_size = mempool_size; - sleep(Duration::from_secs(1)).await; + sleep(Duration::from_secs(2)).await; } if stopper == Stopper::Signal { warn!("Tx sender task signaling to stop"); diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 03b2511..63c4077 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true repository.workspace = true [dependencies] +arc-swap.workspace = true parking_lot.workspace = true async-channel.workspace = true borsh.workspace = true diff --git a/utils/src/expiring_cache.rs b/utils/src/expiring_cache.rs new file mode 100644 index 0000000..175bea5 --- /dev/null +++ b/utils/src/expiring_cache.rs @@ -0,0 +1,152 @@ +use arc_swap::ArcSwapOption; +use std::{ + future::Future, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, + }, + time::{Duration, Instant}, +}; + +struct Entry { + item: T, + timestamp: Instant, +} + +/// An expiring cache for a single object +pub struct ExpiringCache { + store: ArcSwapOption>, + refetch: Duration, + expire: Duration, + fetching: AtomicBool, +} + +impl ExpiringCache { + /// Constructs a new expiring cache where `fetch` is the amount of time required to trigger a data + /// refetch and `expire` is the time duration after which the stored item is guaranteed not to be returned. + /// + /// Panics if `refetch > expire`. + pub fn new(refetch: Duration, expire: Duration) -> Self { + assert!(refetch <= expire); + Self { store: Default::default(), refetch, expire, fetching: Default::default() } + } + + /// Returns the cached item or possibly fetches a new one using the `refetch_future` task. The + /// decision whether to refetch depends on the configured expiration and refetch times for this cache. + pub async fn get(&self, refetch_future: F) -> T + where + F: Future + Send + 'static, + F::Output: Send + 'static, + { + let mut fetching = false; + + { + let guard = self.store.load(); + if let Some(entry) = guard.as_ref() { + if let Some(elapsed) = Instant::now().checked_duration_since(entry.timestamp) { + if elapsed < self.refetch { + return entry.item.clone(); + } + // Refetch is triggered, attempt to capture the task + fetching = self.fetching.compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst).is_ok(); + // If the fetch task is not captured and expire time is not over yet, return with prev value. Another + // thread is refetching the data but we can return with the not-too-old value + if !fetching && elapsed < self.expire { + return entry.item.clone(); + } + } + // else -- In rare cases where now < timestamp, fall through to re-update the cache + } + } + + // We reach here if either we are the refetching thread or the current data has fully expired + let new_item = refetch_future.await; + let timestamp = Instant::now(); + // Update the store even if we were not in charge of refetching - let the last thread make the final update + self.store.store(Some(Arc::new(Entry { item: new_item.clone(), timestamp }))); + + if fetching { + let result = self.fetching.compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst); + assert!(result.is_ok(), "refetching was captured") + } + + new_item + } +} + +#[cfg(test)] +mod tests { + use super::ExpiringCache; + use std::time::Duration; + use tokio::join; + + #[tokio::test] + #[ignore] + // Tested during development but can be sensitive to runtime machine times so there's no point + // in keeping it part of CI. The test should be activated if the ExpiringCache struct changes. + async fn test_expiring_cache() { + let fetch = Duration::from_millis(500); + let expire = Duration::from_millis(1000); + let mid_point = Duration::from_millis(700); + let expire_point = Duration::from_millis(1200); + let cache: ExpiringCache = ExpiringCache::new(fetch, expire); + + // Test two consecutive calls + let item1 = cache + .get(async move { + println!("first call"); + 1 + }) + .await; + assert_eq!(1, item1); + let item2 = cache + .get(async move { + // cache was just updated with item1, refetch should not be triggered + panic!("should not be called"); + }) + .await; + assert_eq!(1, item2); + + // Test two calls after refetch point + // Sleep until after the refetch point but before expire + tokio::time::sleep(mid_point).await; + let call3 = cache.get(async move { + println!("third call before sleep"); + // keep this refetch busy so that call4 still gets the first item + tokio::time::sleep(Duration::from_millis(100)).await; + println!("third call after sleep"); + 3 + }); + let call4 = cache.get(async move { + // refetch is captured by call3 and we should be before expire + panic!("should not be called"); + }); + let (item3, item4) = join!(call3, call4); + println!("item 3: {}, item 4: {}", item3, item4); + assert_eq!(3, item3); + assert_eq!(1, item4); + + // Test 2 calls after expire + tokio::time::sleep(expire_point).await; + let call5 = cache.get(async move { + println!("5th call before sleep"); + tokio::time::sleep(Duration::from_millis(100)).await; + println!("5th call after sleep"); + 5 + }); + let call6 = cache.get(async move { 6 }); + let (item5, item6) = join!(call5, call6); + println!("item 5: {}, item 6: {}", item5, item6); + assert_eq!(5, item5); + assert_eq!(6, item6); + + let item7 = cache + .get(async move { + // cache was just updated with item5, refetch should not be triggered + panic!("should not be called"); + }) + .await; + // call 5 finished after call 6 + assert_eq!(5, item7); + } +} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index 7de47e1..13aeba3 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -2,6 +2,7 @@ pub mod any; pub mod arc; pub mod binary_heap; pub mod channel; +pub mod expiring_cache; pub mod hashmap; pub mod hex; pub mod iter; diff --git a/utils/src/rand/mod.rs b/utils/src/rand/mod.rs new file mode 100644 index 0000000..ed6dcf1 --- /dev/null +++ b/utils/src/rand/mod.rs @@ -0,0 +1 @@ +pub mod seq; diff --git a/utils/src/rand/seq.rs b/utils/src/rand/seq.rs new file mode 100644 index 0000000..708c939 --- /dev/null +++ b/utils/src/rand/seq.rs @@ -0,0 +1,76 @@ +pub mod index { + use rand::{distributions::Uniform, prelude::Distribution, Rng}; + use std::collections::HashSet; + + /// Adaptation of [`rand::seq::index::sample`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. + pub fn sample(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + const C: [f32; 2] = [270.0, 330.0 / 9.0]; + let j = if length < 500_000 { 0 } else { 1 }; + if (length as f32) < C[j] * (amount as f32) { + sample_inplace(rng, length, amount, capacity, filter) + } else { + sample_rejection(rng, length, amount, capacity, filter) + } + } + + /// Adaptation of [`rand::seq::index::sample_inplace`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. + fn sample_inplace(rng: &mut R, length: u32, amount: u32, capacity: u32, filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + debug_assert!(amount <= length); + debug_assert!(filter.len() <= amount as usize); + let mut indices: Vec = Vec::with_capacity(length.max(capacity) as usize); + indices.extend(0..length); + for i in 0..amount { + let mut j: u32 = rng.gen_range(i..length); + // Assumes |filter| << length + while filter.contains(&j) { + j = rng.gen_range(i..length); + } + indices.swap(i as usize, j as usize); + } + indices.truncate(amount as usize); + debug_assert_eq!(indices.len(), amount as usize); + indices + } + + /// Adaptation of [`rand::seq::index::sample_rejection`] for the case where there exists an a priory filter + /// of indices which should not be selected. + /// + /// Assumes `|filter| << length`. + /// + /// The argument `capacity` can be used to ensure a larger allocation within the returned vector. + fn sample_rejection(rng: &mut R, length: u32, amount: u32, capacity: u32, mut filter: HashSet) -> Vec + where + R: Rng + ?Sized, + { + debug_assert!(amount < length); + debug_assert!(filter.len() <= amount as usize); + let distr = Uniform::new(0, length); + let mut indices = Vec::with_capacity(amount.max(capacity) as usize); + for _ in indices.len()..amount as usize { + let mut pos = distr.sample(rng); + while !filter.insert(pos) { + pos = distr.sample(rng); + } + indices.push(pos); + } + + assert_eq!(indices.len(), amount as usize); + indices + } +} diff --git a/utils/src/vec.rs b/utils/src/vec.rs index 01bd59b..fa1d67a 100644 --- a/utils/src/vec.rs +++ b/utils/src/vec.rs @@ -4,6 +4,10 @@ pub trait VecExtensions { /// Inserts the provided `value` at `index` while swapping the item at index to the end of the container fn swap_insert(&mut self, index: usize, value: T); + + /// Merges two containers one into the other and returns the result. The method is identical + /// to [`Vec::append`] but can be used more ergonomically in a fluent calling fashion + fn merge(self, other: Self) -> Self; } impl VecExtensions for Vec { @@ -19,4 +23,9 @@ impl VecExtensions for Vec { let loc = self.len() - 1; self.swap(index, loc); } + + fn merge(mut self, mut other: Self) -> Self { + self.append(&mut other); + self + } } diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index 171a08b..cffb021 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -252,6 +252,17 @@ impl RpcApi for RpcCoreMock { Err(RpcError::NotImplemented) } + async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_fee_estimate_experimental_call( + &self, + _request: GetFeeEstimateExperimentalRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Notification API From ec3b0645c8d6fae4fb6eb1a1f4bac12b4b47dd97 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:40:07 +0200 Subject: [PATCH 09/48] update mimalloc dep * fixes compiling on debian https://github.com/microsoft/mimalloc/issues/883 --- utils/alloc/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/utils/alloc/Cargo.toml b/utils/alloc/Cargo.toml index d290048..d332fb8 100644 --- a/utils/alloc/Cargo.toml +++ b/utils/alloc/Cargo.toml @@ -10,13 +10,13 @@ include.workspace = true repository.workspace = true [target.'cfg(not(target_os = "macos"))'.dependencies] -mimalloc = { version = "0.1.39", default-features = false, features = [ +mimalloc = { version = "0.1.43", default-features = false, features = [ 'override', ] } [target.'cfg(target_os = "macos")'.dependencies] # override is unstable in MacOS and is thus excluded -mimalloc = { version = "0.1.39", default-features = false } +mimalloc = { version = "0.1.43", default-features = false } [features] heap = [] From f24bbb7f7cf81fd1ca1cf7fe801df32170585a32 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 02:50:05 +0200 Subject: [PATCH 10/48] feat: next_block_template_feerate --- consensus/core/src/block.rs | 13 +- .../pipeline/virtual_processor/processor.rs | 53 ++-- .../virtual_processor/test_block_builder.rs | 2 +- mining/src/manager.rs | 142 +++++++++- .../model/{ => frontier}/feerate_key.rs | 43 ++- .../mempool/model/frontier/feerate_weight.rs | 265 ------------------ mining/src/testutils/consensus_mock.rs | 2 +- rpc/service/src/service.rs | 13 +- 8 files changed, 227 insertions(+), 306 deletions(-) rename mining/src/mempool/model/{ => frontier}/feerate_key.rs (50%) delete mode 100644 mining/src/mempool/model/frontier/feerate_weight.rs diff --git a/consensus/core/src/block.rs b/consensus/core/src/block.rs index 36bc1bd..a646e9c 100644 --- a/consensus/core/src/block.rs +++ b/consensus/core/src/block.rs @@ -105,6 +105,8 @@ pub struct BlockTemplate { pub selected_parent_timestamp: u64, pub selected_parent_daa_score: u64, pub selected_parent_hash: Hash, + /// Expected length is one less than txs length due to lack of coinbase transaction + pub calculated_fees: Vec, } impl BlockTemplate { @@ -115,8 +117,17 @@ impl BlockTemplate { selected_parent_timestamp: u64, selected_parent_daa_score: u64, selected_parent_hash: Hash, + calculated_fees: Vec, ) -> Self { - Self { block, miner_data, coinbase_has_red_reward, selected_parent_timestamp, selected_parent_daa_score, selected_parent_hash } + Self { + block, + miner_data, + coinbase_has_red_reward, + selected_parent_timestamp, + selected_parent_daa_score, + selected_parent_hash, + calculated_fees, + } } pub fn to_virtual_state_approx_id(&self) -> VirtualStateApproxId { diff --git a/consensus/src/pipeline/virtual_processor/processor.rs b/consensus/src/pipeline/virtual_processor/processor.rs index bccdd6c..cb44206 100644 --- a/consensus/src/pipeline/virtual_processor/processor.rs +++ b/consensus/src/pipeline/virtual_processor/processor.rs @@ -77,6 +77,7 @@ use spectre_hashes::Hash; use spectre_muhash::MuHash; use spectre_notify::{events::EventType, notifier::Notify}; +use super::errors::{PruningImportError, PruningImportResult}; use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use itertools::Itertools; use parking_lot::{RwLock, RwLockUpgradableReadGuard}; @@ -86,6 +87,7 @@ use rayon::{ ThreadPool, }; use rocksdb::WriteBatch; +use spectre_consensus_core::tx::ValidatedTransaction; use spectre_utils::binary_heap::BinaryHeapExtensions; use std::{ cmp::min, @@ -94,8 +96,6 @@ use std::{ sync::{atomic::Ordering, Arc}, }; -use super::errors::{PruningImportError, PruningImportResult}; - pub struct VirtualStateProcessor { // Channels receiver: CrossbeamReceiver, @@ -833,12 +833,9 @@ impl VirtualStateProcessor { txs: &[Transaction], virtual_state: &VirtualState, utxo_view: &V, - ) -> Vec> { - self.thread_pool.install(|| { - txs.par_iter() - .map(|tx| self.validate_block_template_transaction(tx, virtual_state, &utxo_view)) - .collect::>>() - }) + ) -> Vec> { + self.thread_pool + .install(|| txs.par_iter().map(|tx| self.validate_block_template_transaction(tx, virtual_state, &utxo_view)).collect()) } fn validate_block_template_transaction( @@ -846,13 +843,14 @@ impl VirtualStateProcessor { tx: &Transaction, virtual_state: &VirtualState, utxo_view: &impl UtxoView, - ) -> TxResult<()> { + ) -> TxResult { // No need to validate the transaction in isolation since we rely on the mining manager to submit transactions // which were previously validated through `validate_mempool_transaction_and_populate`, hence we only perform // in-context validations self.transaction_validator.utxo_free_tx_validation(tx, virtual_state.daa_score, virtual_state.past_median_time)?; - self.validate_transaction_in_utxo_context(tx, utxo_view, virtual_state.daa_score, TxValidationFlags::Full)?; - Ok(()) + let ValidatedTransaction { calculated_fee, .. } = + self.validate_transaction_in_utxo_context(tx, utxo_view, virtual_state.daa_score, TxValidationFlags::Full)?; + Ok(calculated_fee) } pub fn build_block_template( @@ -869,7 +867,7 @@ impl VirtualStateProcessor { // optimizing for the common case where all txs are valid. Following selection calls // are called within the lock in order to preserve validness of already validated txs let mut txs = tx_selector.select_transactions(); - + let mut calculated_fees = Vec::with_capacity(txs.len()); let virtual_read = self.virtual_stores.read(); let virtual_state = virtual_read.state.get().unwrap(); let virtual_utxo_view = &virtual_read.utxo_set; @@ -877,9 +875,14 @@ impl VirtualStateProcessor { let mut invalid_transactions = HashMap::new(); let results = self.validate_block_template_transactions_in_parallel(&txs, &virtual_state, &virtual_utxo_view); for (tx, res) in txs.iter().zip(results) { - if let Err(e) = res { - invalid_transactions.insert(tx.id(), e); - tx_selector.reject_selection(tx.id()); + match res { + Err(e) => { + invalid_transactions.insert(tx.id(), e); + tx_selector.reject_selection(tx.id()); + } + Ok(fee) => { + calculated_fees.push(fee); + } } } @@ -894,12 +897,16 @@ impl VirtualStateProcessor { let next_batch_results = self.validate_block_template_transactions_in_parallel(&next_batch, &virtual_state, &virtual_utxo_view); for (tx, res) in next_batch.into_iter().zip(next_batch_results) { - if let Err(e) = res { - invalid_transactions.insert(tx.id(), e); - tx_selector.reject_selection(tx.id()); - has_rejections = true; - } else { - txs.push(tx); + match res { + Err(e) => { + invalid_transactions.insert(tx.id(), e); + tx_selector.reject_selection(tx.id()); + has_rejections = true; + } + Ok(fee) => { + txs.push(tx); + calculated_fees.push(fee); + } } } } @@ -916,7 +923,7 @@ impl VirtualStateProcessor { drop(virtual_read); // Build the template - self.build_block_template_from_virtual_state(virtual_state, miner_data, txs) + self.build_block_template_from_virtual_state(virtual_state, miner_data, txs, calculated_fees) } pub(crate) fn validate_block_template_transactions( @@ -944,6 +951,7 @@ impl VirtualStateProcessor { virtual_state: Arc, miner_data: MinerData, mut txs: Vec, + calculated_fees: Vec, ) -> Result { // [`calc_block_parents`] can use deep blocks below the pruning point for this calculation, so we // need to hold the pruning lock. @@ -997,6 +1005,7 @@ impl VirtualStateProcessor { selected_parent_timestamp, selected_parent_daa_score, selected_parent_hash, + calculated_fees, )) } diff --git a/consensus/src/pipeline/virtual_processor/test_block_builder.rs b/consensus/src/pipeline/virtual_processor/test_block_builder.rs index ce747de..533b982 100644 --- a/consensus/src/pipeline/virtual_processor/test_block_builder.rs +++ b/consensus/src/pipeline/virtual_processor/test_block_builder.rs @@ -61,6 +61,6 @@ impl TestBlockBuilder { let pov_virtual_utxo_view = (&virtual_read.utxo_set).compose(accumulated_diff); self.validate_block_template_transactions(&txs, &pov_virtual_state, &pov_virtual_utxo_view)?; drop(virtual_read); - self.build_block_template_from_virtual_state(pov_virtual_state, miner_data, txs) + self.build_block_template_from_virtual_state(pov_virtual_state, miner_data, txs, vec![]) } } diff --git a/mining/src/manager.rs b/mining/src/manager.rs index c4f398a..d286ab5 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -35,7 +35,7 @@ use spectre_consensus_core::{ use spectre_consensusmanager::{spawn_blocking, ConsensusProxy}; use spectre_core::{debug, error, info, time::Stopwatch, warn}; use spectre_mining_errors::{manager::MiningManagerError, mempool::RuleError}; -use std::sync::Arc; +use std::{ops::Mul, sync::Arc}; use tokio::sync::mpsc::UnboundedSender; pub struct MiningManager { @@ -210,7 +210,11 @@ impl MiningManager { } /// Returns realtime feerate estimations based on internal mempool state with additional verbose data - pub(crate) fn get_realtime_feerate_estimations_verbose(&self) -> FeeEstimateVerbose { + pub(crate) fn get_realtime_feerate_estimations_verbose( + &self, + consensus: &dyn ConsensusApi, + prefix: spectre_addresses::Prefix, + ) -> MiningManagerResult { let args = FeerateEstimatorArgs::new(self.config.network_blocks_per_second, self.config.maximum_mass_per_block); let network_mass_per_second = args.network_mass_per_second(); let mempool_read = self.mempool.read(); @@ -218,16 +222,37 @@ impl MiningManager { let ready_transactions_count = mempool_read.ready_transaction_count(); let ready_transaction_total_mass = mempool_read.ready_transaction_total_mass(); drop(mempool_read); - FeeEstimateVerbose { + let mut resp = FeeEstimateVerbose { estimations: estimator.calc_estimations(self.config.minimum_feerate()), network_mass_per_second, mempool_ready_transactions_count: ready_transactions_count as u64, mempool_ready_transactions_total_mass: ready_transaction_total_mass, - // TODO: Next PR + next_block_template_feerate_min: -1.0, next_block_template_feerate_median: -1.0, next_block_template_feerate_max: -1.0, + }; + // calculate next_block_template_feerate_xxx + { + let script_public_key = spectre_txscript::pay_to_address_script(&spectre_addresses::Address::new( + prefix, + spectre_addresses::Version::PubKey, + &[0u8; 32], + )); + let miner_data: MinerData = MinerData::new(script_public_key, vec![]); + + let BlockTemplate { block: spectre_consensus_core::block::MutableBlock { transactions, .. }, calculated_fees, .. } = + self.get_block_template(consensus, &miner_data)?; + + let Some(Stats { max, median, min }) = feerate_stats(transactions, calculated_fees) else { + return Ok(resp); + }; + + resp.next_block_template_feerate_max = max; + resp.next_block_template_feerate_min = min; + resp.next_block_template_feerate_median = median; } + Ok(resp) } /// Clears the block template cache, forcing the next call to get_block_template to build a new block template. @@ -834,8 +859,12 @@ impl MiningManagerProxy { } /// Returns realtime feerate estimations based on internal mempool state with additional verbose data - pub async fn get_realtime_feerate_estimations_verbose(self) -> FeeEstimateVerbose { - spawn_blocking(move || self.inner.get_realtime_feerate_estimations_verbose()).await.unwrap() + pub async fn get_realtime_feerate_estimations_verbose( + self, + consensus: &ConsensusProxy, + prefix: spectre_addresses::Prefix, + ) -> MiningManagerResult { + consensus.clone().spawn_blocking(move |c| self.inner.get_realtime_feerate_estimations_verbose(c, prefix)).await } /// Validates a transaction and adds it to the set of known transactions that have not yet been @@ -982,3 +1011,104 @@ impl MiningManagerProxy { count } } + +/// Represents statistical information about fee rates of transactions. +struct Stats { + /// The maximum fee rate observed. + max: f64, + /// The median fee rate observed. + median: f64, + /// The minimum fee rate observed. + min: f64, +} +/// Calculates the maximum, median, and minimum fee rates (fee per unit mass) +/// for a set of transactions, excluding the first transaction which is assumed +/// to be the coinbase transaction. +/// +/// # Arguments +/// +/// * `transactions` - A vector of `Transaction` objects. The first transaction +/// is assumed to be the coinbase transaction and is excluded from fee rate +/// calculations. +/// * `calculated_fees` - A vector of fees associated with the transactions. +/// This vector should have one less element than the `transactions` vector +/// since the first transaction (coinbase) does not have a fee. +/// +/// # Returns +/// +/// Returns an `Option` containing the maximum, median, and minimum fee +/// rates if the input vectors are valid. Returns `None` if the vectors are +/// empty or if the lengths are inconsistent. +fn feerate_stats(transactions: Vec, calculated_fees: Vec) -> Option { + if calculated_fees.is_empty() { + return None; + } + if transactions.len() != calculated_fees.len() + 1 { + error!("Transactions length must be one more than `calculated_fees` length"); + return None; + } + debug_assert!(transactions[0].is_coinbase()); + let mut fees_and_masses = calculated_fees + .into_iter() + .zip(transactions + .iter() + // skip coinbase tx + .skip(1) + .map(Transaction::mass)) + .collect_vec(); + + // Sort by fee rate without performing division for each comparison. + // Using multiplication instead of division is faster and avoids the need + // to convert all values to floats. Division is only performed later when + // calculating the min, max, and median fee rates. + fees_and_masses.sort_unstable_by(|(lhs_fee, lhs_mass), (rhs_fee, rhs_mass)| lhs_fee.mul(rhs_mass).cmp(&rhs_fee.mul(lhs_mass))); + + let div_as_f64 = |(fee, mass)| fee as f64 / mass as f64; + let max = div_as_f64(fees_and_masses[fees_and_masses.len() - 1]); + let min = div_as_f64(fees_and_masses[0]); + let median = div_as_f64(fees_and_masses[fees_and_masses.len() / 2]); + + Some(Stats { max, median, min }) +} + +#[cfg(test)] +mod tests { + use super::*; + use spectre_consensus_core::subnets; + use std::iter::repeat; + + fn transactions(length: usize) -> Vec { + let tx = || { + let tx = Transaction::new(0, vec![], vec![], 0, Default::default(), 0, vec![]); + tx.set_mass(2); + tx + }; + let mut txs = repeat(tx()).take(length).collect_vec(); + txs[0].subnetwork_id = subnets::SUBNETWORK_ID_COINBASE; + txs + } + + #[test] + fn feerate_stats_test() { + let calculated_fees = vec![100u64, 200, 300, 400]; + let txs = transactions(calculated_fees.len() + 1); + let Stats { max, median, min } = feerate_stats(txs, calculated_fees).unwrap(); + assert_eq!(max, 200.0); + assert_eq!(median, 150.0); + assert_eq!(min, 50.0); + } + + #[test] + fn feerate_stats_empty_test() { + let calculated_fees = vec![]; + let txs = transactions(calculated_fees.len() + 1); + assert!(feerate_stats(txs, calculated_fees).is_none()); + } + + #[test] + fn feerate_stats_inconsistent_test() { + let calculated_fees = vec![100u64, 200, 300, 400]; + let txs = transactions(calculated_fees.len()); + assert!(feerate_stats(txs, calculated_fees).is_none()); + } +} diff --git a/mining/src/mempool/model/feerate_key.rs b/mining/src/mempool/model/frontier/feerate_key.rs similarity index 50% rename from mining/src/mempool/model/feerate_key.rs rename to mining/src/mempool/model/frontier/feerate_key.rs index c0e9629..74842c5 100644 --- a/mining/src/mempool/model/feerate_key.rs +++ b/mining/src/mempool/model/frontier/feerate_key.rs @@ -1,6 +1,4 @@ -use crate::block_template::selector::ALPHA; - -use super::tx::MempoolTransaction; +use crate::{block_template::selector::ALPHA, mempool::model::tx::MempoolTransaction}; use spectre_consensus_core::tx::Transaction; use std::sync::Arc; @@ -22,6 +20,9 @@ impl PartialEq for FeerateTransactionKey { impl FeerateTransactionKey { pub fn new(fee: u64, mass: u64, tx: Arc) -> Self { + // NOTE: any change to the way this weight is calculated (such as scaling by some factor) + // requires a reversed update to total_weight in `Frontier::build_feerate_estimator`. This + // is because the math methods in FeeEstimator assume this specific weight function. Self { fee, mass, weight: (fee as f64 / mass as f64).powi(ALPHA), tx } } @@ -49,13 +50,15 @@ impl PartialOrd for FeerateTransactionKey { impl Ord for FeerateTransactionKey { fn cmp(&self, other: &Self) -> std::cmp::Ordering { - // Our first priority is the feerate - match self.feerate().total_cmp(&other.feerate()) { + // Our first priority is the feerate. + // The weight function is monotonic in feerate so we prefer using it + // since it is cached + match self.weight().total_cmp(&other.weight()) { core::cmp::Ordering::Equal => {} ord => return ord, } - // If feerates are equal, prefer the higher fee in absolute value + // If feerates (and thus weights) are equal, prefer the higher fee in absolute value match self.fee.cmp(&other.fee) { core::cmp::Ordering::Equal => {} ord => return ord, @@ -74,6 +77,32 @@ impl Ord for FeerateTransactionKey { impl From<&MempoolTransaction> for FeerateTransactionKey { fn from(tx: &MempoolTransaction) -> Self { - Self::new(tx.mtx.calculated_fee.unwrap(), tx.mtx.tx.mass(), tx.mtx.tx.clone()) + let mass = tx.mtx.tx.mass(); + let fee = tx.mtx.calculated_fee.expect("fee is expected to be populated"); + assert_ne!(mass, 0, "mass field is expected to be set when inserting to the mempool"); + Self::new(fee, mass, tx.mtx.tx.clone()) + } +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use spectre_consensus_core::{ + subnets::SUBNETWORK_ID_NATIVE, + tx::{Transaction, TransactionInput, TransactionOutpoint}, + }; + use spectre_hashes::{HasherBase, TransactionID}; + use std::sync::Arc; + + fn generate_unique_tx(i: u64) -> Arc { + let mut hasher = TransactionID::new(); + let prev = hasher.update(i.to_le_bytes()).clone().finalize(); + let input = TransactionInput::new(TransactionOutpoint::new(prev, 0), vec![], 0, 0); + Arc::new(Transaction::new(0, vec![input], vec![], 0, SUBNETWORK_ID_NATIVE, 0, vec![])) + } + + /// Test helper for generating a feerate key with a unique tx (per u64 id) + pub(crate) fn build_feerate_key(fee: u64, mass: u64, id: u64) -> FeerateTransactionKey { + FeerateTransactionKey::new(fee, mass, generate_unique_tx(id)) } } diff --git a/mining/src/mempool/model/frontier/feerate_weight.rs b/mining/src/mempool/model/frontier/feerate_weight.rs deleted file mode 100644 index 8421c7b..0000000 --- a/mining/src/mempool/model/frontier/feerate_weight.rs +++ /dev/null @@ -1,265 +0,0 @@ -use super::feerate_key::FeerateTransactionKey; -use sweep_bptree::tree::visit::{DescendVisit, DescendVisitResult}; -use sweep_bptree::tree::{Argument, SearchArgument}; -use sweep_bptree::{BPlusTree, NodeStoreVec}; - -type FeerateKey = FeerateTransactionKey; - -#[derive(Clone, Copy, Debug, Default)] -struct FeerateWeight(f64); - -impl FeerateWeight { - /// Returns the weight value - pub fn weight(&self) -> f64 { - self.0 - } -} - -impl Argument for FeerateWeight { - fn from_leaf(keys: &[FeerateKey]) -> Self { - Self(keys.iter().map(|k| k.weight()).sum()) - } - - fn from_inner(_keys: &[FeerateKey], arguments: &[Self]) -> Self { - Self(arguments.iter().map(|a| a.0).sum()) - } -} - -impl SearchArgument for FeerateWeight { - type Query = f64; - - fn locate_in_leaf(query: Self::Query, keys: &[FeerateKey]) -> Option { - let mut sum = 0.0; - for (i, k) in keys.iter().enumerate() { - let w = k.weight(); - sum += w; - if query < sum { - return Some(i); - } - } - // In order to avoid sensitivity to floating number arithmetics, - // we logically "clamp" the search, returning the last leaf if the query - // value is out of bounds - match keys.len() { - 0 => None, - n => Some(n - 1), - } - } - - fn locate_in_inner(mut query: Self::Query, _keys: &[FeerateKey], arguments: &[Self]) -> Option<(usize, Self::Query)> { - for (i, a) in arguments.iter().enumerate() { - if query >= a.0 { - query -= a.0; - } else { - return Some((i, query)); - } - } - // In order to avoid sensitivity to floating number arithmetics, - // we logically "clamp" the search, returning the last subtree if the query - // value is out of bounds. Eventually this will lead to the return of the - // last leaf (see locate_in_leaf as well) - match arguments.len() { - 0 => None, - n => Some((n - 1, arguments[n - 1].0)), - } - } -} - -struct PrefixWeightVisitor<'a> { - key: &'a FeerateKey, - accumulated_weight: f64, -} - -impl<'a> PrefixWeightVisitor<'a> { - pub fn new(key: &'a FeerateKey) -> Self { - Self { key, accumulated_weight: Default::default() } - } - - fn search_in_keys(&self, keys: &[FeerateKey]) -> usize { - match keys.binary_search(self.key) { - Err(idx) => { - // The idx is the place where a matching element could be inserted while maintaining - // sorted order, go to left child - idx - } - Ok(idx) => { - // Exact match, go to right child. - idx + 1 - } - } - } -} - -impl<'a> DescendVisit for PrefixWeightVisitor<'a> { - type Result = f64; - - fn visit_inner(&mut self, keys: &[FeerateKey], arguments: &[FeerateWeight]) -> DescendVisitResult { - let idx = self.search_in_keys(keys); - // trace!("[visit_inner] {}, {}, {}", keys.len(), arguments.len(), idx); - for argument in arguments.iter().take(idx) { - self.accumulated_weight += argument.weight(); - } - DescendVisitResult::GoDown(idx) - } - - fn visit_leaf(&mut self, keys: &[FeerateKey], _values: &[()]) -> Option { - let idx = self.search_in_keys(keys); - // trace!("[visit_leaf] {}, {}", keys.len(), idx); - for key in keys.iter().take(idx) { - self.accumulated_weight += key.weight(); - } - Some(self.accumulated_weight) - } -} - -type InnerTree = BPlusTree>; - -pub struct SearchTree { - tree: InnerTree, -} - -impl Default for SearchTree { - fn default() -> Self { - Self { tree: InnerTree::new(Default::default()) } - } -} - -impl SearchTree { - pub fn new() -> Self { - Self { tree: InnerTree::new(Default::default()) } - } - - pub fn len(&self) -> usize { - self.tree.len() - } - - pub fn is_empty(&self) -> bool { - self.len() == 0 - } - - pub fn insert(&mut self, key: FeerateKey) -> bool { - self.tree.insert(key, ()).is_none() - } - - pub fn remove(&mut self, key: &FeerateKey) -> bool { - self.tree.remove(key).is_some() - } - - pub fn search(&self, query: f64) -> &FeerateKey { - self.tree.get_by_argument(query).expect("clamped").0 - } - - pub fn total_weight(&self) -> f64 { - self.tree.root_argument().weight() - } - - pub fn prefix_weight(&self, key: &FeerateKey) -> f64 { - self.tree.descend_visit(PrefixWeightVisitor::new(key)).unwrap() - } - - pub fn descending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { - self.tree.iter().rev().map(|(key, ())| key) - } - - pub fn ascending_iter(&self) -> impl DoubleEndedIterator + ExactSizeIterator { - self.tree.iter().map(|(key, ())| key) - } - - pub fn first(&self) -> Option<&FeerateKey> { - self.tree.first().map(|(k, ())| k) - } - - pub fn last(&self) -> Option<&FeerateKey> { - self.tree.last().map(|(k, ())| k) - } -} - -#[cfg(test)] -mod tests { - use super::super::feerate_key::tests::build_feerate_key; - use super::*; - use itertools::Itertools; - use std::collections::HashSet; - - #[test] - fn test_feerate_weight_queries() { - let mut btree = SearchTree::new(); - let mass = 2000; - // The btree stores N=64 keys at each node/leaf, so we make sure the tree has more than - // 64^2 keys in order to trigger at least a few intermediate tree nodes - let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); - - #[allow(clippy::mutable_key_type)] - let mut s = HashSet::with_capacity(fees.len()); - for (i, fee) in fees.iter().copied().enumerate() { - let key = build_feerate_key(fee, mass, i as u64); - s.insert(key.clone()); - btree.insert(key); - } - - // Randomly remove 1/6 of the items - let remove = s.iter().take(fees.len() / 6).cloned().collect_vec(); - for r in remove { - s.remove(&r); - btree.remove(&r); - } - - // Collect to vec and sort for reference - let mut v = s.into_iter().collect_vec(); - v.sort(); - - // Test reverse iteration - for (expected, item) in v.iter().rev().zip(btree.descending_iter()) { - assert_eq!(&expected, &item); - assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well - } - - // Sweep through the tree and verify that weight search queries are handled correctly - let eps: f64 = 0.001; - let mut sum = 0.0; - for expected in v { - let weight = expected.weight(); - let eps = eps.min(weight / 3.0); - let samples = [sum + eps, sum + weight / 2.0, sum + weight - eps]; - for sample in samples { - let key = btree.search(sample); - assert_eq!(&expected, key); - assert!(expected.cmp(key).is_eq()); // Assert Ord equality as well - } - sum += weight; - } - - println!("{}, {}", sum, btree.total_weight()); - - // Test clamped search bounds - assert_eq!(btree.first(), Some(btree.search(f64::NEG_INFINITY))); - assert_eq!(btree.first(), Some(btree.search(-1.0))); - assert_eq!(btree.first(), Some(btree.search(-eps))); - assert_eq!(btree.first(), Some(btree.search(0.0))); - assert_eq!(btree.last(), Some(btree.search(sum))); - assert_eq!(btree.last(), Some(btree.search(sum + eps))); - assert_eq!(btree.last(), Some(btree.search(sum + 1.0))); - assert_eq!(btree.last(), Some(btree.search(1.0 / 0.0))); - assert_eq!(btree.last(), Some(btree.search(f64::INFINITY))); - let _ = btree.search(f64::NAN); - } - - #[test] - fn test_btree_rev_iter() { - let mut btree = SearchTree::new(); - let mass = 2000; - let fees = vec![[123, 113, 10_000, 1000, 2050, 2048]; 64 * (64 + 1)].into_iter().flatten().collect_vec(); - let mut v = Vec::with_capacity(fees.len()); - for (i, fee) in fees.iter().copied().enumerate() { - let key = build_feerate_key(fee, mass, i as u64); - v.push(key.clone()); - btree.insert(key); - } - v.sort(); - - for (expected, item) in v.into_iter().rev().zip(btree.descending_iter()) { - assert_eq!(&expected, item); - assert!(expected.cmp(item).is_eq()); // Assert Ord equality as well - } - } -} diff --git a/mining/src/testutils/consensus_mock.rs b/mining/src/testutils/consensus_mock.rs index 09be800..33fe3b8 100644 --- a/mining/src/testutils/consensus_mock.rs +++ b/mining/src/testutils/consensus_mock.rs @@ -103,7 +103,7 @@ impl ConsensusApi for ConsensusMock { ); let mutable_block = MutableBlock::new(header, txs); - Ok(BlockTemplate::new(mutable_block, miner_data, coinbase.has_red_reward, now, 0, ZERO_HASH)) + Ok(BlockTemplate::new(mutable_block, miner_data, coinbase.has_red_reward, now, 0, ZERO_HASH, vec![])) } fn validate_mempool_transaction(&self, mutable_tx: &mut MutableTransaction, _: &TransactionValidationArgs) -> TxResult<()> { diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index f947084..e31bdac 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -35,6 +35,7 @@ use spectre_index_core::{ connection::IndexChannelConnection, indexed_utxos::UtxoSetByScriptPublicKey, notification::Notification as IndexNotification, notifier::IndexNotifier, }; +use spectre_mining::feerate::FeeEstimateVerbose; use spectre_mining::model::tx_query::TransactionQuery; use spectre_mining::{manager::MiningManagerProxy, mempool::tx::Orphan}; use spectre_notify::listener::ListenerLifespan; @@ -113,7 +114,7 @@ pub struct RpcCoreService { p2p_tower_counters: Arc, grpc_tower_counters: Arc, fee_estimate_cache: ExpiringCache, - fee_estimate_verbose_cache: ExpiringCache, + fee_estimate_verbose_cache: ExpiringCache>, } const RPC_CORE: &str = "rpc-core"; @@ -683,10 +684,16 @@ NOTE: This error usually indicates an RPC conversion error between the node and ) -> RpcResult { if request.verbose { let mining_manager = self.mining_manager.clone(); + let consensus_manager = self.consensus_manager.clone(); + let prefix = self.config.prefix(); + let response = self .fee_estimate_verbose_cache - .get(async move { mining_manager.get_realtime_feerate_estimations_verbose().await.into_rpc() }) - .await; + .get(async move { + let session = consensus_manager.consensus().unguarded_session(); + mining_manager.get_realtime_feerate_estimations_verbose(&session, prefix).await.map(FeeEstimateVerbose::into_rpc) + }) + .await?; Ok(response) } else { let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; From 411bc42aa7b3da8f8f88316f305872442c9ed1eb Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 13:28:08 +0200 Subject: [PATCH 11/48] semaphore tracing feature (for tracing prune readers vs writers time) --- mining/src/manager.rs | 25 +++++----- simpa/Cargo.toml | 2 + simpa/src/main.rs | 15 +++++- spectred/Cargo.toml | 5 +- spectred/src/daemon.rs | 8 +++- utils/Cargo.toml | 8 +++- utils/src/sync/mod.rs | 5 ++ utils/src/sync/semaphore.rs | 92 +++++++++++++++++++++++++++++++++++-- 8 files changed, 138 insertions(+), 22 deletions(-) diff --git a/mining/src/manager.rs b/mining/src/manager.rs index d286ab5..f07754a 100644 --- a/mining/src/manager.rs +++ b/mining/src/manager.rs @@ -35,7 +35,7 @@ use spectre_consensus_core::{ use spectre_consensusmanager::{spawn_blocking, ConsensusProxy}; use spectre_core::{debug, error, info, time::Stopwatch, warn}; use spectre_mining_errors::{manager::MiningManagerError, mempool::RuleError}; -use std::{ops::Mul, sync::Arc}; +use std::sync::Arc; use tokio::sync::mpsc::UnboundedSender; pub struct MiningManager { @@ -1044,29 +1044,28 @@ fn feerate_stats(transactions: Vec, calculated_fees: Vec) -> O return None; } if transactions.len() != calculated_fees.len() + 1 { - error!("Transactions length must be one more than `calculated_fees` length"); + error!( + "[feerate_stats] block template transactions length ({}) is expected to be one more than `calculated_fees` length ({})", + transactions.len(), + calculated_fees.len() + ); return None; } debug_assert!(transactions[0].is_coinbase()); - let mut fees_and_masses = calculated_fees + let mut feerates = calculated_fees .into_iter() .zip(transactions .iter() // skip coinbase tx .skip(1) .map(Transaction::mass)) + .map(|(fee, mass)| fee as f64 / mass as f64) .collect_vec(); + feerates.sort_unstable_by(f64::total_cmp); - // Sort by fee rate without performing division for each comparison. - // Using multiplication instead of division is faster and avoids the need - // to convert all values to floats. Division is only performed later when - // calculating the min, max, and median fee rates. - fees_and_masses.sort_unstable_by(|(lhs_fee, lhs_mass), (rhs_fee, rhs_mass)| lhs_fee.mul(rhs_mass).cmp(&rhs_fee.mul(lhs_mass))); - - let div_as_f64 = |(fee, mass)| fee as f64 / mass as f64; - let max = div_as_f64(fees_and_masses[fees_and_masses.len() - 1]); - let min = div_as_f64(fees_and_masses[0]); - let median = div_as_f64(fees_and_masses[fees_and_masses.len() / 2]); + let max = feerates[feerates.len() - 1]; + let min = feerates[0]; + let median = feerates[feerates.len() / 2]; Some(Stats { max, median, min }) } diff --git a/simpa/Cargo.toml b/simpa/Cargo.toml index 97db6b1..7ee240d 100644 --- a/simpa/Cargo.toml +++ b/simpa/Cargo.toml @@ -22,6 +22,7 @@ spectre-perf-monitor.workspace = true spectre-utils.workspace = true async-channel.workspace = true +cfg-if.workspace = true clap.workspace = true dhat = { workspace = true, optional = true } futures-util.workspace = true @@ -38,3 +39,4 @@ tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } [features] heap = ["dhat", "spectre-alloc/heap"] +semaphore-trace = ["spectre-utils/semaphore-trace"] diff --git a/simpa/src/main.rs b/simpa/src/main.rs index 4ac225d..4e4abe1 100644 --- a/simpa/src/main.rs +++ b/simpa/src/main.rs @@ -21,7 +21,12 @@ use spectre_consensus_core::{ BlockHashSet, BlockLevel, HashMapCustomHasher, }; use spectre_consensus_notify::root::ConsensusNotificationRoot; -use spectre_core::{info, task::service::AsyncService, task::tick::TickService, time::unix_now, trace, warn}; +use spectre_core::{ + info, + task::{service::AsyncService, tick::TickService}, + time::unix_now, + trace, warn, +}; use spectre_database::prelude::ConnBuilder; use spectre_database::{create_temp_db, load_existing_db}; use spectre_hashes::Hash; @@ -133,7 +138,13 @@ fn main() { let args = Args::parse(); // Initialize the logger - spectre_core::log::init_logger(None, &args.log_level); + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + spectre_core::log::init_logger(None, &format!("{},{}=debug", args.log_level, spectre_utils::sync::semaphore_module_path())); + } else { + spectre_core::log::init_logger(None, &args.log_level); + } + }; // Configure the panic behavior // As we log the panic, we want to set it up after the logger diff --git a/spectred/Cargo.toml b/spectred/Cargo.toml index 177c06e..0c56efc 100644 --- a/spectred/Cargo.toml +++ b/spectred/Cargo.toml @@ -41,22 +41,25 @@ spectre-utxoindex.workspace = true spectre-wrpc-server.workspace = true async-channel.workspace = true +cfg-if.workspace = true clap.workspace = true dhat = { workspace = true, optional = true } -serde.workspace = true dirs.workspace = true futures-util.workspace = true log.workspace = true num_cpus.workspace = true rand.workspace = true rayon.workspace = true +serde.workspace = true tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["rt", "macros", "rt-multi-thread"] } workflow-log.workspace = true + toml = "0.8.10" serde_with = "3.7.0" [features] heap = ["dhat", "spectre-alloc/heap"] devnet-prealloc = ["spectre-consensus/devnet-prealloc"] +semaphore-trace = ["spectre-utils/semaphore-trace"] diff --git a/spectred/src/daemon.rs b/spectred/src/daemon.rs index cc47487..fe0ca87 100644 --- a/spectred/src/daemon.rs +++ b/spectred/src/daemon.rs @@ -161,7 +161,13 @@ impl Runtime { let log_dir = get_log_dir(args); // Initialize the logger - spectre_core::log::init_logger(log_dir.as_deref(), &args.log_level); + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + spectre_core::log::init_logger(log_dir.as_deref(), &format!("{},{}=debug", args.log_level, spectre_utils::sync::semaphore_module_path())); + } else { + spectre_core::log::init_logger(log_dir.as_deref(), &args.log_level); + } + }; // Configure the panic behavior // As we log the panic, we want to set it up after the logger diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 63c4077..2c49879 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -11,7 +11,6 @@ repository.workspace = true [dependencies] arc-swap.workspace = true -parking_lot.workspace = true async-channel.workspace = true borsh.workspace = true cfg-if.workspace = true @@ -19,12 +18,14 @@ event-listener.workspace = true faster-hex.workspace = true ipnet.workspace = true itertools.workspace = true +log.workspace = true +once_cell.workspace = true +parking_lot.workspace = true serde.workspace = true smallvec.workspace = true thiserror.workspace = true triggered.workspace = true uuid.workspace = true -log.workspace = true wasm-bindgen.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] @@ -42,3 +43,6 @@ rand.workspace = true [[bench]] name = "bench" harness = false + +[features] +semaphore-trace = [] diff --git a/utils/src/sync/mod.rs b/utils/src/sync/mod.rs index 40fb147..14afe79 100644 --- a/utils/src/sync/mod.rs +++ b/utils/src/sync/mod.rs @@ -1,2 +1,7 @@ pub mod rwlock; pub(crate) mod semaphore; + +#[cfg(feature = "semaphore-trace")] +pub fn semaphore_module_path() -> &'static str { + semaphore::get_module_path() +} diff --git a/utils/src/sync/semaphore.rs b/utils/src/sync/semaphore.rs index 4b94e8f..2ea6dcc 100644 --- a/utils/src/sync/semaphore.rs +++ b/utils/src/sync/semaphore.rs @@ -4,6 +4,64 @@ use std::{ time::Duration, }; +#[cfg(feature = "semaphore-trace")] +mod trace { + use super::*; + use log::debug; + use once_cell::sync::Lazy; + use std::sync::atomic::AtomicU64; + use std::time::SystemTime; + + static SYS_START: Lazy = Lazy::new(SystemTime::now); + + #[inline] + pub(super) fn sys_now() -> u64 { + SystemTime::now().duration_since(*SYS_START).unwrap_or_default().as_micros() as u64 + } + + #[derive(Debug, Default)] + pub struct TraceInner { + readers_start: AtomicU64, + readers_time: AtomicU64, + log_time: AtomicU64, + log_value: AtomicU64, + } + + impl TraceInner { + pub(super) fn mark_readers_start(&self) { + self.readers_start.store(sys_now(), Ordering::Relaxed); + } + + pub(super) fn mark_readers_end(&self) { + let start = self.readers_start.load(Ordering::Relaxed); + let now = sys_now(); + if start < now { + let readers_time = self.readers_time.fetch_add(now - start, Ordering::Relaxed) + now - start; + let log_time = self.log_time.load(Ordering::Relaxed); + if log_time + (Duration::from_secs(10).as_micros() as u64) < now { + let log_value = self.log_value.load(Ordering::Relaxed); + debug!( + "Semaphore: log interval: {:?}, readers time: {:?}, fraction: {:.2}", + Duration::from_micros(now - log_time), + Duration::from_micros(readers_time - log_value), + (readers_time - log_value) as f64 / (now - log_time) as f64 + ); + self.log_value.store(readers_time, Ordering::Relaxed); + self.log_time.store(now, Ordering::Relaxed); + } + } + } + } +} + +#[cfg(feature = "semaphore-trace")] +use trace::*; + +#[cfg(feature = "semaphore-trace")] +pub(crate) fn get_module_path() -> &'static str { + module_path!() +} + /// A low-level non-fair semaphore. The semaphore is non-fair in the sense that clients acquiring /// a lower number of permits might get their allocation before earlier clients which requested more /// permits -- if the semaphore can provide the lower allocation but not the larger. This non-fairness @@ -15,13 +73,28 @@ use std::{ pub(crate) struct Semaphore { counter: AtomicUsize, signal: Event, + #[cfg(feature = "semaphore-trace")] + trace_inner: TraceInner, } impl Semaphore { pub const MAX_PERMITS: usize = usize::MAX; - pub const fn new(available_permits: usize) -> Semaphore { - Semaphore { counter: AtomicUsize::new(available_permits), signal: Event::new() } + pub fn new(available_permits: usize) -> Semaphore { + cfg_if::cfg_if! { + if #[cfg(feature = "semaphore-trace")] { + Semaphore { + counter: AtomicUsize::new(available_permits), + signal: Event::new(), + trace_inner: Default::default(), + } + } else { + Semaphore { + counter: AtomicUsize::new(available_permits), + signal: Event::new(), + } + } + } } /// Tries to acquire `permits` slots from the semaphore. Upon success, returns the acquired slot @@ -33,7 +106,14 @@ impl Semaphore { } match self.counter.compare_exchange_weak(count, count - permits, Ordering::AcqRel, Ordering::Acquire) { - Ok(_) => return Some(count), + Ok(_) => { + #[cfg(feature = "semaphore-trace")] + if permits == 1 && count == Self::MAX_PERMITS { + // permits == 1 indicates a reader, count == Self::MAX_PERMITS indicates it is the first reader + self.trace_inner.mark_readers_start(); + } + return Some(count); + } Err(c) => count = c, } } @@ -75,6 +155,12 @@ impl Semaphore { /// Returns the released slot pub fn release(&self, permits: usize) -> usize { let slot = self.counter.fetch_add(permits, Ordering::AcqRel) + permits; + + #[cfg(feature = "semaphore-trace")] + if permits == 1 && slot == Self::MAX_PERMITS { + // permits == 1 indicates a reader, slot == Self::MAX_PERMITS indicates it is the last reader + self.trace_inner.mark_readers_end(); + } self.signal.notify(permits); slot } From 6b46dc664dcc824e538b8798e3a164a863d0bf42 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 13:44:42 +0200 Subject: [PATCH 12/48] TN11 bug fix: activate mass hashing when modifying a cached block template * fix transaction hash merkle root calculation in modify block template to consider storage mass activation * avoid similar future errors: expose only a single calc_hash_merkle_root function with `include_mass_field` arg and update all test usages explicitly * move subnet checks to inner location * reorganize cache mem estimators --- consensus/core/src/api/mod.rs | 4 + consensus/core/src/block.rs | 15 + consensus/core/src/config/genesis.rs | 2 +- consensus/core/src/errors/tx.rs | 4 + consensus/core/src/header.rs | 7 + consensus/core/src/merkle.rs | 8 +- consensus/core/src/subnets.rs | 257 +++++++++--------- consensus/core/src/tx.rs | 17 ++ consensus/core/src/utxo/utxo_diff.rs | 2 +- consensus/src/consensus/mod.rs | 6 + consensus/src/consensus/test_consensus.rs | 2 +- consensus/src/model/stores/headers.rs | 4 +- .../body_validation_in_context.rs | 6 +- .../body_validation_in_isolation.rs | 10 +- .../pipeline/virtual_processor/processor.rs | 4 +- .../tx_validation_in_isolation.rs | 15 +- mining/src/block_template/builder.rs | 4 +- mining/src/testutils/consensus_mock.rs | 8 +- protocol/flows/src/flowcontext/orphans.rs | 2 +- 19 files changed, 220 insertions(+), 157 deletions(-) diff --git a/consensus/core/src/api/mod.rs b/consensus/core/src/api/mod.rs index 99ae2be..85d9d84 100644 --- a/consensus/core/src/api/mod.rs +++ b/consensus/core/src/api/mod.rs @@ -190,6 +190,10 @@ pub trait ConsensusApi: Send + Sync { unimplemented!() } + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], pov_daa_score: u64) -> Hash { + unimplemented!() + } + fn validate_pruning_proof(&self, proof: &PruningPointProof) -> PruningImportResult<()> { unimplemented!() } diff --git a/consensus/core/src/block.rs b/consensus/core/src/block.rs index a646e9c..f2f5022 100644 --- a/consensus/core/src/block.rs +++ b/consensus/core/src/block.rs @@ -5,6 +5,7 @@ use crate::{ BlueWorkType, }; use spectre_hashes::Hash; +use spectre_utils::mem_size::MemSizeEstimator; use std::sync::Arc; /// A mutable block structure where header and transactions within can still be mutated. @@ -66,6 +67,20 @@ impl Block { pub fn from_precomputed_hash(hash: Hash, parents: Vec) -> Block { Block::from_header(Header::from_precomputed_hash(hash, parents)) } + + pub fn asses_for_cache(&self) -> Option<()> { + (self.estimate_mem_bytes() < 1_000_000).then_some(()) + } +} + +impl MemSizeEstimator for Block { + fn estimate_mem_bytes(&self) -> usize { + // Calculates mem bytes of the block (for cache tracking purposes) + size_of::() + + self.header.estimate_mem_bytes() + + size_of::>() + + self.transactions.iter().map(Transaction::estimate_mem_bytes).sum::() + } } /// An abstraction for a recallable transaction selector with persistent state diff --git a/consensus/core/src/config/genesis.rs b/consensus/core/src/config/genesis.rs index f252fbc..2de02d6 100644 --- a/consensus/core/src/config/genesis.rs +++ b/consensus/core/src/config/genesis.rs @@ -199,7 +199,7 @@ mod tests { fn test_genesis_hashes() { [GENESIS, TESTNET_GENESIS, TESTNET11_GENESIS, SIMNET_GENESIS, DEVNET_GENESIS].into_iter().for_each(|genesis| { let block: Block = (&genesis).into(); - assert_hashes_eq(calc_hash_merkle_root(block.transactions.iter()), block.header.hash_merkle_root); + assert_hashes_eq(calc_hash_merkle_root(block.transactions.iter(), false), block.header.hash_merkle_root); assert_hashes_eq(block.hash(), genesis.hash); }); } diff --git a/consensus/core/src/errors/tx.rs b/consensus/core/src/errors/tx.rs index 25e215e..84a10ed 100644 --- a/consensus/core/src/errors/tx.rs +++ b/consensus/core/src/errors/tx.rs @@ -1,4 +1,5 @@ use crate::constants::MAX_SOMPI; +use crate::subnets::SubnetworkId; use crate::tx::TransactionOutpoint; use spectre_txscript_errors::TxScriptError; use thiserror::Error; @@ -92,6 +93,9 @@ pub enum TxRuleError { #[error("calculated contextual mass (including storage mass) {0} is not equal to the committed mass field {1}")] WrongMass(u64, u64), + #[error("transaction subnetwork id {0} is neither native nor coinbase")] + SubnetworksDisabled(SubnetworkId), + /// [`TxRuleError::FeerateTooLow`] is not a consensus error but a mempool error triggered by the /// fee/mass RBF validation rule #[error("fee rate per contextual mass gram is not greater than the fee rate of the replaced transaction")] diff --git a/consensus/core/src/header.rs b/consensus/core/src/header.rs index 35dd5aa..61ed2e8 100644 --- a/consensus/core/src/header.rs +++ b/consensus/core/src/header.rs @@ -2,6 +2,7 @@ use crate::{hashing, BlueWorkType}; use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; use spectre_hashes::Hash; +use spectre_utils::mem_size::MemSizeEstimator; /// @category Consensus #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] @@ -92,6 +93,12 @@ impl Header { } } +impl MemSizeEstimator for Header { + fn estimate_mem_bytes(&self) -> usize { + size_of::() + self.parents_by_level.iter().map(|l| l.len()).sum::() * size_of::() + } +} + #[cfg(test)] mod tests { use super::*; diff --git a/consensus/core/src/merkle.rs b/consensus/core/src/merkle.rs index 7de2221..efb9c0f 100644 --- a/consensus/core/src/merkle.rs +++ b/consensus/core/src/merkle.rs @@ -2,14 +2,10 @@ use crate::{hashing, tx::Transaction}; use spectre_hashes::Hash; use spectre_merkle::calc_merkle_root; -pub fn calc_hash_merkle_root_with_options<'a>(txs: impl ExactSizeIterator, include_mass_field: bool) -> Hash { +pub fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator, include_mass_field: bool) -> Hash { calc_merkle_root(txs.map(|tx| hashing::tx::hash(tx, include_mass_field))) } -pub fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { - calc_merkle_root(txs.map(|tx| hashing::tx::hash(tx, false))) -} - #[cfg(test)] mod tests { use crate::merkle::calc_hash_merkle_root; @@ -242,7 +238,7 @@ mod tests { ), ]; assert_eq!( - calc_hash_merkle_root(txs.iter()), + calc_hash_merkle_root(txs.iter(), false), Hash::from_slice(&[ 0x46, 0xec, 0xf4, 0x5b, 0xe3, 0xba, 0xca, 0x34, 0x9d, 0xfe, 0x8a, 0x78, 0xde, 0xaf, 0x05, 0x3b, 0x0a, 0xa6, 0xd5, 0x38, 0x97, 0x4d, 0xa5, 0x0f, 0xd6, 0xef, 0xb4, 0xd2, 0x66, 0xbc, 0x8d, 0x21, diff --git a/consensus/core/src/subnets.rs b/consensus/core/src/subnets.rs index a3b9152..b99b07b 100644 --- a/consensus/core/src/subnets.rs +++ b/consensus/core/src/subnets.rs @@ -1,76 +1,79 @@ -use std::fmt::{Debug, Display, Formatter}; -use std::str::{self, FromStr}; - -use borsh::{BorshDeserialize, BorshSerialize}; -use spectre_utils::hex::{FromHex, ToHex}; -use spectre_utils::{serde_impl_deser_fixed_bytes_ref, serde_impl_ser_fixed_bytes_ref}; +use std::fmt::{Debug, Display, Formatter}; +use std::str::{self, FromStr}; + +use borsh::{BorshDeserialize, BorshSerialize}; +use spectre_utils::hex::{FromHex, ToHex}; +use spectre_utils::{serde_impl_deser_fixed_bytes_ref, serde_impl_ser_fixed_bytes_ref}; use thiserror::Error; - -/// The size of the array used to store subnetwork IDs. -pub const SUBNETWORK_ID_SIZE: usize = 20; - -/// The domain representation of a Subnetwork ID -#[derive(Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize)] -pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); - -impl Debug for SubnetworkId { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - f.debug_struct("SubnetworkId").field("", &self.to_hex()).finish() - } -} - -serde_impl_ser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); -serde_impl_deser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); - -impl AsRef<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { - fn as_ref(&self) -> &[u8; SUBNETWORK_ID_SIZE] { - &self.0 - } -} - -impl AsRef<[u8]> for SubnetworkId { - fn as_ref(&self) -> &[u8] { - &self.0 - } -} - -impl From<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { - fn from(value: [u8; SUBNETWORK_ID_SIZE]) -> Self { - Self::from_bytes(value) - } -} - -impl SubnetworkId { - pub const fn from_byte(b: u8) -> SubnetworkId { - let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; - bytes[0] = b; - SubnetworkId(bytes) - } - - pub const fn from_bytes(bytes: [u8; SUBNETWORK_ID_SIZE]) -> SubnetworkId { - SubnetworkId(bytes) - } - - /// Returns true if the subnetwork is a built-in subnetwork, which - /// means all nodes, including partial nodes, must validate it, and its transactions - /// always use 0 gas. - #[inline] - pub fn is_builtin(&self) -> bool { - *self == SUBNETWORK_ID_COINBASE || *self == SUBNETWORK_ID_REGISTRY - } - - /// Returns true if the subnetwork is the native or a built-in subnetwork - #[inline] - pub fn is_builtin_or_native(&self) -> bool { - *self == SUBNETWORK_ID_NATIVE || self.is_builtin() - } -} - + +/// The size of the array used to store subnetwork IDs. +pub const SUBNETWORK_ID_SIZE: usize = 20; + +/// The domain representation of a Subnetwork ID +#[derive(Clone, Default, Eq, PartialEq, Ord, PartialOrd, Hash, BorshSerialize, BorshDeserialize)] +pub struct SubnetworkId([u8; SUBNETWORK_ID_SIZE]); + +impl Debug for SubnetworkId { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SubnetworkId").field("", &self.to_hex()).finish() + } +} + +serde_impl_ser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); +serde_impl_deser_fixed_bytes_ref!(SubnetworkId, SUBNETWORK_ID_SIZE); + +impl AsRef<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { + fn as_ref(&self) -> &[u8; SUBNETWORK_ID_SIZE] { + &self.0 + } +} + +impl AsRef<[u8]> for SubnetworkId { + fn as_ref(&self) -> &[u8] { + &self.0 + } +} + +impl From<[u8; SUBNETWORK_ID_SIZE]> for SubnetworkId { + fn from(value: [u8; SUBNETWORK_ID_SIZE]) -> Self { + Self::from_bytes(value) + } +} + +impl SubnetworkId { + pub const fn from_byte(b: u8) -> SubnetworkId { + let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; + bytes[0] = b; + SubnetworkId(bytes) + } + + pub const fn from_bytes(bytes: [u8; SUBNETWORK_ID_SIZE]) -> SubnetworkId { + SubnetworkId(bytes) + } + + /// Returns true if the subnetwork is a built-in subnetwork, which + /// means all nodes, including partial nodes, must validate it, and its transactions + /// always use 0 gas. + #[inline] + pub fn is_builtin(&self) -> bool { + *self == SUBNETWORK_ID_COINBASE || *self == SUBNETWORK_ID_REGISTRY + } + + /// Returns true if the subnetwork is the native subnetwork + #[inline] + pub fn is_native(&self) -> bool { + *self == SUBNETWORK_ID_NATIVE + } + + /// Returns true if the subnetwork is the native or a built-in subnetwork + #[inline] + pub fn is_builtin_or_native(&self) -> bool { + self.is_native() || self.is_builtin() + } +} + #[derive(Error, Debug, Clone)] pub enum SubnetworkConversionError { - #[error("Invalid bytes")] - InvalidBytes, - #[error(transparent)] SliceError(#[from] std::array::TryFromSliceError), @@ -78,69 +81,57 @@ pub enum SubnetworkConversionError { HexError(#[from] faster_hex::Error), } -impl TryFrom<&[u8]> for SubnetworkId { +impl TryFrom<&[u8]> for SubnetworkId { type Error = SubnetworkConversionError; - - fn try_from(value: &[u8]) -> Result { - let bytes = <[u8; SUBNETWORK_ID_SIZE]>::try_from(value)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Error::InvalidBytes) - } else { - Ok(Self(bytes)) - } - } -} - -impl Display for SubnetworkId { - #[inline] - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - let mut hex = [0u8; SUBNETWORK_ID_SIZE * 2]; - faster_hex::hex_encode(&self.0, &mut hex).expect("The output is exactly twice the size of the input"); - f.write_str(str::from_utf8(&hex).expect("hex is always valid UTF-8")) - } -} - -impl ToHex for SubnetworkId { - fn to_hex(&self) -> String { - let mut hex = [0u8; SUBNETWORK_ID_SIZE * 2]; - faster_hex::hex_encode(&self.0, &mut hex).expect("The output is exactly twice the size of the input"); - str::from_utf8(&hex).expect("hex is always valid UTF-8").to_string() - } -} - -impl FromStr for SubnetworkId { + + fn try_from(value: &[u8]) -> Result { + let bytes = <[u8; SUBNETWORK_ID_SIZE]>::try_from(value)?; + Ok(Self(bytes)) + } +} + +impl Display for SubnetworkId { + #[inline] + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + let mut hex = [0u8; SUBNETWORK_ID_SIZE * 2]; + faster_hex::hex_encode(&self.0, &mut hex).expect("The output is exactly twice the size of the input"); + f.write_str(str::from_utf8(&hex).expect("hex is always valid UTF-8")) + } +} + +impl ToHex for SubnetworkId { + fn to_hex(&self) -> String { + let mut hex = [0u8; SUBNETWORK_ID_SIZE * 2]; + faster_hex::hex_encode(&self.0, &mut hex).expect("The output is exactly twice the size of the input"); + str::from_utf8(&hex).expect("hex is always valid UTF-8").to_string() + } +} + +impl FromStr for SubnetworkId { type Err = SubnetworkConversionError; - - #[inline] - fn from_str(hex_str: &str) -> Result { - let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; - faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Err::InvalidBytes) - } else { - Ok(Self(bytes)) - } - } -} - -impl FromHex for SubnetworkId { + + #[inline] + fn from_str(hex_str: &str) -> Result { + let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; + faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; + Ok(Self(bytes)) + } +} + +impl FromHex for SubnetworkId { type Error = SubnetworkConversionError; - fn from_hex(hex_str: &str) -> Result { - let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; - faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; - if bytes != Self::from_byte(0).0 && bytes != Self::from_byte(1).0 { - Err(Self::Error::InvalidBytes) - } else { - Ok(Self(bytes)) - } - } -} - -/// The default subnetwork ID which is used for transactions without related payload data -pub const SUBNETWORK_ID_NATIVE: SubnetworkId = SubnetworkId::from_byte(0); - -/// The subnetwork ID which is used for the coinbase transaction -pub const SUBNETWORK_ID_COINBASE: SubnetworkId = SubnetworkId::from_byte(1); - -/// The subnetwork ID which is used for adding new sub networks to the registry -pub const SUBNETWORK_ID_REGISTRY: SubnetworkId = SubnetworkId::from_byte(2); + fn from_hex(hex_str: &str) -> Result { + let mut bytes = [0u8; SUBNETWORK_ID_SIZE]; + faster_hex::hex_decode(hex_str.as_bytes(), &mut bytes)?; + Ok(Self(bytes)) + } +} + +/// The default subnetwork ID which is used for transactions without related payload data +pub const SUBNETWORK_ID_NATIVE: SubnetworkId = SubnetworkId::from_byte(0); + +/// The subnetwork ID which is used for the coinbase transaction +pub const SUBNETWORK_ID_COINBASE: SubnetworkId = SubnetworkId::from_byte(1); + +/// The subnetwork ID which is used for adding new sub networks to the registry +pub const SUBNETWORK_ID_REGISTRY: SubnetworkId = SubnetworkId::from_byte(2); diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 64fd83f..69f9dd2 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -230,6 +230,23 @@ impl Transaction { } } +impl MemSizeEstimator for Transaction { + fn estimate_mem_bytes(&self) -> usize { + // Calculates mem bytes of the transaction (for cache tracking purposes) + size_of::() + + self.payload.len() + + self + .inputs + .iter() + .map(|i| i.signature_script.len() + size_of::()) + .chain(self.outputs.iter().map(|o| { + // size_of::() already counts SCRIPT_VECTOR_SIZE bytes within, so we only add the delta + o.script_public_key.script().len().saturating_sub(SCRIPT_VECTOR_SIZE) + size_of::() + })) + .sum::() + } +} + /// Represents any kind of transaction which has populated UTXO entry data and can be verified/signed etc pub trait VerifiableTransaction { fn tx(&self) -> &Transaction; diff --git a/consensus/core/src/utxo/utxo_diff.rs b/consensus/core/src/utxo/utxo_diff.rs index 8d507c0..aafd1ce 100644 --- a/consensus/core/src/utxo/utxo_diff.rs +++ b/consensus/core/src/utxo/utxo_diff.rs @@ -5,7 +5,7 @@ use super::{ use crate::tx::{TransactionOutpoint, UtxoEntry, VerifiableTransaction}; use serde::{Deserialize, Serialize}; use spectre_utils::mem_size::MemSizeEstimator; -use std::{collections::hash_map::Entry::Vacant, mem::size_of}; +use std::collections::hash_map::Entry::Vacant; pub trait ImmutableUtxoDiff { fn added(&self) -> &UtxoCollection; diff --git a/consensus/src/consensus/mod.rs b/consensus/src/consensus/mod.rs index 328b856..55611ff 100644 --- a/consensus/src/consensus/mod.rs +++ b/consensus/src/consensus/mod.rs @@ -58,6 +58,7 @@ use spectre_consensus_core::{ tx::TxResult, }, header::Header, + merkle::calc_hash_merkle_root, muhash::MuHashExtensions, network::NetworkType, pruning::{PruningPointProof, PruningPointTrustedData, PruningPointsList}, @@ -676,6 +677,11 @@ impl ConsensusApi for Consensus { self.services.coinbase_manager.modify_coinbase_payload(payload, miner_data) } + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], pov_daa_score: u64) -> Hash { + let storage_mass_activated = pov_daa_score > self.config.storage_mass_activation_daa_score; + calc_hash_merkle_root(txs.iter(), storage_mass_activated) + } + fn validate_pruning_proof(&self, proof: &PruningPointProof) -> Result<(), PruningImportError> { self.services.pruning_proof_manager.validate_pruning_point_proof(proof) } diff --git a/consensus/src/consensus/test_consensus.rs b/consensus/src/consensus/test_consensus.rs index 1505b03..a50fd57 100644 --- a/consensus/src/consensus/test_consensus.rs +++ b/consensus/src/consensus/test_consensus.rs @@ -190,7 +190,7 @@ impl TestConsensus { let cb = Transaction::new(TX_VERSION, vec![], vec![], 0, SUBNETWORK_ID_COINBASE, 0, cb_payload); txs.insert(0, cb); - header.hash_merkle_root = calc_hash_merkle_root(txs.iter()); + header.hash_merkle_root = calc_hash_merkle_root(txs.iter(), false); MutableBlock::new(header, txs) } diff --git a/consensus/src/model/stores/headers.rs b/consensus/src/model/stores/headers.rs index ec15684..9834c37 100644 --- a/consensus/src/model/stores/headers.rs +++ b/consensus/src/model/stores/headers.rs @@ -29,9 +29,7 @@ pub struct HeaderWithBlockLevel { impl MemSizeEstimator for HeaderWithBlockLevel { fn estimate_mem_bytes(&self) -> usize { - size_of::
() - + self.header.parents_by_level.iter().map(|l| l.len()).sum::() * size_of::() - + size_of::() + self.header.as_ref().estimate_mem_bytes() + size_of::() } } diff --git a/consensus/src/pipeline/body_processor/body_validation_in_context.rs b/consensus/src/pipeline/body_processor/body_validation_in_context.rs index 75061db..e7e9e63 100644 --- a/consensus/src/pipeline/body_processor/body_validation_in_context.rs +++ b/consensus/src/pipeline/body_processor/body_validation_in_context.rs @@ -108,13 +108,17 @@ mod tests { }; use spectre_consensus_core::{ api::ConsensusApi, - merkle::calc_hash_merkle_root, + merkle::calc_hash_merkle_root as calc_hash_merkle_root_with_options, subnets::SUBNETWORK_ID_NATIVE, tx::{Transaction, TransactionInput, TransactionOutpoint}, }; use spectre_core::assert_match; use spectre_hashes::Hash; + fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { + calc_hash_merkle_root_with_options(txs, false) + } + #[tokio::test] async fn validate_body_in_context_test() { let config = ConfigBuilder::new(DEVNET_PARAMS) diff --git a/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs b/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs index def5613..360a085 100644 --- a/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs +++ b/consensus/src/pipeline/body_processor/body_validation_in_isolation.rs @@ -2,7 +2,7 @@ use std::{collections::HashSet, sync::Arc}; use super::BlockBodyProcessor; use crate::errors::{BlockProcessResult, RuleError}; -use spectre_consensus_core::{block::Block, merkle::calc_hash_merkle_root_with_options, tx::TransactionOutpoint}; +use spectre_consensus_core::{block::Block, merkle::calc_hash_merkle_root, tx::TransactionOutpoint}; impl BlockBodyProcessor { pub fn validate_body_in_isolation(self: &Arc, block: &Block) -> BlockProcessResult { @@ -29,7 +29,7 @@ impl BlockBodyProcessor { } fn check_hash_merkle_root(block: &Block, storage_mass_activated: bool) -> BlockProcessResult<()> { - let calculated = calc_hash_merkle_root_with_options(block.transactions.iter(), storage_mass_activated); + let calculated = calc_hash_merkle_root(block.transactions.iter(), storage_mass_activated); if calculated != block.header.hash_merkle_root { return Err(RuleError::BadMerkleRoot(block.header.hash_merkle_root, calculated)); } @@ -137,13 +137,17 @@ mod tests { api::{BlockValidationFutures, ConsensusApi}, block::MutableBlock, header::Header, - merkle::calc_hash_merkle_root, + merkle::calc_hash_merkle_root as calc_hash_merkle_root_with_options, subnets::{SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, tx::{scriptvec, ScriptPublicKey, Transaction, TransactionId, TransactionInput, TransactionOutpoint, TransactionOutput}, }; use spectre_core::assert_match; use spectre_hashes::Hash; + fn calc_hash_merkle_root<'a>(txs: impl ExactSizeIterator) -> Hash { + calc_hash_merkle_root_with_options(txs, false) + } + #[test] fn validate_body_in_isolation_test() { let consensus = TestConsensus::new(&Config::new(MAINNET_PARAMS)); diff --git a/consensus/src/pipeline/virtual_processor/processor.rs b/consensus/src/pipeline/virtual_processor/processor.rs index cb44206..4712408 100644 --- a/consensus/src/pipeline/virtual_processor/processor.rs +++ b/consensus/src/pipeline/virtual_processor/processor.rs @@ -54,7 +54,7 @@ use spectre_consensus_core::{ coinbase::MinerData, config::genesis::GenesisBlock, header::Header, - merkle::calc_hash_merkle_root_with_options, + merkle::calc_hash_merkle_root, pruning::PruningPointsList, tx::{MutableTransaction, Transaction}, utxo::{ @@ -975,7 +975,7 @@ impl VirtualStateProcessor { // Hash according to hardfork activation let storage_mass_activated = virtual_state.daa_score > self.storage_mass_activation_daa_score; - let hash_merkle_root = calc_hash_merkle_root_with_options(txs.iter(), storage_mass_activated); + let hash_merkle_root = calc_hash_merkle_root(txs.iter(), storage_mass_activated); let accepted_id_merkle_root = spectre_merkle::calc_merkle_root(virtual_state.accepted_tx_ids.iter().copied()); let utxo_commitment = virtual_state.multiset.clone().finalize(); diff --git a/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs b/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs index ea30f6b..37fd0bd 100644 --- a/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs +++ b/consensus/src/processes/transaction_validator/tx_validation_in_isolation.rs @@ -17,6 +17,7 @@ impl TransactionValidator { check_duplicate_transaction_inputs(tx)?; check_gas(tx)?; check_transaction_payload(tx)?; + check_transaction_subnetwork(tx)?; check_transaction_version(tx) } @@ -146,10 +147,18 @@ fn check_transaction_output_value_ranges(tx: &Transaction) -> TxResult<()> { Ok(()) } +fn check_transaction_subnetwork(tx: &Transaction) -> TxResult<()> { + if tx.is_coinbase() || tx.subnetwork_id.is_native() { + Ok(()) + } else { + Err(TxRuleError::SubnetworksDisabled(tx.subnetwork_id.clone())) + } +} + #[cfg(test)] mod tests { use spectre_consensus_core::{ - subnets::{SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, + subnets::{SubnetworkId, SUBNETWORK_ID_COINBASE, SUBNETWORK_ID_NATIVE}, tx::{scriptvec, ScriptPublicKey, Transaction, TransactionId, TransactionInput, TransactionOutpoint, TransactionOutput}, }; use spectre_core::assert_match; @@ -261,6 +270,10 @@ mod tests { tv.validate_tx_in_isolation(&valid_tx).unwrap(); + let mut tx: Transaction = valid_tx.clone(); + tx.subnetwork_id = SubnetworkId::from_byte(3); + assert_match!(tv.validate_tx_in_isolation(&tx), Err(TxRuleError::SubnetworksDisabled(_))); + let mut tx = valid_tx.clone(); tx.inputs = vec![]; assert_match!(tv.validate_tx_in_isolation(&tx), Err(TxRuleError::NoTxInputs)); diff --git a/mining/src/block_template/builder.rs b/mining/src/block_template/builder.rs index fe44197..d87b46f 100644 --- a/mining/src/block_template/builder.rs +++ b/mining/src/block_template/builder.rs @@ -3,7 +3,6 @@ use spectre_consensus_core::{ api::ConsensusApi, block::{BlockTemplate, TemplateBuildMode, TemplateTransactionSelector}, coinbase::MinerData, - merkle::calc_hash_merkle_root, tx::COINBASE_TRANSACTION_INDEX, }; use spectre_core::time::{unix_now, Stopwatch}; @@ -106,7 +105,8 @@ impl BlockTemplateBuilder { coinbase_tx.outputs.last_mut().unwrap().script_public_key = new_miner_data.script_public_key.clone(); } // Update the hash merkle root according to the modified transactions - block_template.block.header.hash_merkle_root = calc_hash_merkle_root(block_template.block.transactions.iter()); + block_template.block.header.hash_merkle_root = + consensus.calc_transaction_hash_merkle_root(&block_template.block.transactions, block_template.block.header.daa_score); let new_timestamp = unix_now(); if new_timestamp > block_template.block.header.timestamp { // Only if new time stamp is later than current, update the header. Otherwise, diff --git a/mining/src/testutils/consensus_mock.rs b/mining/src/testutils/consensus_mock.rs index 33fe3b8..7521b36 100644 --- a/mining/src/testutils/consensus_mock.rs +++ b/mining/src/testutils/consensus_mock.rs @@ -19,7 +19,7 @@ use spectre_consensus_core::{ utxo::utxo_collection::UtxoCollection, }; use spectre_core::time::unix_now; -use spectre_hashes::ZERO_HASH; +use spectre_hashes::{Hash, ZERO_HASH}; use parking_lot::RwLock; use std::{collections::HashMap, sync::Arc}; @@ -86,7 +86,7 @@ impl ConsensusApi for ConsensusMock { let coinbase = coinbase_manager.expected_coinbase_transaction(miner_data.clone()); txs.insert(0, coinbase.tx); let now = unix_now(); - let hash_merkle_root = calc_hash_merkle_root(txs.iter()); + let hash_merkle_root = self.calc_transaction_hash_merkle_root(&txs, 0); let header = Header::new_finalized( BLOCK_VERSION, vec![], @@ -177,4 +177,8 @@ impl ConsensusApi for ConsensusMock { let coinbase_manager = CoinbaseManagerMock::new(); Ok(coinbase_manager.modify_coinbase_payload(payload, miner_data)) } + + fn calc_transaction_hash_merkle_root(&self, txs: &[Transaction], _pov_daa_score: u64) -> Hash { + calc_hash_merkle_root(txs.iter(), false) + } } diff --git a/protocol/flows/src/flowcontext/orphans.rs b/protocol/flows/src/flowcontext/orphans.rs index 232d148..6e067b0 100644 --- a/protocol/flows/src/flowcontext/orphans.rs +++ b/protocol/flows/src/flowcontext/orphans.rs @@ -78,7 +78,7 @@ impl OrphanBlocksPool { if self.orphans.contains_key(&orphan_hash) { return None; } - + orphan_block.asses_for_cache()?; let (roots, orphan_ancestors) = match self.get_orphan_roots(consensus, orphan_block.header.direct_parents().iter().copied().collect()).await { FindRootsOutput::Roots(roots, orphan_ancestors) => (roots, orphan_ancestors), From 694da52b710c13c9235fcfdc933b99b048d06317 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 13:53:41 +0200 Subject: [PATCH 13/48] optimizations related to multi-level relations * In `ParentsBuilder` optimize the execution path of the common case for high levels * Prune relations for all levels below the affiliated proof level for block * Keep relations for multi-level parents of pruning point and its anticone --- .../pipeline/pruning_processor/processor.rs | 87 +++++++---- consensus/src/processes/parents_builder.rs | 139 ++++++++++-------- .../src/processes/reachability/tests/mod.rs | 6 + utils/src/sync/semaphore.rs | 2 +- 4 files changed, 141 insertions(+), 93 deletions(-) diff --git a/consensus/src/pipeline/pruning_processor/processor.rs b/consensus/src/pipeline/pruning_processor/processor.rs index a537b47..e5ff89e 100644 --- a/consensus/src/pipeline/pruning_processor/processor.rs +++ b/consensus/src/pipeline/pruning_processor/processor.rs @@ -2,7 +2,7 @@ use crate::{ consensus::{ - services::{ConsensusServices, DbGhostdagManager, DbPruningPointManager}, + services::{ConsensusServices, DbGhostdagManager, DbParentsManager, DbPruningPointManager}, storage::ConsensusStorage, }, model::{ @@ -33,7 +33,7 @@ use spectre_consensus_core::{ muhash::MuHashExtensions, pruning::{PruningPointProof, PruningPointTrustedData}, trusted::ExternalGhostdagData, - BlockHashSet, + BlockHashMap, BlockHashSet, BlockLevel, }; use spectre_consensusmanager::SessionLock; use spectre_core::{debug, info, warn}; @@ -42,7 +42,7 @@ use spectre_hashes::Hash; use spectre_muhash::MuHash; use spectre_utils::iter::IterExtensions; use std::{ - collections::VecDeque, + collections::{hash_map::Entry::Vacant, VecDeque}, ops::Deref, sync::{ atomic::{AtomicBool, Ordering}, @@ -72,6 +72,7 @@ pub struct PruningProcessor { ghostdag_managers: Arc>, pruning_point_manager: DbPruningPointManager, pruning_proof_manager: Arc, + parents_manager: DbParentsManager, // Pruning lock pruning_lock: SessionLock, @@ -109,6 +110,7 @@ impl PruningProcessor { ghostdag_managers: services.ghostdag_managers.clone(), pruning_point_manager: services.pruning_point_manager.clone(), pruning_proof_manager: services.pruning_proof_manager.clone(), + parents_manager: services.parents_manager.clone(), pruning_lock, config, is_consensus_exiting, @@ -262,41 +264,35 @@ impl PruningProcessor { // We keep full data for pruning point and its anticone, relations for DAA/GD // windows and pruning proof, and only headers for past pruning points let keep_blocks: BlockHashSet = data.anticone.iter().copied().collect(); - let keep_relations: BlockHashSet = std::iter::empty() - .chain(data.anticone.iter().copied()) - .chain(data.daa_window_blocks.iter().map(|th| th.header.hash)) - .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) - .chain(proof.iter().flatten().map(|h| h.hash)) - .collect(); - let keep_level_zero_relations: BlockHashSet = std::iter::empty() + let mut keep_relations: BlockHashMap = std::iter::empty() .chain(data.anticone.iter().copied()) .chain(data.daa_window_blocks.iter().map(|th| th.header.hash)) .chain(data.ghostdag_blocks.iter().map(|gd| gd.hash)) .chain(proof[0].iter().map(|h| h.hash)) + .map(|h| (h, 0)) // Mark block level 0 for all the above. Note that below we add the remaining levels .collect(); let keep_headers: BlockHashSet = self.past_pruning_points(); info!("Header and Block pruning: waiting for consensus write permissions..."); let mut prune_guard = self.pruning_lock.blocking_write(); - let mut lock_acquire_time = Instant::now(); - let mut reachability_read = self.reachability_store.upgradable_read(); info!("Starting Header and Block pruning..."); { let mut counter = 0; let mut batch = WriteBatch::default(); - for kept in keep_level_zero_relations.iter().copied() { + // At this point keep_relations only holds level-0 relations which is the correct filtering criteria for primary GHOSTDAG + for kept in keep_relations.keys().copied() { let Some(ghostdag) = self.ghostdag_primary_store.get_data(kept).unwrap_option() else { continue; }; - if ghostdag.unordered_mergeset().any(|h| !keep_level_zero_relations.contains(&h)) { + if ghostdag.unordered_mergeset().any(|h| !keep_relations.contains_key(&h)) { let mut mutable_ghostdag: ExternalGhostdagData = ghostdag.as_ref().into(); - mutable_ghostdag.mergeset_blues.retain(|h| keep_level_zero_relations.contains(h)); - mutable_ghostdag.mergeset_reds.retain(|h| keep_level_zero_relations.contains(h)); - mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_level_zero_relations.contains(k)); - if !keep_level_zero_relations.contains(&mutable_ghostdag.selected_parent) { + mutable_ghostdag.mergeset_blues.retain(|h| keep_relations.contains_key(h)); + mutable_ghostdag.mergeset_reds.retain(|h| keep_relations.contains_key(h)); + mutable_ghostdag.blues_anticone_sizes.retain(|k, _| keep_relations.contains_key(k)); + if !keep_relations.contains_key(&mutable_ghostdag.selected_parent) { mutable_ghostdag.selected_parent = ORIGIN; } counter += 1; @@ -307,6 +303,45 @@ impl PruningProcessor { info!("Header and Block pruning: updated ghostdag data for {} blocks", counter); } + // No need to hold the prune guard while we continue populating keep_relations + drop(prune_guard); + + // Add additional levels only after filtering GHOSTDAG data via level 0 + for (level, level_proof) in proof.iter().enumerate().skip(1) { + let level = level as BlockLevel; + // We obtain the headers of the pruning point anticone (including the pruning point) + // in order to mark all parents of anticone roots at level as not-to-be-deleted. + // This optimizes multi-level parent validation (see ParentsManager) + // by avoiding the deletion of high-level parents which might still be needed for future + // header validation (avoiding the need for reference blocks; see therein). + // + // Notes: + // + // 1. Normally, such blocks would be part of the proof for this level, but here we address the rare case + // where there are a few such parallel blocks (since the proof only contains the past of the pruning point's + // selected-tip-at-level) + // 2. We refer to the pp anticone as roots even though technically it might contain blocks which are not a pure + // antichain (i.e., some of them are in the past of others). These blocks only add redundant info which would + // be included anyway. + let roots_parents_at_level = data + .anticone + .iter() + .copied() + .map(|hash| self.headers_store.get_header_with_block_level(hash).expect("pruning point anticone is not pruned")) + .filter(|root| level > root.block_level) // If the root itself is at level, there's no need for its level-parents + .flat_map(|root| self.parents_manager.parents_at_level(&root.header, level).iter().copied().collect_vec()); + for hash in level_proof.iter().map(|header| header.hash).chain(roots_parents_at_level) { + if let Vacant(e) = keep_relations.entry(hash) { + // This hash was not added by any lower level -- mark it as affiliated with proof level `level` + e.insert(level); + } + } + } + + prune_guard = self.pruning_lock.blocking_write(); + let mut lock_acquire_time = Instant::now(); + let mut reachability_read = self.reachability_store.upgradable_read(); + { // Start with a batch for pruning body tips and selected chain stores let mut batch = WriteBatch::default(); @@ -394,7 +429,7 @@ impl PruningProcessor { self.acceptance_data_store.delete_batch(&mut batch, current).unwrap(); self.block_transactions_store.delete_batch(&mut batch, current).unwrap(); - if keep_relations.contains(¤t) { + if let Some(&affiliated_proof_level) = keep_relations.get(¤t) { if statuses_write.get(current).unwrap_option().is_some_and(|s| s.is_valid()) { // We set the status to header-only only if it was previously set to a valid // status. This is important since some proof headers might not have their status set @@ -403,17 +438,13 @@ impl PruningProcessor { statuses_write.set_batch(&mut batch, current, StatusHeaderOnly).unwrap(); } - // Delete level-0 relations for blocks which only belong to higher proof levels. - // Note: it is also possible to delete level relations for level x > 0 for any block that only belongs - // to proof levels higher than x, but this requires maintaining such per level usage mapping. - // Since the main motivation of this deletion step is to reduce the - // number of origin's children in level 0, and this is not a bottleneck in any other - // level, we currently chose to only delete level-0 redundant relations. - if !keep_level_zero_relations.contains(¤t) { - let mut staging_level_relations = StagingRelationsStore::new(&mut level_relations_write[0]); + // Delete level-x relations for blocks which only belong to higher-than-x proof levels. + // This preserves the semantic that for each level, relations represent a contiguous DAG area in that level + for lower_level in 0..affiliated_proof_level as usize { + let mut staging_level_relations = StagingRelationsStore::new(&mut level_relations_write[lower_level]); relations::delete_level_relations(MemoryWriter, &mut staging_level_relations, current).unwrap_option(); staging_level_relations.commit(&mut batch).unwrap(); - self.ghostdag_stores[0].delete_batch(&mut batch, current).unwrap_option(); + self.ghostdag_stores[lower_level].delete_batch(&mut batch, current).unwrap_option(); } } else { // Count only blocks which get fully pruned including DAG relations diff --git a/consensus/src/processes/parents_builder.rs b/consensus/src/processes/parents_builder.rs index 89028eb..7a65dcd 100644 --- a/consensus/src/processes/parents_builder.rs +++ b/consensus/src/processes/parents_builder.rs @@ -53,7 +53,7 @@ impl let mut origin_children_headers = None; let mut parents = Vec::with_capacity(self.max_block_level as usize); - for block_level in 0..self.max_block_level { + for block_level in 0..=self.max_block_level { // Direct parents are guaranteed to be in one another's anticones so add them all to // all the block levels they occupy. let mut level_candidates_to_reference_blocks = direct_parent_headers @@ -91,78 +91,89 @@ impl .collect::>() }; - for (i, parent) in grandparents.into_iter().enumerate() { - let has_reachability_data = self.reachability_service.has_reachability_data(parent); - - // Reference blocks are the blocks that are used in reachability queries to check if - // a candidate is in the future of another candidate. In most cases this is just the - // block itself, but in the case where a block doesn't have reachability data we need - // to use some blocks in its future as reference instead. - // If we make sure to add a parent in the future of the pruning point first, we can - // know that any pruned candidate that is in the past of some blocks in the pruning - // point anticone should be a parent (in the relevant level) of one of - // the origin children in the pruning point anticone. So we can check which - // origin children have this block as parent and use those block as - // reference blocks. - let reference_blocks = if has_reachability_data { - smallvec![parent] - } else { - // Here we explicitly declare the type because otherwise Rust would make it mutable. - let origin_children_headers: &Vec<_> = origin_children_headers.get_or_insert_with(|| { - self.relations_service - .get_children(ORIGIN) - .unwrap() - .read() - .iter() - .copied() - .map(|parent| self.headers_store.get_header(parent).unwrap()) - .collect_vec() - }); - let mut reference_blocks = SmallVec::with_capacity(origin_children_headers.len()); - for child_header in origin_children_headers.iter() { - if self.parents_at_level(child_header, block_level).contains(&parent) { - reference_blocks.push(child_header.hash); + let parents_at_level = if level_candidates_to_reference_blocks.is_empty() && first_parent_marker == grandparents.len() { + // Optimization: this is a common case for high levels where none of the direct parents is on the level + // and all direct parents have the same level parents. The condition captures this case because all grandparents + // will be below the first parent marker and there will be no additional grandparents. Bcs all grandparents come + // from a single, already validated parent, there's no need to run any additional antichain checks and we can return + // this set. + grandparents.into_iter().collect() + } else { + // + // Iterate through grandparents in order to find an antichain + for (i, parent) in grandparents.into_iter().enumerate() { + let has_reachability_data = self.reachability_service.has_reachability_data(parent); + + // Reference blocks are the blocks that are used in reachability queries to check if + // a candidate is in the future of another candidate. In most cases this is just the + // block itself, but in the case where a block doesn't have reachability data we need + // to use some blocks in its future as reference instead. + // If we make sure to add a parent in the future of the pruning point first, we can + // know that any pruned candidate that is in the past of some blocks in the pruning + // point anticone should be a parent (in the relevant level) of one of + // the origin children in the pruning point anticone. So we can check which + // origin children have this block as parent and use those block as + // reference blocks. + let reference_blocks = if has_reachability_data { + smallvec![parent] + } else { + // Here we explicitly declare the type because otherwise Rust would make it mutable. + let origin_children_headers: &Vec<_> = origin_children_headers.get_or_insert_with(|| { + self.relations_service + .get_children(ORIGIN) + .unwrap() + .read() + .iter() + .copied() + .map(|parent| self.headers_store.get_header(parent).unwrap()) + .collect_vec() + }); + let mut reference_blocks = SmallVec::with_capacity(origin_children_headers.len()); + for child_header in origin_children_headers.iter() { + if self.parents_at_level(child_header, block_level).contains(&parent) { + reference_blocks.push(child_header.hash); + } } + reference_blocks + }; + + // Make sure we process and insert all first parent's parents. See comments above. + // Note that as parents of an already validated block, they all form an antichain, + // hence no need for reachability queries yet. + if i < first_parent_marker { + level_candidates_to_reference_blocks.insert(parent, reference_blocks); + continue; } - reference_blocks - }; - - // Make sure we process and insert all first parent's parents. See comments above. - // Note that as parents of an already validated block, they all form an antichain, - // hence no need for reachability queries yet. - if i < first_parent_marker { - level_candidates_to_reference_blocks.insert(parent, reference_blocks); - continue; - } - if !has_reachability_data { - continue; - } + if !has_reachability_data { + continue; + } - let len_before_retain = level_candidates_to_reference_blocks.len(); - level_candidates_to_reference_blocks - .retain(|_, refs| !self.reachability_service.is_any_dag_ancestor(&mut refs.iter().copied(), parent)); - let is_any_candidate_ancestor_of = level_candidates_to_reference_blocks.len() < len_before_retain; - - // We should add the block as a candidate if it's in the future of another candidate - // or in the anticone of all candidates. - if is_any_candidate_ancestor_of - || !level_candidates_to_reference_blocks.iter().any(|(_, candidate_references)| { - self.reachability_service.is_dag_ancestor_of_any(parent, &mut candidate_references.iter().copied()) - }) - { - level_candidates_to_reference_blocks.insert(parent, reference_blocks); + let len_before_retain = level_candidates_to_reference_blocks.len(); + level_candidates_to_reference_blocks + .retain(|_, refs| !self.reachability_service.is_any_dag_ancestor(&mut refs.iter().copied(), parent)); + let is_any_candidate_ancestor_of = level_candidates_to_reference_blocks.len() < len_before_retain; + + // We should add the block as a candidate if it's in the future of another candidate + // or in the anticone of all candidates. + if is_any_candidate_ancestor_of + || !level_candidates_to_reference_blocks.iter().any(|(_, candidate_references)| { + self.reachability_service.is_dag_ancestor_of_any(parent, &mut candidate_references.iter().copied()) + }) + { + level_candidates_to_reference_blocks.insert(parent, reference_blocks); + } } - } - if block_level > 0 - && level_candidates_to_reference_blocks.len() == 1 - && level_candidates_to_reference_blocks.contains_key(&self.genesis_hash) - { + // After processing all grandparents, collect the successful level candidates + level_candidates_to_reference_blocks.keys().copied().collect_vec() + }; + + if block_level > 0 && parents_at_level.as_slice() == std::slice::from_ref(&self.genesis_hash) { break; } - parents.push(level_candidates_to_reference_blocks.keys().copied().collect_vec()); + parents.push(parents_at_level); } parents diff --git a/consensus/src/processes/reachability/tests/mod.rs b/consensus/src/processes/reachability/tests/mod.rs index 946a08b..aeb0eb0 100644 --- a/consensus/src/processes/reachability/tests/mod.rs +++ b/consensus/src/processes/reachability/tests/mod.rs @@ -105,6 +105,12 @@ impl DagBlock { } } +impl From<(u64, &[u64])> for DagBlock { + fn from(value: (u64, &[u64])) -> Self { + Self::new(value.0.into(), value.1.iter().map(|&i| i.into()).collect()) + } +} + /// A struct with fluent API to streamline DAG building pub struct DagBuilder<'a, T: ReachabilityStore + ?Sized, S: RelationsStore + ChildrenStore + ?Sized> { reachability: &'a mut T, diff --git a/utils/src/sync/semaphore.rs b/utils/src/sync/semaphore.rs index 2ea6dcc..c0ffec8 100644 --- a/utils/src/sync/semaphore.rs +++ b/utils/src/sync/semaphore.rs @@ -41,7 +41,7 @@ mod trace { if log_time + (Duration::from_secs(10).as_micros() as u64) < now { let log_value = self.log_value.load(Ordering::Relaxed); debug!( - "Semaphore: log interval: {:?}, readers time: {:?}, fraction: {:.2}", + "Semaphore: log interval: {:?}, readers time: {:?}, fraction: {:.4}", Duration::from_micros(now - log_time), Duration::from_micros(readers_time - log_value), (readers_time - log_value) as f64 / (now - log_time) as f64 From 4e35e0bfd22256aa756075876d0c6705039ed702 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 13:55:36 +0200 Subject: [PATCH 14/48] query all DNS seeders if missing many connections --- components/connectionmanager/src/lib.rs | 79 +++++++++++++++++-------- 1 file changed, 54 insertions(+), 25 deletions(-) diff --git a/components/connectionmanager/src/lib.rs b/components/connectionmanager/src/lib.rs index 27b8c2a..331f6c1 100644 --- a/components/connectionmanager/src/lib.rs +++ b/components/connectionmanager/src/lib.rs @@ -7,7 +7,7 @@ use std::{ }; use duration_string::DurationString; -use futures_util::future::join_all; +use futures_util::future::{join_all, try_join_all}; use itertools::Itertools; use parking_lot::Mutex as ParkingLotMutex; use rand::{seq::SliceRandom, thread_rng}; @@ -227,12 +227,14 @@ impl ConnectionManager { } if missing_connections > 0 && !self.dns_seeders.is_empty() { - let cmgr = self.clone(); - // DNS lookup is a blocking i/o operation, so we spawn it as a blocking task - let _ = tokio::task::spawn_blocking(move || { - cmgr.dns_seed(missing_connections); //TODO: Consider putting a number higher than `missing_connections`. - }) - .await; + if missing_connections > self.outbound_target / 2 { + // If we are missing more than half of our target, query all in parallel. + // This will always be the case on new node start-up and is the most resilient strategy in such a case. + self.dns_seed_many(self.dns_seeders.len()).await; + } else { + // Try to obtain at least twice the number of missing connections + self.dns_seed_with_address_target(2 * missing_connections).await; + } } } @@ -251,26 +253,17 @@ impl ConnectionManager { join_all(futures).await; } - fn dns_seed(self: &Arc, mut min_addresses_to_fetch: usize) { + /// Queries DNS seeders in random order, one after the other, until obtaining `min_addresses_to_fetch` addresses + async fn dns_seed_with_address_target(self: &Arc, min_addresses_to_fetch: usize) { + let cmgr = self.clone(); + tokio::task::spawn_blocking(move || cmgr.dns_seed_with_address_target_blocking(min_addresses_to_fetch)).await.unwrap(); + } + + fn dns_seed_with_address_target_blocking(self: &Arc, mut min_addresses_to_fetch: usize) { let shuffled_dns_seeders = self.dns_seeders.choose_multiple(&mut thread_rng(), self.dns_seeders.len()); for &seeder in shuffled_dns_seeders { - info!("Querying DNS seeder {}", seeder); - // Since the DNS lookup protocol doesn't come with a port, we must assume that the default port is used. - let addrs = match (seeder, self.default_port).to_socket_addrs() { - Ok(addrs) => addrs, - Err(e) => { - warn!("Error connecting to DNS seeder {}: {}", seeder, e); - continue; - } - }; - - let addrs_len = addrs.len(); - info!("Retrieved {} addresses from DNS seeder {}", addrs_len, seeder); - let mut amgr_lock = self.address_manager.lock(); - for addr in addrs { - amgr_lock.add_address(NetAddress::new(addr.ip().into(), addr.port())); - } - + // Query seeders sequentially until reaching the desired number of addresses + let addrs_len = self.dns_seed_single(seeder); if addrs_len >= min_addresses_to_fetch { break; } else { @@ -279,6 +272,42 @@ impl ConnectionManager { } } + /// Queries `num_seeders_to_query` random DNS seeders in parallel + async fn dns_seed_many(self: &Arc, num_seeders_to_query: usize) -> usize { + info!("Querying {} DNS seeders", num_seeders_to_query); + let shuffled_dns_seeders = self.dns_seeders.choose_multiple(&mut thread_rng(), num_seeders_to_query); + let jobs = shuffled_dns_seeders.map(|seeder| { + let cmgr = self.clone(); + tokio::task::spawn_blocking(move || cmgr.dns_seed_single(seeder)) + }); + try_join_all(jobs).await.unwrap().into_iter().sum() + } + + /// Query a single DNS seeder and add the obtained addresses to the address manager. + /// + /// DNS lookup is a blocking i/o operation so this function is assumed to be called + /// from a blocking execution context. + fn dns_seed_single(self: &Arc, seeder: &str) -> usize { + info!("Querying DNS seeder {}", seeder); + // Since the DNS lookup protocol doesn't come with a port, we must assume that the default port is used. + let addrs = match (seeder, self.default_port).to_socket_addrs() { + Ok(addrs) => addrs, + Err(e) => { + warn!("Error connecting to DNS seeder {}: {}", seeder, e); + return 0; + } + }; + + let addrs_len = addrs.len(); + info!("Retrieved {} addresses from DNS seeder {}", addrs_len, seeder); + let mut amgr_lock = self.address_manager.lock(); + for addr in addrs { + amgr_lock.add_address(NetAddress::new(addr.ip().into(), addr.port())); + } + + addrs_len + } + /// Bans the given IP and disconnects from all the peers with that IP. /// /// _GO-SPECTRED: BanByIP_ From 052e65cea752d7e7d1bb9d859a62521b1070fbd6 Mon Sep 17 00:00:00 2001 From: x100111010 <167847953+x100111010@users.noreply.github.com> Date: Sat, 19 Oct 2024 21:21:13 +0200 Subject: [PATCH 15/48] wRPC v2, integrations, typescript-v2, resolver v2 * refactored Borsh serialization (wRPC v2) * refactored watch-only * fix Rust v1.81 lints * using latest resolver APIs (guards) * updated TypeScript bindings (typescript-v2) * fixes related to "insufficient-funds" * `Hex View for ScriptBuilder`: * formatted hex view for scripts * `Kip9 Updates`: * improvements to mass calculator (Kip9::Beta variant) ** New RPC Methods * `get_sync_status()`: * node sync status polling * `get_connection_count()`: * total TCP sockets in use * `get_system_info()`: * unique ID, git commit hash, total RAM, CPU cores, internal FD limit * `get_metrics()`: * provisional dictionary*based map for custom metrics ### Important Notes * `get_server_info()`: * broken due to data structure changes * `wRPC v2 Breaking Changes`: * new public node infrastructure deployment * custom serializers for elastic data structure serialization --- .github/workflows/ci.yaml | 23 +- .github/workflows/deploy.yaml | 21 +- Cargo.toml | 81 +- README.md | 93 +- TESTING.md | 1 + cli/Cargo.toml | 6 +- cli/src/cli.rs | 47 +- cli/src/error.rs | 6 + cli/src/extensions/transaction.rs | 29 +- cli/src/imports.rs | 2 +- cli/src/modules/account.rs | 81 +- cli/src/modules/connect.rs | 3 +- cli/src/modules/details.rs | 12 + cli/src/modules/history.rs | 14 +- cli/src/modules/mod.rs | 3 +- cli/src/modules/pssb.rs | 266 ++ cli/src/modules/pssb_redeem.json | 99 + cli/src/modules/reload.rs | 6 +- cli/src/modules/rpc.rs | 78 +- cli/src/modules/send.rs | 2 +- cli/src/modules/settings.rs | 7 +- cli/src/modules/wallet.rs | 16 +- cli/src/wizards/wallet.rs | 70 +- consensus/client/Cargo.toml | 4 +- consensus/client/src/header.rs | 11 +- consensus/client/src/input.rs | 44 +- consensus/client/src/lib.rs | 20 +- consensus/client/src/outpoint.rs | 9 + consensus/client/src/output.rs | 49 +- consensus/client/src/serializable/mod.rs | 8 +- consensus/client/src/serializable/numeric.rs | 19 +- consensus/client/src/serializable/string.rs | 6 +- consensus/client/src/sign.rs | 14 +- consensus/client/src/transaction.rs | 107 +- consensus/client/src/utils.rs | 81 + consensus/client/src/utxo.rs | 70 +- consensus/client/src/vtx.rs | 35 - consensus/core/Cargo.toml | 5 +- consensus/core/src/api/stats.rs | 24 +- consensus/core/src/hashing/mod.rs | 2 + consensus/core/src/hashing/wasm.rs | 27 + consensus/core/src/header.rs | 6 + consensus/core/src/mass/mod.rs | 11 + consensus/core/src/network.rs | 7 +- consensus/core/src/sign.rs | 19 +- consensus/core/src/tx.rs | 13 +- consensus/core/src/tx/script_public_key.rs | 37 +- consensus/pow/Cargo.toml | 3 + consensus/pow/src/wasm.rs | 76 +- consensus/src/consensus/storage.rs | 2 +- consensus/src/model/stores/acceptance_data.rs | 1 - .../src/model/stores/block_transactions.rs | 1 - consensus/src/model/stores/ghostdag.rs | 1 - consensus/src/model/stores/headers.rs | 1 - consensus/src/model/stores/mod.rs | 7 +- consensus/src/model/stores/utxo_set.rs | 5 +- .../body_validation_in_context.rs | 2 +- consensus/src/processes/coinbase.rs | 2 +- consensus/src/processes/mass.rs | 12 +- consensus/src/processes/pruning.rs | 2 +- consensus/src/processes/sync/mod.rs | 2 +- consensus/wasm/Cargo.toml | 4 +- crypto/addresses/Cargo.toml | 4 +- crypto/addresses/src/bech32.rs | 7 +- crypto/addresses/src/lib.rs | 26 +- crypto/hashes/src/lib.rs | 7 +- crypto/muhash/fuzz/fuzz_targets/u3072.rs | 1 - crypto/muhash/src/u3072.rs | 4 +- crypto/txscript/Cargo.toml | 11 + crypto/txscript/src/caches.rs | 3 +- crypto/txscript/src/data_stack.rs | 1 - crypto/txscript/src/error.rs | 89 + crypto/txscript/src/lib.rs | 4 + crypto/txscript/src/opcodes/mod.rs | 2 - crypto/txscript/src/result.rs | 1 + crypto/txscript/src/script_builder.rs | 13 +- crypto/txscript/src/script_class.rs | 1 + crypto/txscript/src/standard.rs | 1 - crypto/txscript/src/wasm/builder.rs | 179 ++ crypto/txscript/src/wasm/mod.rs | 15 + .../txscript/src/wasm/opcodes.rs | 241 +- database/src/registry.rs | 4 +- indexes/utxoindex/src/stores/indexed_utxos.rs | 4 +- math/src/uint.rs | 2 +- metrics/core/src/data.rs | 223 +- metrics/core/src/error.rs | 3 + metrics/core/src/lib.rs | 181 +- mining/src/testutils/coinbase_mock.rs | 1 - notify/Cargo.toml | 1 + notify/src/scope.rs | 146 + notify/src/subscription/mod.rs | 1 + protocol/flows/src/flowcontext/orphans.rs | 7 +- protocol/p2p/src/convert/net_address.rs | 5 +- rothschild/src/main.rs | 26 +- rpc/core/Cargo.toml | 6 +- rpc/core/src/api/connection.rs | 7 + rpc/core/src/api/mod.rs | 1 + rpc/core/src/api/notifications.rs | 98 +- rpc/core/src/api/ops.rs | 167 +- rpc/core/src/api/rpc.rs | 272 +- rpc/core/src/convert/block.rs | 47 +- rpc/core/src/convert/tx.rs | 22 +- rpc/core/src/convert/utxo.rs | 6 +- rpc/core/src/model/address.rs | 42 +- rpc/core/src/model/block.rs | 105 +- rpc/core/src/model/feerate_estimate.rs | 58 +- rpc/core/src/model/header.rs | 332 ++- rpc/core/src/model/mempool.rs | 40 +- rpc/core/src/model/message.rs | 2415 ++++++++++++++++- rpc/core/src/model/mod.rs | 1 + rpc/core/src/model/tests.rs | 1325 +++++++++ rpc/core/src/model/tx.rs | 275 +- rpc/core/src/wasm/convert.rs | 5 +- rpc/core/src/wasm/message.rs | 311 ++- rpc/grpc/client/src/lib.rs | 2 + rpc/grpc/client/src/route.rs | 4 +- rpc/grpc/core/proto/messages.proto | 8 +- rpc/grpc/core/proto/rpc.proto | 49 +- rpc/grpc/core/src/convert/block.rs | 19 + rpc/grpc/core/src/convert/header.rs | 85 +- rpc/grpc/core/src/convert/message.rs | 75 +- rpc/grpc/core/src/convert/metrics.rs | 24 + rpc/grpc/core/src/convert/spectred.rs | 4 + rpc/grpc/core/src/ops.rs | 2 + .../server/src/request_handler/factory.rs | 2 + rpc/grpc/server/src/tests/rpc_core_mock.rs | 168 +- rpc/macros/src/grpc/server.rs | 5 +- rpc/macros/src/lib.rs | 6 + rpc/macros/src/wrpc/client.rs | 11 +- rpc/macros/src/wrpc/mod.rs | 1 + rpc/macros/src/wrpc/server.rs | 10 +- rpc/macros/src/wrpc/test.rs | 60 + rpc/macros/src/wrpc/wasm.rs | 4 +- rpc/service/Cargo.toml | 2 +- rpc/service/src/converter/consensus.rs | 2 +- rpc/service/src/service.rs | 217 +- rpc/wrpc/client/Cargo.toml | 1 + rpc/wrpc/client/Resolvers.toml | 9 +- rpc/wrpc/client/src/client.rs | 53 +- rpc/wrpc/client/src/node.rs | 8 +- rpc/wrpc/client/src/resolver.rs | 166 +- rpc/wrpc/proxy/src/main.rs | 4 +- rpc/wrpc/resolver/src/args.rs | 54 - rpc/wrpc/resolver/src/connection.rs | 2 +- rpc/wrpc/resolver/src/error.rs | 53 - rpc/wrpc/resolver/src/log.rs | 44 - rpc/wrpc/resolver/src/main.rs | 41 - rpc/wrpc/resolver/src/monitor.rs | 241 -- rpc/wrpc/resolver/src/node.rs | 75 - rpc/wrpc/resolver/src/panic.rs | 10 - rpc/wrpc/resolver/src/params.rs | 146 - rpc/wrpc/resolver/src/result.rs | 1 - rpc/wrpc/resolver/src/server.rs | 149 - rpc/wrpc/resolver/src/transport.rs | 8 - rpc/wrpc/server/Cargo.toml | 3 +- rpc/wrpc/server/src/address.rs | 1 - rpc/wrpc/server/src/connection.rs | 3 +- rpc/wrpc/server/src/router.rs | 17 +- rpc/wrpc/server/src/service.rs | 14 +- rpc/wrpc/wasm/Cargo.toml | 4 +- rpc/wrpc/wasm/src/client.rs | 27 +- rpc/wrpc/wasm/src/resolver.rs | 35 +- spectred/src/daemon.rs | 7 +- testing/integration/src/common/utils.rs | 10 +- .../src/daemon_integration_tests.rs | 7 +- testing/integration/src/rpc_tests.rs | 250 +- testing/integration/src/tasks/block/miner.rs | 10 +- .../integration/src/tasks/block/submitter.rs | 10 +- utils/Cargo.toml | 7 + utils/build.rs | 82 + utils/src/git.rs | 53 + utils/src/lib.rs | 5 + utils/src/mem_size.rs | 2 +- utils/src/networking.rs | 32 +- utils/src/option.rs | 5 +- utils/src/sysinfo.rs | 81 + wallet/bip32/src/mnemonic/phrase.rs | 6 +- wallet/bip32/src/xpublic_key.rs | 12 +- wallet/core/Cargo.toml | 5 +- wallet/core/src/account/descriptor.rs | 21 +- wallet/core/src/account/kind.rs | 27 +- wallet/core/src/account/mod.rs | 73 +- wallet/core/src/account/pssb.rs | 360 +++ wallet/core/src/account/variants/bip32.rs | 11 +- .../core/src/account/variants/bip32watch.rs | 9 +- wallet/core/src/account/variants/keypair.rs | 13 +- wallet/core/src/account/variants/legacy.rs | 5 +- wallet/core/src/account/variants/multisig.rs | 13 +- wallet/core/src/account/variants/resident.rs | 1 + wallet/core/src/api/message.rs | 41 + wallet/core/src/api/traits.rs | 8 +- wallet/core/src/api/transport.rs | 2 +- wallet/core/src/compat/gen1.rs | 8 +- wallet/core/src/derivation.rs | 16 +- wallet/core/src/deterministic.rs | 22 +- wallet/core/src/encryption.rs | 2 +- wallet/core/src/error.rs | 25 +- wallet/core/src/imports.rs | 2 +- wallet/core/src/lib.rs | 5 + wallet/core/src/prelude.rs | 5 + wallet/core/src/serializer.rs | 4 +- wallet/core/src/storage/account.rs | 28 +- wallet/core/src/storage/binding.rs | 39 + wallet/core/src/storage/keydata/data.rs | 9 +- wallet/core/src/storage/local/interface.rs | 2 +- wallet/core/src/storage/local/payload.rs | 12 +- .../src/storage/local/transaction/fsio.rs | 7 +- .../src/storage/local/transaction/indexdb.rs | 171 +- wallet/core/src/storage/local/wallet.rs | 18 +- wallet/core/src/storage/metadata.rs | 8 +- wallet/core/src/storage/mod.rs | 2 +- wallet/core/src/storage/transaction/data.rs | 100 +- wallet/core/src/storage/transaction/record.rs | 61 +- wallet/core/src/tests/rpc_core_mock.rs | 168 +- wallet/core/src/tests/storage.rs | 4 +- wallet/core/src/tx/generator/generator.rs | 62 +- wallet/core/src/tx/generator/pending.rs | 48 +- wallet/core/src/tx/generator/settings.rs | 7 + wallet/core/src/tx/generator/test.rs | 14 +- wallet/core/src/tx/mass.rs | 123 +- wallet/core/src/tx/payment.rs | 16 +- wallet/core/src/utxo/balance.rs | 1 + wallet/core/src/utxo/context.rs | 15 +- wallet/core/src/utxo/processor.rs | 66 +- wallet/core/src/utxo/reference.rs | 6 +- wallet/core/src/utxo/settings.rs | 133 +- wallet/core/src/wallet/api.rs | 125 +- wallet/core/src/wallet/mod.rs | 137 +- wallet/core/src/wasm/api/message.rs | 19 +- wallet/core/src/wasm/cryptobox.rs | 16 +- wallet/core/src/wasm/message.rs | 8 +- wallet/core/src/wasm/notify.rs | 141 +- wallet/core/src/wasm/signer.rs | 39 +- wallet/core/src/wasm/tx/consensus.rs | 7 + wallet/core/src/wasm/tx/fees.rs | 2 +- .../core/src/wasm/tx/generator/generator.rs | 24 +- wallet/core/src/wasm/tx/generator/pending.rs | 50 +- wallet/core/src/wasm/tx/mass.rs | 170 +- wallet/core/src/wasm/tx/mod.rs | 2 - wallet/core/src/wasm/tx/utils.rs | 38 +- wallet/core/src/wasm/utxo/context.rs | 14 +- wallet/core/src/wasm/utxo/processor.rs | 53 +- wallet/core/src/wasm/wallet/account.rs | 104 +- wallet/core/src/wasm/wallet/mod.rs | 1 - wallet/keys/Cargo.toml | 4 +- wallet/keys/src/derivation/gen0/hd.rs | 2 +- wallet/keys/src/derivation_path.rs | 7 +- wallet/keys/src/keypair.rs | 5 +- wallet/keys/src/privatekey.rs | 7 +- wallet/keys/src/pubkeygen.rs | 4 +- wallet/keys/src/publickey.rs | 17 +- wallet/keys/src/xprv.rs | 71 +- wallet/keys/src/xpub.rs | 55 +- wallet/macros/src/wallet/client.rs | 2 +- wallet/macros/src/wallet/server.rs | 2 +- wallet/psst/Cargo.toml | 13 +- wallet/psst/examples/multisig.rs | 2 +- wallet/psst/src/bundle.rs | 353 +++ wallet/psst/src/convert.rs | 109 + wallet/psst/src/error.rs | 55 +- wallet/psst/src/global.rs | 5 +- wallet/psst/src/input.rs | 9 +- wallet/psst/src/lib.rs | 484 +--- wallet/psst/src/output.rs | 3 +- wallet/psst/src/psst.rs | 472 ++++ wallet/psst/src/wasm/bundle.rs | 1 + wallet/psst/src/wasm/error.rs | 64 + wallet/psst/src/wasm/input.rs | 1 + wallet/psst/src/wasm/mod.rs | 6 + wallet/psst/src/wasm/output.rs | 1 + wallet/psst/src/wasm/psst.rs | 320 +++ wallet/psst/src/wasm/result.rs | 1 + wasm/CHANGELOG.md | 19 + wasm/Cargo.toml | 4 + wasm/README.md | 28 +- wasm/build-node-dev | 3 +- wasm/core/Cargo.toml | 6 +- wasm/core/src/hex.rs | 152 ++ wasm/core/src/lib.rs | 3 +- wasm/core/src/types.rs | 2 +- .../nodejs/javascript/general/derivation.js | 17 +- .../{mining-state.js => mining-pow.js} | 8 +- .../javascript/transactions/serialize.js | 48 + .../transactions/simple-transaction.js | 2 +- .../transactions/single-transaction-demo.js | 8 +- .../nodejs/typescript/src/scriptBuilder.ts | 13 + wasm/src/lib.rs | 3 + 287 files changed, 12408 insertions(+), 3703 deletions(-) create mode 100644 cli/src/modules/pssb.rs create mode 100644 cli/src/modules/pssb_redeem.json create mode 100644 consensus/client/src/utils.rs delete mode 100644 consensus/client/src/vtx.rs create mode 100644 consensus/core/src/hashing/wasm.rs create mode 100644 crypto/txscript/src/error.rs create mode 100644 crypto/txscript/src/result.rs create mode 100644 crypto/txscript/src/wasm/builder.rs create mode 100644 crypto/txscript/src/wasm/mod.rs rename consensus/client/src/script.rs => crypto/txscript/src/wasm/opcodes.rs (51%) create mode 100644 rpc/core/src/api/connection.rs create mode 100644 rpc/core/src/model/tests.rs create mode 100644 rpc/macros/src/wrpc/test.rs delete mode 100644 rpc/wrpc/resolver/src/args.rs delete mode 100644 rpc/wrpc/resolver/src/error.rs delete mode 100644 rpc/wrpc/resolver/src/log.rs delete mode 100644 rpc/wrpc/resolver/src/main.rs delete mode 100644 rpc/wrpc/resolver/src/monitor.rs delete mode 100644 rpc/wrpc/resolver/src/node.rs delete mode 100644 rpc/wrpc/resolver/src/panic.rs delete mode 100644 rpc/wrpc/resolver/src/params.rs delete mode 100644 rpc/wrpc/resolver/src/result.rs delete mode 100644 rpc/wrpc/resolver/src/server.rs delete mode 100644 rpc/wrpc/resolver/src/transport.rs create mode 100644 utils/build.rs create mode 100644 utils/src/git.rs create mode 100644 utils/src/sysinfo.rs create mode 100644 wallet/core/src/account/pssb.rs create mode 100644 wallet/psst/src/bundle.rs create mode 100644 wallet/psst/src/convert.rs create mode 100644 wallet/psst/src/psst.rs create mode 100644 wallet/psst/src/wasm/bundle.rs create mode 100644 wallet/psst/src/wasm/error.rs create mode 100644 wallet/psst/src/wasm/input.rs create mode 100644 wallet/psst/src/wasm/mod.rs create mode 100644 wallet/psst/src/wasm/output.rs create mode 100644 wallet/psst/src/wasm/psst.rs create mode 100644 wallet/psst/src/wasm/result.rs create mode 100644 wasm/core/src/hex.rs rename wasm/examples/nodejs/javascript/general/{mining-state.js => mining-pow.js} (90%) create mode 100644 wasm/examples/nodejs/javascript/transactions/serialize.js create mode 100644 wasm/examples/nodejs/typescript/src/scriptBuilder.ts diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 5a264fe..d3991c3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -160,7 +160,7 @@ jobs: run: cargo clippy --workspace --tests --benches -- -D warnings check-wasm32: - name: Check Wasm32 + name: Check WASM32 runs-on: ubuntu-latest steps: - name: Checkout sources @@ -231,12 +231,16 @@ jobs: run: cargo clippy -p spectre-wasm --target wasm32-unknown-unknown build-wasm32: - name: Build Wasm32 + name: Build WASM32 SDK runs-on: ubuntu-latest steps: - name: Checkout sources uses: actions/checkout@v4 + - name: Setup Environment + shell: bash + run: echo "SHORT_SHA=`git rev-parse --short HEAD`" >> $GITHUB_ENV + - name: Install Protoc uses: arduino/setup-protoc@v3 with: @@ -293,17 +297,22 @@ jobs: target/ key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - - name: Build wasm release + - name: Build WASM release env: TARGET_CC: clang TARGET_CFLAGS: -I/usr/include - run: cd wasm && bash build-release + run: | + pushd . + cd wasm + bash build-release + popd + mv wasm/release/spectre-wasm32-sdk.zip wasm/release/spectre-wasm32-sdk-${{ env.SHORT_SHA }}.zip - - name: Upload wasm binary to GitHub + - name: Upload WASM build to GitHub uses: actions/upload-artifact@v4 with: - name: spectre-wasm32-sdk - path: wasm/release/spectre-wasm32-sdk.zip + name: spectre-wasm32-sdk-${{ env.SHORT_SHA }}.zip + path: wasm/release/spectre-wasm32-sdk-${{ env.SHORT_SHA }}.zip build-release: name: Build Ubuntu Release diff --git a/.github/workflows/deploy.yaml b/.github/workflows/deploy.yaml index 5c21a56..8579fbc 100644 --- a/.github/workflows/deploy.yaml +++ b/.github/workflows/deploy.yaml @@ -328,8 +328,7 @@ jobs: sudo update-alternatives --install /usr/bin/c++ c++ /usr/lib/llvm-15/bin/clang++ 0 - name: Install gcc-multilib - - # gcc-multilib allows clang to find gnu libraries properly. + # gcc-multilib allows clang to find gnu libraries properly run: sudo apt install -y gcc-multilib - name: Install stable toolchain @@ -366,13 +365,21 @@ jobs: TARGET_CC: clang TARGET_CFLAGS: -I/usr/include run: | - mkdir sdk || true cd wasm bash build-release - mv release/spectre-wasm32-sdk.zip ../sdk/spectre-wasm32-sdk-${{ github.event.release.tag_name }}.zip + mv release/spectre-wasm32-sdk.zip ../spectre-wasm32-sdk-${{ github.event.release.tag_name }}.zip + + archive="spectre-wasm32-sdk-${{ github.event.release.tag_name }}.zip" + asset_name="spectre-wasm32-sdk-${{ github.event.release.tag_name }}.zip" + echo "archive=${archive}" >> $GITHUB_ENV + echo "asset_name=${asset_name}" >> $GITHUB_ENV - name: Upload WASM32 SDK - uses: softprops/action-gh-release@v2 + uses: actions/upload-release-asset@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: - files: | - sdk/*.zip + upload_url: ${{ github.event.release.upload_url }} + asset_path: "./${{ env.archive }}" + asset_name: "${{ env.asset_name }}" + asset_content_type: application/zip \ No newline at end of file diff --git a/Cargo.toml b/Cargo.toml index 920aca9..46ef7e8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,7 +36,6 @@ members = [ "rpc/grpc/core", "rpc/grpc/client", "rpc/grpc/server", - "rpc/wrpc/resolver", "rpc/wrpc/server", "rpc/wrpc/client", "rpc/wrpc/proxy", @@ -62,7 +61,7 @@ members = [ ] [workspace.package] -rust-version = "1.78.0" +rust-version = "1.80.0" version = "0.3.15" authors = ["Spectre developers"] license = "ISC" @@ -84,7 +83,6 @@ include = [ spectre-addresses = { version = "0.3.15", path = "crypto/addresses" } spectre-addressmanager = { version = "0.3.15", path = "components/addressmanager" } spectre-bip32 = { version = "0.3.15", path = "wallet/bip32" } -spectre-resolver = { version = "0.3.15", path = "rpr/wrpc/resolver" } spectre-cli = { version = "0.3.15", path = "cli" } spectre-connectionmanager = { version = "0.3.15", path = "components/connectionmanager" } spectre-consensus = { version = "0.3.15", path = "consensus" } @@ -124,13 +122,12 @@ spectre-utxoindex = { version = "0.3.15", path = "indexes/utxoindex" } spectre-wallet = { version = "0.3.15", path = "wallet/native" } spectre-wallet-cli-wasm = { version = "0.3.15", path = "wallet/wasm" } spectre-wallet-keys = { version = "0.3.15", path = "wallet/keys" } -spectre-wallet-core = { version = "0.3.15", path = "wallet/core" } spectre-wallet-psst = { version = "0.3.15", path = "wallet/psst" } +spectre-wallet-core = { version = "0.3.15", path = "wallet/core" } spectre-wallet-macros = { version = "0.3.15", path = "wallet/macros" } spectre-wasm = { version = "0.3.15", path = "wasm" } spectre-wasm-core = { version = "0.3.15", path = "wasm/core" } spectre-wrpc-client = { version = "0.3.15", path = "rpc/wrpc/client" } -spectre-wrpc-core = { version = "0.3.15", path = "rpc/wrpc/core" } spectre-wrpc-proxy = { version = "0.3.15", path = "rpc/wrpc/proxy" } spectre-wrpc-server = { version = "0.3.15", path = "rpc/wrpc/server" } spectre-wrpc-wasm = { version = "0.3.15", path = "rpc/wrpc/wasm" } @@ -147,10 +144,10 @@ async-channel = "2.0.0" async-std = { version = "1.12.0", features = ['attributes'] } async-stream = "0.3.5" async-trait = "0.1.74" -base64 = "0.21.5" +base64 = "0.22.1" bincode = { version = "1.3.3", default-features = false } blake2b_simd = "1.0.2" -borsh = { version = "0.9.1", features = ["rc"] } # please keep this fixed +borsh = { version = "1.5.1", features = ["derive", "rc"] } bs58 = { version = "0.5.0", features = ["check"], default-features = false } cc = "1.0.83" cfb-mode = "0.8.2" @@ -162,42 +159,45 @@ criterion = { version = "0.5.1", default-features = false } crossbeam-channel = "0.5.8" ctrlc = "3.4.1" crypto_box = { version = "0.9.1", features = ["chacha20"] } -dashmap = "5.5.3" +dashmap = "6.0.1" derivative = "2.2.0" derive_builder = "0.20.0" derive_more = "0.99.17" +# derive_more = { version = "1.0.0", features = ["full"] } dhat = "0.3.2" dirs = "5.0.1" downcast = "0.11.0" downcast-rs = "1.2.0" -duration-string = "0.3.0" -enum-primitive-derive = "0.2.2" +duration-string = "0.4.0" +enum-primitive-derive = "0.3.0" event-listener = "2.5.3" # TODO "3.0.1" evpkdf = "0.2.0" -faster-hex = "0.6.1" # TODO "0.8.1" - fails unit tests +faster-hex = "0.9.0" fixedstr = { version = "0.5.4", features = ["serde"] } flate2 = "1.0.28" futures = { version = "0.3.29" } -futures-util = { version = "0.3.29", default-features = false, features = [ - "alloc", -] } +futures-util = { version = "0.3.29", default-features = false, features = ["alloc"] } getrandom = { version = "0.2.10", features = ["js"] } -h2 = "0.3.21" -heapless = "0.7.16" +h2 = "0.4.6" +# h2 = "0.3.21" +heapless = "0.8.0" +# heapless = "0.7.16" hex = { version = "0.4.3", features = ["serde"] } hex-literal = "0.4.1" +hexplay = "0.3.0" hmac = { version = "0.12.1", default-features = false } home = "0.5.5" igd-next = { version = "0.14.2", features = ["aio_tokio"] } indexmap = "2.1.0" intertrait = "0.2.2" ipnet = "2.9.0" -itertools = "0.11.0" -js-sys = "0.3.67" +itertools = "0.13.0" +js-sys = "0.3.70" keccak = "0.1.4" -local-ip-address = "0.5.6" +local-ip-address = "0.6.1" log = "0.4.20" log4rs = "1.2.0" +mac_address = "1.1.7" malachite-base = "0.4.4" malachite-nz = "0.4.4" md-5 = "0.10.6" @@ -211,6 +211,7 @@ paste = "1.0.14" pbkdf2 = "0.12.2" portable-atomic = { version = "1.5.1", features = ["float"] } prost = "0.12.1" +# prost = "0.13.1" rand = "0.8.5" rand_chacha = "0.3.1" rand_core = { version = "0.6.4", features = ["std"] } @@ -219,8 +220,8 @@ rayon = "1.8.0" regex = "1.10.2" ripemd = { version = "0.1.3", default-features = false } rlimit = "0.10.1" -rocksdb = "0.21.0" -secp256k1 = { version = "0.28.2", features = [ +rocksdb = "0.22.0" +secp256k1 = { version = "0.29.0", features = [ "global-context", "rand-std", "serde", @@ -243,6 +244,7 @@ spectrex = ">=0.3.17" statest = "0.2.2" statrs = "0.13.0" # TODO "0.16.0" subtle = { version = "2.5.0", default-features = false } +sysinfo = "0.31.2" tempfile = "3.8.1" textwrap = "0.16.0" thiserror = "1.0.50" @@ -253,10 +255,10 @@ tonic = { version = "0.10.2", features = ["tls", "gzip", "transport"] } tonic-build = { version = "0.10.2", features = ["prost"] } triggered = "0.1.2" uuid = { version = "1.5.0", features = ["v4", "fast-rng", "serde"] } -wasm-bindgen = { version = "0.2.92", features = ["serde-serialize"] } -wasm-bindgen-futures = "0.4.40" -wasm-bindgen-test = "0.3.37" -web-sys = "0.3.67" +wasm-bindgen = { version = "0.2.93", features = ["serde-serialize"] } +wasm-bindgen-futures = "0.4.43" +wasm-bindgen-test = "0.3.43" +web-sys = "0.3.70" xxhash-rust = { version = "0.8.7", features = ["xxh3"] } zeroize = { version = "1.6.0", default-features = false, features = ["alloc"] } pin-project-lite = "0.2.13" @@ -267,7 +269,7 @@ tower-http = { version = "0.4.4", features = [ tower = "0.4.7" hyper = "0.14.27" chrono = "0.4.31" -indexed_db_futures = "0.4.1" +indexed_db_futures = "0.5.0" # workflow dependencies that are not a part of core libraries # workflow-perf-monitor = { path = "../../../workflow-perf-monitor-rs" } @@ -275,17 +277,18 @@ workflow-perf-monitor = "0.0.2" nw-sys = "0.1.6" # workflow dependencies -workflow-core = { version = "0.12.1" } -workflow-d3 = { version = "0.12.1" } -workflow-dom = { version = "0.12.1" } -workflow-http = { version = "0.12.1" } -workflow-log = { version = "0.12.1" } -workflow-node = { version = "0.12.1" } -workflow-nw = { version = "0.12.1" } -workflow-rpc = { version = "0.12.1" } -workflow-store = { version = "0.12.1" } -workflow-terminal = { version = "0.12.1" } -workflow-wasm = { version = "0.12.1" } +workflow-core = { version = "0.17.0" } +workflow-d3 = { version = "0.17.0" } +workflow-dom = { version = "0.17.0" } +workflow-http = { version = "0.17.0" } +workflow-log = { version = "0.17.0" } +workflow-node = { version = "0.17.0" } +workflow-nw = { version = "0.17.0" } +workflow-rpc = { version = "0.17.0" } +workflow-serializer = { version = "0.17.0" } +workflow-store = { version = "0.17.0" } +workflow-terminal = { version = "0.17.0" } +workflow-wasm = { version = "0.17.0" } # if below is enabled, this means that there is an ongoing work # on the workflow-rs crate. This requires that you clone workflow-rs @@ -298,6 +301,7 @@ workflow-wasm = { version = "0.12.1" } # workflow-node = { path = "../workflow-rs/node" } # workflow-nw = { path = "../workflow-rs/nw" } # workflow-rpc = { path = "../workflow-rs/rpc" } +# workflow-serializer = { path = "../workflow-rs/serializer" } # workflow-store = { path = "../workflow-rs/store" } # workflow-terminal = { path = "../workflow-rs/terminal" } # workflow-wasm = { path = "../workflow-rs/wasm" } @@ -311,6 +315,7 @@ workflow-wasm = { version = "0.12.1" } # workflow-node = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } # workflow-nw = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } # workflow-rpc = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } +# workflow-serializer = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } # workflow-store = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } # workflow-terminal = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } # workflow-wasm = { git = "https://github.com/workflow-rs/workflow-rs.git", branch = "master" } @@ -326,3 +331,5 @@ inherits = "release" debug = true strip = false +[workspace.lints.clippy] +empty_docs = "allow" diff --git a/README.md b/README.md index 8475a27..17d59a4 100644 --- a/README.md +++ b/README.md @@ -31,16 +31,16 @@ introducing CPU-only mining algorithm [SpectreX](https://github.com/spectre-proj SpectreX is based on [AstroBWTv3](https://github.com/deroproject/derohe/tree/main/astrobwt/astrobwtv3) and proof-of-work calculation is done in the following steps: -* Step 1: SHA-3 -* Step 2: AstroBWTv3 -* Step 3: HeavyHash +- Step 1: SHA-3 +- Step 2: AstroBWTv3 +- Step 3: HeavyHash Spectre will add full non-disclosable privacy and anonymous transactions in future implemented with the GhostFACE protocol build by a team of anonymous crypto algorithm researchers and engineers. Simple and plain goal: -* PHANTOM Protocol + GhostDAG + GhostFACE = Spectre +- PHANTOM Protocol + GhostDAG + GhostFACE = Spectre Spectre will become a ghostchain; nothing more, nothing less. Design decisions have been made already and more details about the GhostFACE @@ -58,17 +58,17 @@ Why another fork? Kaspa is great but we love privacy, Monero and DERO are great but we love speed! So lets join the cool things from both. We decided to take Kaspa as codebase, quick comparison: -Feature | Spectre | Kaspa | Monero | DERO ------------------------------|----------|------------|---------|----------- -PoW Algorithm | SpectreX | kHeavyHash | RandomX | AstroBWTv3 -Balance Encryption | Future | No | Yes | Yes -Transaction Encryption | Future | No | Yes | Yes -Message Encyrption | Future | No | No | Yes -Untraceable Transactions | Future | No | Yes | Yes -Untraceable Mining | Yes | No | No | Yes -Built-in multicore CPU-miner | Yes | No | Yes | Yes -High BPS | Yes | Yes | No | No -High TPS | Yes | Yes | No | No +| Feature | Spectre | Kaspa | Monero | DERO | +| ---------------------------- | -------- | ---------- | ------- | ---------- | +| PoW Algorithm | SpectreX | kHeavyHash | RandomX | AstroBWTv3 | +| Balance Encryption | Future | No | Yes | Yes | +| Transaction Encryption | Future | No | Yes | Yes | +| Message Encyrption | Future | No | No | Yes | +| Untraceable Transactions | Future | No | Yes | Yes | +| Untraceable Mining | Yes | No | No | Yes | +| Built-in multicore CPU-miner | Yes | No | Yes | Yes | +| High BPS | Yes | Yes | No | No | +| High TPS | Yes | Yes | No | No | Untraceable Mining is already achieved with AstroBWTv3 and a multicore miner is already being shipped with Spectre, working on ARM/x86. We @@ -79,7 +79,7 @@ leave it up to the community to build an highly optimized CPU-miner. We love numbers, you will find a lot of mathematical constants in the source code, in the genesis hash, genesis payload, genesis merkle hash and more. Mathematical constants like [Pi](https://en.wikipedia.org/wiki/Pi), -[E](https://en.wikipedia.org/wiki/E_(mathematical_constant)) and +[E]() and several prime numbers used as starting values for nonce or difficulty. The first released version is `0.3.14`, the famous Pi divided by 10. @@ -92,19 +92,19 @@ Spectre full-node daemon, CLI wallet application, and testing utilities, all designed to promote decentralization. Here's an overview of the different builds: -Build | Description -----------------------|--------------------------------------------- -linux-gnu-aarch64 | Dynamically linked Linux (arm64) -linux-gnu-powerpc64 | Dynamically linked Linux (ppc64) -linux-gnu-powerpc64le | Dynamically linked Linux (ppc64le) -linux-gnu-riscv64 | Dynamically linked Linux (riscv64) -linux-gnu-amd64 | Dynamically linked Linux (x86_64) -linux-musl-aarch64 | Statically linked Linux (arm64) -linux-musl-amd64 | Statically linked Linux (x86_64) -windows-gnullvm-amd64 | Windows version using GNU ABI from Clang/LLVM -windows-msvc-amd64 | Windows version using Microsoft ABI, requires [MSVC runtime](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) -macos-amd64 | macOS version for Intel-based systems -macos-aarch64 | macOS version for Arm-based systems (M1, M2, etc.) +| Build | Description | +| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| linux-gnu-aarch64 | Dynamically linked Linux (arm64) | +| linux-gnu-powerpc64 | Dynamically linked Linux (ppc64) | +| linux-gnu-powerpc64le | Dynamically linked Linux (ppc64le) | +| linux-gnu-riscv64 | Dynamically linked Linux (riscv64) | +| linux-gnu-amd64 | Dynamically linked Linux (x86_64) | +| linux-musl-aarch64 | Statically linked Linux (arm64) | +| linux-musl-amd64 | Statically linked Linux (x86_64) | +| windows-gnullvm-amd64 | Windows version using GNU ABI from Clang/LLVM | +| windows-msvc-amd64 | Windows version using Microsoft ABI, requires [MSVC runtime](https://learn.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170) | +| macos-amd64 | macOS version for Intel-based systems | +| macos-aarch64 | macOS version for Arm-based systems (M1, M2, etc.) | The dynamically linked versions are always preferred for security reasons. However, for older Linux distributions, statically linked @@ -146,24 +146,24 @@ the MSVC runtime EULA. llvm python3-clang ``` -3. Install the [rust toolchain](https://rustup.rs/) - +4. Install the [rust toolchain](https://rustup.rs/) + If you already have rust installed, update it by running: `rustup update`. -4. Install wasm-pack +5. Install wasm-pack ```bash cargo install wasm-pack ``` -4. Install wasm32 target +6. Install wasm32 target ```bash rustup target add wasm32-unknown-unknown - ``` + ``` -5. Clone the repo +7. Clone the repo ```bash git clone https://github.com/spectre-project/rusty-spectre @@ -180,7 +180,7 @@ the MSVC runtime EULA. Add the `bin` directory of the LLVM installation (`C:\Program Files\LLVM\bin`) to PATH. - + Set `LIBCLANG_PATH` environment variable to point to the `bin` directory as well. @@ -193,9 +193,9 @@ the MSVC runtime EULA. WASM32 targets). Currently, the best way to address this, is as follows: after installing LLVM on Windows, go to the target `bin` installation directory and copy or rename `LLVM_AR.exe` to `AR.exe`. - + 4. Install the [rust toolchain](https://rustup.rs/) - + If you already have rust installed, update it by running: `rustup update`. @@ -209,7 +209,7 @@ the MSVC runtime EULA. ```bash rustup target add wasm32-unknown-unknown - ``` + ``` 7. Clone the repo @@ -226,8 +226,8 @@ the MSVC runtime EULA. brew install protobuf ``` -2. Install LLVM. - +2. Install LLVM. + The default XCode installation of `llvm` does not support WASM build targets. To build WASM on MacOS you need to install `llvm` from homebrew (at the time of writing, the llvm version for MacOS @@ -272,7 +272,7 @@ the MSVC runtime EULA. ``` 3. Install the [rust toolchain](https://rustup.rs/) - + If you already have rust installed, update it by running: `rustup update`. @@ -282,13 +282,13 @@ the MSVC runtime EULA. cargo install wasm-pack ``` -4. Install wasm32 target +5. Install wasm32 target ```bash rustup target add wasm32-unknown-unknown ``` -5. Clone the repo +6. Clone the repo ```bash git clone https://github.com/spectre-project/rusty-spectre @@ -358,7 +358,7 @@ cargo install basic-http-server basic-http-server ``` -The *basic-http-server* will serve on port 4000 by default, so open +The _basic-http-server_ will serve on port 4000 by default, so open your web browser and load http://localhost:4000 The framework is compatible with all major desktop and mobile browsers. @@ -386,7 +386,7 @@ cargo run --release --bin spectred -- -C /path/to/configfile.toml ``` - The config file should be a list of \ = \ - separated by newlines. + separated by newlines. - Whitespace around the `=` is fine, `arg=value` and `arg = value` are both parsed correctly. - Values with special characters like `.` or `=` will require quoting @@ -395,6 +395,7 @@ cargo run --release --bin spectred -- -C /path/to/configfile.toml like `addpeer = ["10.0.0.1", "1.2.3.4"]`. For example: + ``` testnet = true utxoindex = false diff --git a/TESTING.md b/TESTING.md index eaeb187..b04c48c 100644 --- a/TESTING.md +++ b/TESTING.md @@ -21,6 +21,7 @@ while attempting to fill each block with up to 200 transactions. ```bash cargo run --release --bin simpa -- -t=200 -d=2 -b=8 -n=1000 ``` + ## Heap Profiling Heap-profiling in `spectred` and `simpa` can be done by enabling diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 36e31b7..6ebf90a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -33,6 +33,7 @@ downcast.workspace = true faster-hex.workspace = true futures.workspace = true js-sys.workspace = true +hex.workspace = true spectre-addresses.workspace = true spectre-bip32.workspace = true spectre-consensus-core.workspace = true @@ -43,6 +44,7 @@ spectre-rpc-core.workspace = true spectre-utils.workspace = true spectre-wallet-core.workspace = true spectre-wallet-keys.workspace = true +spectre-wallet-psst.workspace = true spectre-wrpc-client.workspace = true nw-sys.workspace = true pad.workspace = true @@ -80,5 +82,5 @@ features = [ [target.'cfg(not(target_arch = "wasm32"))'.dependencies] tokio.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 5dec558..c5effd9 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -9,7 +9,7 @@ use spectre_daemon::{DaemonEvent, DaemonKind, Daemons}; use spectre_wallet_core::account::Account; use spectre_wallet_core::rpc::DynRpcApi; use spectre_wallet_core::storage::{IdT, PrvKeyDataInfo}; -use spectre_wrpc_client::SpectreRpcClient; +use spectre_wrpc_client::{Resolver, SpectreRpcClient}; use workflow_core::channel::*; use workflow_core::time::Instant; use workflow_log::*; @@ -103,7 +103,7 @@ impl SpectreCli { } pub async fn try_new_arc(options: Options) -> Result> { - let wallet = Arc::new(Wallet::try_new(Wallet::local_store()?, None, None)?); + let wallet = Arc::new(Wallet::try_new(Wallet::local_store()?, Some(Resolver::default()), None)?); let spectre_cli = Arc::new(SpectreCli { term: Arc::new(Mutex::new(None)), @@ -312,7 +312,9 @@ impl SpectreCli { Events::SyncState { sync_state } => { if sync_state.is_synced() && this.wallet().is_open() { - if let Err(error) = this.wallet().reload(false).await { + let guard = this.wallet().guard(); + let guard = guard.lock().await; + if let Err(error) = this.wallet().reload(false, &guard).await { terrorln!(this, "Unable to reload wallet: {error}"); } } @@ -384,8 +386,11 @@ impl SpectreCli { record } => { if !this.is_mutted() || (this.is_mutted() && this.flags.get(Track::Pending)) { + let guard = this.wallet.guard(); + let guard = guard.lock().await; + let include_utxos = this.flags.get(Track::Utxo); - let tx = record.format_transaction_with_state(&this.wallet,Some("reorg"),include_utxos).await; + let tx = record.format_transaction_with_state(&this.wallet,Some("reorg"),include_utxos, &guard).await; tx.iter().for_each(|line|tprintln!(this,"{NOTIFY} {line}")); } }, @@ -394,8 +399,11 @@ impl SpectreCli { } => { // Pending and coinbase stasis fall under the same `Track` category if !this.is_mutted() || (this.is_mutted() && this.flags.get(Track::Pending)) { + let guard = this.wallet.guard(); + let guard = guard.lock().await; + let include_utxos = this.flags.get(Track::Utxo); - let tx = record.format_transaction_with_state(&this.wallet,Some("stasis"),include_utxos).await; + let tx = record.format_transaction_with_state(&this.wallet,Some("stasis"),include_utxos, &guard).await; tx.iter().for_each(|line|tprintln!(this,"{NOTIFY} {line}")); } }, @@ -412,8 +420,11 @@ impl SpectreCli { record } => { if !this.is_mutted() || (this.is_mutted() && this.flags.get(Track::Pending)) { + let guard = this.wallet.guard(); + let guard = guard.lock().await; + let include_utxos = this.flags.get(Track::Utxo); - let tx = record.format_transaction_with_state(&this.wallet,Some("pending"),include_utxos).await; + let tx = record.format_transaction_with_state(&this.wallet,Some("pending"),include_utxos, &guard).await; tx.iter().for_each(|line|tprintln!(this,"{NOTIFY} {line}")); } }, @@ -421,8 +432,11 @@ impl SpectreCli { record } => { if !this.is_mutted() || (this.is_mutted() && this.flags.get(Track::Tx)) { + let guard = this.wallet.guard(); + let guard = guard.lock().await; + let include_utxos = this.flags.get(Track::Utxo); - let tx = record.format_transaction_with_state(&this.wallet,Some("confirmed"),include_utxos).await; + let tx = record.format_transaction_with_state(&this.wallet,Some("confirmed"),include_utxos, &guard).await; tx.iter().for_each(|line|tprintln!(this,"{NOTIFY} {line}")); } }, @@ -533,6 +547,9 @@ impl SpectreCli { } async fn select_account_with_args(&self, autoselect: bool) -> Result> { + let guard = self.wallet.guard(); + let guard = guard.lock().await; + let mut selection = None; let mut list_by_key = Vec::<(Arc, Vec<(usize, Arc)>)>::new(); @@ -541,7 +558,7 @@ impl SpectreCli { let mut keys = self.wallet.keys().await?; while let Some(key) = keys.try_next().await? { let mut prv_key_accounts = Vec::new(); - let mut accounts = self.wallet.accounts(Some(key.id)).await?; + let mut accounts = self.wallet.accounts(Some(key.id), &guard).await?; while let Some(account) = accounts.next().await { let account = account?; prv_key_accounts.push((flat_list.len(), account.clone())); @@ -552,7 +569,7 @@ impl SpectreCli { } let mut watch_accounts = Vec::<(usize, Arc)>::new(); - let mut unfiltered_accounts = self.wallet.accounts(None).await?; + let mut unfiltered_accounts = self.wallet.accounts(None, &guard).await?; while let Some(account) = unfiltered_accounts.try_next().await? { if account.feature().is_some() { @@ -664,12 +681,16 @@ impl SpectreCli { } pub async fn list(&self) -> Result<()> { + let guard = self.wallet.guard(); + let guard = guard.lock().await; + let mut keys = self.wallet.keys().await?; tprintln!(self); while let Some(key) = keys.try_next().await? { tprintln!(self, "• {}", style(&key).dim()); - let mut accounts = self.wallet.accounts(Some(key.id)).await?; + + let mut accounts = self.wallet.accounts(Some(key.id), &guard).await?; while let Some(account) = accounts.try_next().await? { let receive_address = account.receive_address()?; tprintln!(self, " • {}", account.get_list_string()?); @@ -677,7 +698,7 @@ impl SpectreCli { } } - let mut unfiltered_accounts = self.wallet.accounts(None).await?; + let mut unfiltered_accounts = self.wallet.accounts(None, &guard).await?; let mut feature_header_printed = false; while let Some(account) = unfiltered_accounts.try_next().await? { if let Some(feature) = account.feature() { @@ -995,7 +1016,7 @@ mod panic_handler { fn stack(error: &Error) -> String; } - pub fn process(info: &std::panic::PanicInfo) -> String { + pub fn process(info: &std::panic::PanicHookInfo) -> String { let mut msg = info.to_string(); // Add the error stack to our message. @@ -1032,7 +1053,7 @@ mod panic_handler { impl SpectreCli { pub fn init_panic_hook(self: &Arc) { let this = self.clone(); - let handler = move |info: &std::panic::PanicInfo| { + let handler = move |info: &std::panic::PanicHookInfo| { let msg = panic_handler::process(info); this.term().writeln(msg.crlf()); panic_handler::console_error(msg); diff --git a/cli/src/error.rs b/cli/src/error.rs index 3907f59..cc6e824 100644 --- a/cli/src/error.rs +++ b/cli/src/error.rs @@ -128,6 +128,12 @@ pub enum Error { #[error(transparent)] SpectreWalletKeys(#[from] spectre_wallet_keys::error::Error), + + #[error(transparent)] + PssbLockScriptSigError(#[from] spectre_wallet_psst::error::Error), + + #[error("To hex serialization error")] + PssbSerializeToHexError, } impl Error { diff --git a/cli/src/extensions/transaction.rs b/cli/src/extensions/transaction.rs index fca7843..d6c6181 100644 --- a/cli/src/extensions/transaction.rs +++ b/cli/src/extensions/transaction.rs @@ -2,6 +2,7 @@ use crate::imports::*; use spectre_consensus_core::tx::{TransactionInput, TransactionOutpoint}; use spectre_wallet_core::storage::Binding; use spectre_wallet_core::storage::{TransactionData, TransactionKind, TransactionRecord}; +use spectre_wallet_core::wallet::WalletGuard; use workflow_log::style; pub trait TransactionTypeExtension { @@ -48,8 +49,14 @@ impl TransactionTypeExtension for TransactionKind { #[async_trait] pub trait TransactionExtension { - async fn format_transaction(&self, wallet: &Arc, include_utxos: bool) -> Vec; - async fn format_transaction_with_state(&self, wallet: &Arc, state: Option<&str>, include_utxos: bool) -> Vec; + async fn format_transaction(&self, wallet: &Arc, include_utxos: bool, guard: &WalletGuard) -> Vec; + async fn format_transaction_with_state( + &self, + wallet: &Arc, + state: Option<&str>, + include_utxos: bool, + guard: &WalletGuard, + ) -> Vec; async fn format_transaction_with_args( &self, wallet: &Arc, @@ -58,17 +65,24 @@ pub trait TransactionExtension { include_utxos: bool, history: bool, account: Option>, + guard: &WalletGuard, ) -> Vec; } #[async_trait] impl TransactionExtension for TransactionRecord { - async fn format_transaction(&self, wallet: &Arc, include_utxos: bool) -> Vec { - self.format_transaction_with_args(wallet, None, None, include_utxos, false, None).await + async fn format_transaction(&self, wallet: &Arc, include_utxos: bool, guard: &WalletGuard) -> Vec { + self.format_transaction_with_args(wallet, None, None, include_utxos, false, None, guard).await } - async fn format_transaction_with_state(&self, wallet: &Arc, state: Option<&str>, include_utxos: bool) -> Vec { - self.format_transaction_with_args(wallet, state, None, include_utxos, false, None).await + async fn format_transaction_with_state( + &self, + wallet: &Arc, + state: Option<&str>, + include_utxos: bool, + guard: &WalletGuard, + ) -> Vec { + self.format_transaction_with_args(wallet, state, None, include_utxos, false, None, guard).await } async fn format_transaction_with_args( @@ -79,6 +93,7 @@ impl TransactionExtension for TransactionRecord { include_utxos: bool, history: bool, account: Option>, + guard: &WalletGuard, ) -> Vec { let TransactionRecord { id, binding, block_daa_score, transaction_data, .. } = self; @@ -88,7 +103,7 @@ impl TransactionExtension for TransactionRecord { let account = if let Some(account) = account { Some(account) } else { - wallet.get_account_by_id(account_id).await.ok().flatten() + wallet.get_account_by_id(account_id, guard).await.ok().flatten() }; if let Some(account) = account { diff --git a/cli/src/imports.rs b/cli/src/imports.rs index edbf92c..2c8c48f 100644 --- a/cli/src/imports.rs +++ b/cli/src/imports.rs @@ -19,7 +19,7 @@ pub use spectre_utils::hex::*; pub use spectre_wallet_core::compat::*; pub use spectre_wallet_core::prelude::*; pub use spectre_wallet_core::settings::{DefaultSettings, SettingsStore, WalletSettings}; -pub use spectre_wallet_core::utils::*; +pub use spectre_wrpc_client::prelude::*; pub use std::collections::HashMap; pub use std::collections::VecDeque; pub use std::ops::Deref; diff --git a/cli/src/modules/account.rs b/cli/src/modules/account.rs index e09c114..429c681 100644 --- a/cli/src/modules/account.rs +++ b/cli/src/modules/account.rs @@ -68,27 +68,23 @@ impl Account { tprintln!(ctx, ""); ctx.term().help( &[ - ( - "account import legacy-data", - "Import Spectre Desktop keydata file or web wallet data on the same domain", - ), + ("account import legacy-data", "Import Spectre Desktop keydata file or Spectre Network Web Wallet: https://wallet.spectre-network.org/ data on the same domain"), ( "account import mnemonic bip32", - "Import Bip32 (12 or 24 word mnemonics used by spectrewallet, spectre-mobile, etc.)", + "Import Bip32 (12 or 24 word mnemonics used by spectrewallet, spectre-mobile etc.)", ), ( "account import mnemonic legacy", - "Import accounts 12 word mnemonic used by legacy applications (Spectre Desktop and web wallet)", + "Import accounts 12 word mnemonic used by legacy applications (Spectre Desktop and Spectre Network Web Wallet: https://wallet.spectre-network.org/)", ), ( "account import mnemonic multisig [additional keys]", "Import mnemonic and additional keys for a multisig account", ), - ("account import bip32-watch", "Import a extended public key for a watch-only bip32 account"), - ("account import multisig-watch", "Import extended public keys for a watch-only multisig account"), ], None, )?; + return Ok(()); } @@ -150,7 +146,7 @@ impl Account { if argv.is_empty() { tprintln!(ctx, "Usage: 'account import mnemonic '"); tprintln!(ctx, "Please specify the mnemonic type"); - tprintln!(ctx, "Use 'legacy' for 12-word Spectre Desktop and web wallet mnemonics\r\n"); + tprintln!(ctx, "Please use 'legacy' for 12-word Spectre Desktop and Spectre Network Web Wallet: https://wallet.spectre-network.org/ mnemonics\r\n"); return Ok(()); } @@ -169,43 +165,53 @@ impl Account { crate::wizards::import::import_with_mnemonic(&ctx, account_kind, &argv).await?; } _ => { - tprintln!(ctx, "Account import is not supported for this account type: '{}'\r\n", account_kind); + tprintln!(ctx, "account import is not supported for this account type: '{account_kind}'\r\n"); return Ok(()); } } return Ok(()); } - "bip32-watch" => { - let account_name = if argv.is_empty() { - None - } else { - let name = argv.remove(0); - let name = name.trim().to_string(); - Some(name) - }; - - let account_name = account_name.as_deref(); - wizards::account::bip32_watch(&ctx, account_name).await?; + _ => { + tprintln!(ctx, "Unknown account import type: '{import_kind}'"); + tprintln!(ctx, "supported import types are: 'mnemonic', 'legacy-data' or 'multisig-watch'\r\n"); + return Ok(()); } - "multisig-watch" => { - let account_name = if argv.is_empty() { - None - } else { - let name = argv.remove(0); - let name = name.trim().to_string(); + } + } + "watch" => { + if argv.is_empty() { + tprintln!(ctx, "Usage: 'account watch [account name]'"); + tprintln!(ctx, ""); + tprintln!(ctx, "examples:"); + tprintln!(ctx, ""); + ctx.term().help( + &[ + ("account watch bip32", "Import a extended public key for a watch-only bip32 account"), + ("account watch multisig", "Import extended public keys for a watch-only multisig account"), + ], + None, + )?; - Some(name) - }; + return Ok(()); + } - let account_name = account_name.as_deref(); - wizards::account::multisig_watch(&ctx, account_name).await?; + let watch_kind = argv.remove(0); - return Ok(()); + let account_name = argv.first().map(|name| name.trim()).filter(|name| !name.is_empty()).map(|name| name.to_string()); + + let account_name = account_name.as_deref(); + + match watch_kind.as_ref() { + "bip32" => { + wizards::account::bip32_watch(&ctx, account_name).await?; + } + "multisig" => { + wizards::account::multisig_watch(&ctx, account_name).await?; } _ => { - tprintln!(ctx, "Unknown account import type: '{}'", import_kind); - tprintln!(ctx, "Supported import types are: 'mnemonic', 'legacy-data' or 'multisig-watch'\r\n"); + tprintln!(ctx, "Unknown account watch type: '{watch_kind}'"); + tprintln!(ctx, "supported watch types are: 'bip32' or 'multisig'\r\n"); return Ok(()); } } @@ -229,7 +235,7 @@ impl Account { self.derivation_scan(&ctx, start, count, window, sweep).await?; } v => { - tprintln!(ctx, "Unknown command: '{}'\r\n", v); + tprintln!(ctx, "Unknown command: '{v}'\r\n"); return self.display_help(ctx, argv).await; } } @@ -244,14 +250,15 @@ impl Account { ( "import [ [extra keys]]", "Import accounts from a private key using 24 or 12 word mnemonic or legacy data \ - (Spectre Desktop and web wallet). Use 'account import' for additional help.", + (Spectre Desktop and Spectre Network Web Wallet: https://wallet.spectre-network.org/). Use 'account import' for additional help.", ), - ("name ", "Name or rename the selected account (use 'remove' to remove the name)"), + ("name ", "Name or rename the selected account (use 'remove' to remove the name"), ("scan [] or scan [] []", "Scan extended address derivation chain (legacy accounts)"), ( "sweep [] or sweep [] []", "Sweep extended address derivation chain (legacy accounts)", ), + // ("purge", "Purge an account from the wallet"), ], None, )?; diff --git a/cli/src/modules/connect.rs b/cli/src/modules/connect.rs index 51664b7..3bbfa91 100644 --- a/cli/src/modules/connect.rs +++ b/cli/src/modules/connect.rs @@ -1,8 +1,7 @@ use crate::imports::*; -use spectre_wrpc_client::Resolver; #[derive(Default, Handler)] -#[help("Connect to a Spectre network (mainnet or testnet)")] +#[help("Connect to a Spectre Network (mainnet or testnet)")] pub struct Connect; impl Connect { diff --git a/cli/src/modules/details.rs b/cli/src/modules/details.rs index 4a56708..dc7c0fc 100644 --- a/cli/src/modules/details.rs +++ b/cli/src/modules/details.rs @@ -39,6 +39,18 @@ impl Details { tprintln!(ctx.term(), "{:>4}{}", "", style(address.to_string()).blue()); }); + if let Some(xpub_keys) = account.xpub_keys() { + if account.feature().is_some() { + if let Some(feature) = account.feature() { + tprintln!(ctx.term(), "Feature: {}", style(feature).cyan()); + } + tprintln!(ctx.term(), "Extended public keys:"); + xpub_keys.iter().for_each(|xpub| { + tprintln!(ctx.term(), "{:>4}{}", "", style(ctx.wallet().network_format_xpub(xpub)).dim()); + }); + } + } + Ok(()) } } diff --git a/cli/src/modules/history.rs b/cli/src/modules/history.rs index d26f236..072026a 100644 --- a/cli/src/modules/history.rs +++ b/cli/src/modules/history.rs @@ -11,6 +11,9 @@ impl History { async fn main(self: Arc, ctx: &Arc, mut argv: Vec, _cmd: &str) -> Result<()> { let ctx = ctx.clone().downcast_arc::()?; + let guard = ctx.wallet().guard(); + let guard = guard.lock().await; + if argv.is_empty() { self.display_help(ctx, argv).await?; return Ok(()); @@ -33,7 +36,15 @@ impl History { match store.load_single(&binding, &network_id, &txid).await { Ok(tx) => { let lines = tx - .format_transaction_with_args(&ctx.wallet(), None, current_daa_score, true, true, Some(account.clone())) + .format_transaction_with_args( + &ctx.wallet(), + None, + current_daa_score, + true, + true, + Some(account.clone()), + &guard, + ) .await; lines.iter().for_each(|line| tprintln!(ctx, "{line}")); } @@ -113,6 +124,7 @@ impl History { include_utxo, true, Some(account.clone()), + &guard, ) .await; lines.iter().for_each(|line| tprintln!(ctx, "{line}")); diff --git a/cli/src/modules/mod.rs b/cli/src/modules/mod.rs index 829702c..7632239 100644 --- a/cli/src/modules/mod.rs +++ b/cli/src/modules/mod.rs @@ -26,6 +26,7 @@ pub mod network; pub mod node; pub mod open; pub mod ping; +pub mod pssb; pub mod reload; pub mod rpc; pub mod select; @@ -57,7 +58,7 @@ pub fn register_handlers(cli: &Arc) -> Result<()> { cli.handlers(), [ account, address, close, connect, details, disconnect, estimate, exit, export, guide, help, history, rpc, list, miner, - message, monitor, mute, network, node, open, ping, reload, select, send, server, settings, sweep, track, transfer, + message, monitor, mute, network, node, open, ping, pssb, reload, select, send, server, settings, sweep, track, transfer, wallet, // halt, // theme, start, stop diff --git a/cli/src/modules/pssb.rs b/cli/src/modules/pssb.rs new file mode 100644 index 0000000..c2019ed --- /dev/null +++ b/cli/src/modules/pssb.rs @@ -0,0 +1,266 @@ +#![allow(unused_imports)] + +use crate::imports::*; +use spectre_addresses::Prefix; +use spectre_consensus_core::tx::{TransactionOutpoint, UtxoEntry}; +use spectre_wallet_core::account::pssb::finalize_psst_one_or_more_sig_and_redeem_script; +use spectre_wallet_psst::{ + prelude::{lock_script_sig_templating, script_sig_to_address, unlock_utxos_as_pssb, Bundle, Signer, PSST}, + psst::Inner, +}; + +#[derive(Default, Handler)] +#[help("Send a Spectre transaction to a public address")] +pub struct Pssb; + +impl Pssb { + async fn main(self: Arc, ctx: &Arc, mut argv: Vec, _cmd: &str) -> Result<()> { + let ctx = ctx.clone().downcast_arc::()?; + + if !ctx.wallet().is_open() { + return Err(Error::WalletIsNotOpen); + } + + if argv.is_empty() { + return self.display_help(ctx, argv).await; + } + + let action = argv.remove(0); + + match action.as_str() { + "create" => { + if argv.len() < 2 || argv.len() > 3 { + return self.display_help(ctx, argv).await; + } + let (wallet_secret, payment_secret) = ctx.ask_wallet_secret(None).await?; + let _ = ctx.notifier().show(Notification::Processing).await; + + let address = Address::try_from(argv.first().unwrap().as_str())?; + let amount_sompi = try_parse_required_nonzero_spectre_as_sompi_u64(argv.get(1))?; + let outputs = PaymentOutputs::from((address, amount_sompi)); + let priority_fee_sompi = try_parse_optional_spectre_as_sompi_i64(argv.get(2))?.unwrap_or(0); + let abortable = Abortable::default(); + + let account: Arc = ctx.wallet().account()?; + let signer = account + .pssb_from_send_generator( + outputs.into(), + priority_fee_sompi.into(), + None, + wallet_secret.clone(), + payment_secret.clone(), + &abortable, + ) + .await?; + + match signer.serialize() { + Ok(encoded) => tprintln!(ctx, "{encoded}"), + Err(e) => return Err(e.into()), + } + } + "script" => { + if argv.len() < 2 || argv.len() > 4 { + return self.display_help(ctx, argv).await; + } + let subcommand = argv.remove(0); + let payload = argv.remove(0); + let account = ctx.wallet().account()?; + let receive_address = account.receive_address()?; + let (wallet_secret, payment_secret) = ctx.ask_wallet_secret(None).await?; + let _ = ctx.notifier().show(Notification::Processing).await; + + let script_sig = match lock_script_sig_templating(payload.clone(), Some(&receive_address.payload)) { + Ok(value) => value, + Err(e) => { + terrorln!(ctx, "{}", e.to_string()); + return Err(e.into()); + } + }; + + let script_p2sh = match script_sig_to_address(&script_sig, ctx.wallet().address_prefix()?) { + Ok(p2sh) => p2sh, + Err(e) => { + terrorln!(ctx, "Error generating script address: {}", e.to_string()); + return Err(e.into()); + } + }; + + match subcommand.as_str() { + "lock" => { + let amount_sompi = try_parse_required_nonzero_spectre_as_sompi_u64(argv.first())?; + let outputs = PaymentOutputs::from((script_p2sh, amount_sompi)); + let priority_fee_sompi = try_parse_optional_spectre_as_sompi_i64(argv.get(1))?.unwrap_or(0); + let abortable = Abortable::default(); + + let signer = account + .pssb_from_send_generator( + outputs.into(), + priority_fee_sompi.into(), + None, + wallet_secret.clone(), + payment_secret.clone(), + &abortable, + ) + .await?; + + match signer.serialize() { + Ok(encoded) => tprintln!(ctx, "{encoded}"), + Err(e) => return Err(e.into()), + } + } + "unlock" => { + if argv.len() != 1 { + return self.display_help(ctx, argv).await; + } + + // Get locked UTXO set. + let spend_utxos: Vec = + ctx.wallet().rpc_api().get_utxos_by_addresses(vec![script_p2sh.clone()]).await?; + let priority_fee_sompi = try_parse_optional_spectre_as_sompi_i64(argv.first())?.unwrap_or(0) as u64; + + if spend_utxos.is_empty() { + twarnln!(ctx, "No locked UTXO set found."); + return Ok(()); + } + + let references: Vec<(UtxoEntry, TransactionOutpoint)> = + spend_utxos.iter().map(|entry| (entry.utxo_entry.clone().into(), entry.outpoint.into())).collect(); + + let total_locked_sompi: u64 = spend_utxos.iter().map(|entry| entry.utxo_entry.amount).sum(); + + tprintln!( + ctx, + "{} locked UTXO{} found with total amount of {} SPR", + spend_utxos.len(), + if spend_utxos.len() == 1 { "" } else { "s" }, + sompi_to_spectre(total_locked_sompi) + ); + + // Sweep UTXO set. + match unlock_utxos_as_pssb(references, &receive_address, script_sig, priority_fee_sompi as u64) { + Ok(pssb) => { + let pssb_hex = pssb.serialize()?; + tprintln!(ctx, "{pssb_hex}"); + } + Err(e) => tprintln!(ctx, "Error generating unlock PSSB: {}", e.to_string()), + } + } + "sign" => { + let pssb = Self::parse_input_pssb(argv.first().unwrap().as_str())?; + + // Sign PSSB using the account's receiver address. + match account.pssb_sign(&pssb, wallet_secret.clone(), payment_secret.clone(), Some(&receive_address)).await { + Ok(signed_pssb) => { + let pssb_pack = String::try_from(signed_pssb)?; + tprintln!(ctx, "{pssb_pack}"); + } + Err(e) => terrorln!(ctx, "{}", e.to_string()), + } + } + "address" => { + tprintln!(ctx, "\r\nP2SH address: {}", script_p2sh); + } + v => { + terrorln!(ctx, "unknown command: '{v}'\r\n"); + return self.display_help(ctx, argv).await; + } + } + } + "sign" => { + if argv.len() != 1 { + return self.display_help(ctx, argv).await; + } + let (wallet_secret, payment_secret) = ctx.ask_wallet_secret(None).await?; + let pssb = Self::parse_input_pssb(argv.first().unwrap().as_str())?; + let account = ctx.wallet().account()?; + match account.pssb_sign(&pssb, wallet_secret.clone(), payment_secret.clone(), None).await { + Ok(signed_pssb) => { + let pssb_pack = String::try_from(signed_pssb)?; + tprintln!(ctx, "{pssb_pack}"); + } + Err(e) => terrorln!(ctx, "{}", e.to_string()), + } + } + "send" => { + if argv.len() != 1 { + return self.display_help(ctx, argv).await; + } + let pssb = Self::parse_input_pssb(argv.first().unwrap().as_str())?; + let account = ctx.wallet().account()?; + match account.pssb_broadcast(&pssb).await { + Ok(sent) => tprintln!(ctx, "Sent transactions {:?}", sent), + Err(e) => terrorln!(ctx, "Send error {:?}", e), + } + } + "debug" => { + if argv.len() != 1 { + return self.display_help(ctx, argv).await; + } + let pssb = Self::parse_input_pssb(argv.first().unwrap().as_str())?; + tprintln!(ctx, "{:?}", pssb); + } + "parse" => { + if argv.len() != 1 { + return self.display_help(ctx, argv).await; + } + let pssb = Self::parse_input_pssb(argv.first().unwrap().as_str())?; + tprintln!(ctx, "{}", pssb.display_format(ctx.wallet().network_id()?, sompi_to_spectre_string_with_suffix)); + + for (psst_index, bundle_inner) in pssb.0.iter().enumerate() { + tprintln!(ctx, "PSST #{:03} finalized check:", psst_index + 1); + let psst: PSST = PSST::::from(bundle_inner.to_owned()); + + let finalizer = psst.finalizer(); + + if let Ok(psst_finalizer) = finalize_psst_one_or_more_sig_and_redeem_script(finalizer) { + // Verify if extraction is possible. + match psst_finalizer.extractor() { + Ok(ex) => match ex.extract_tx() { + Ok(_) => tprintln!( + ctx, + " Transaction extracted successfully: PSST is finalized with a valid script signature." + ), + Err(e) => terrorln!(ctx, " PSST transaction extraction error: {}", e.to_string()), + }, + Err(_) => twarnln!(ctx, " PSST not finalized"), + } + } else { + twarnln!(ctx, " PSST not signed"); + } + } + } + v => { + tprintln!(ctx, "unknown command: '{v}'\r\n"); + return self.display_help(ctx, argv).await; + } + } + Ok(()) + } + + fn parse_input_pssb(input: &str) -> Result { + match Bundle::try_from(input) { + Ok(bundle) => Ok(bundle), + Err(e) => Err(Error::custom(format!("Error while parsing input PSSB {}", e))), + } + } + + async fn display_help(self: Arc, ctx: Arc, _argv: Vec) -> Result<()> { + ctx.term().help( + &[ + ("pssb create
", "Create a PSSB from single send transaction"), + ("pssb sign ", "Sign given PSSB"), + ("pssb send ", "Broadcast bundled transactions"), + ("pssb debug ", "Print PSSB debug view"), + ("pssb parse ", "Print PSSB formatted view"), + ("pssb script lock [priority fee]", "Generate a PSSB with one send transaction to given P2SH payload. Optional public key placeholder in payload: {{pubkey}}"), + ("pssb script unlock ", "Generate a PSSB to unlock UTXOS one by one from given P2SH payload. Fee amount will be applied to every spent UTXO, meaning every transaction. Optional public key placeholder in payload: {{pubkey}}"), + ("pssb script sign ", "Sign all PSSB's P2SH locked inputs"), + ("pssb script sign ", "Sign all PSSB's P2SH locked inputs"), + ("pssb script address ", "Prints P2SH address"), + ], + None, + )?; + + Ok(()) + } +} diff --git a/cli/src/modules/pssb_redeem.json b/cli/src/modules/pssb_redeem.json new file mode 100644 index 0000000..9f3e00d --- /dev/null +++ b/cli/src/modules/pssb_redeem.json @@ -0,0 +1,99 @@ +{ + "inner_list": [ + { + "global": { + "version": 0, + "tx_version": 0, + "fallback_lock_time": null, + "inputs_modifiable": false, + "outputs_modifiable": false, + "input_count": 1, + "output_count": 0, + "xpubs": {}, + "id": null, + "proprietaries": {} + }, + "inputs": [ + { + "utxo_entry": { + "amount": 3300000000, + "scriptPublicKey": "000020f2b9984ea0bf9464b028534a51c84dca8d8e8037499ce9a01f43d717dc4b157cac", + "blockDaaScore": 160987746, + "isCoinbase": false + }, + "previous_outpoint": { + "transactionId": "6323bee26238b82f926176c08b8919c42cdb204a14c17fd35c0e2664326133ff", + "index": 0 + }, + "sequence": null, + "min_time": null, + "partial_sigs": {}, + "sighash_type": 1, + "redeem_script": null, + "sig_op_count": 1, + "bip32_derivations": {}, + "final_script_sig": null, + "proprietaries": {} + }, + { + "utxo_entry": { + "amount": 33210791745569, + "scriptPublicKey": "0000200e6227c375a68b02da57c58f8eaf837a003c59e957890baf1cf18aaf28aca76dac", + "blockDaaScore": 160987746, + "isCoinbase": false + }, + "previous_outpoint": { + "transactionId": "6323bee26238b82f926176c08b8919c42cdb204a14c17fd35c0e2664326133ff", + "index": 1 + }, + "sequence": null, + "min_time": null, + "partial_sigs": {}, + "sighash_type": 1, + "redeem_script": null, + "sig_op_count": 1, + "bip32_derivations": {}, + "final_script_sig": null, + "proprietaries": {} + }, + { + "utxo_entry": { + "amount": 3300000000, + "scriptPublicKey": "000020f2b9984ea0bf9464b028534a51c84dca8d8e8037499ce9a01f43d717dc4b157cac", + "blockDaaScore": 160987746, + "isCoinbase": false + }, + "previous_outpoint": { + "transactionId": "w", + "index": 0 + }, + "sequence": null, + "min_time": null, + "partial_sigs": {}, + "sighash_type": 1, + "redeem_script": "ac", + "sig_op_count": 1, + "bip32_derivations": {}, + "final_script_sig": null, + "proprietaries": {} + } + ], + "outputs": [ + { + "amount": 6400000000, + "script_public_key": "000020f2b9984ea0bf9464b028534a51c84dca8d8e8037499ce9a01f43d717dc4b157cac", + "redeem_script": null, + "bip32_derivations": {}, + "proprietaries": {} + }, + { + "amount": 33207691742259, + "script_public_key": "0000200e6227c375a68b02da57c58f8eaf837a003c59e957890baf1cf18aaf28aca76dac", + "redeem_script": null, + "bip32_derivations": {}, + "proprietaries": {} + } + ] + } + ] +} \ No newline at end of file diff --git a/cli/src/modules/reload.rs b/cli/src/modules/reload.rs index 320b65c..234a14a 100644 --- a/cli/src/modules/reload.rs +++ b/cli/src/modules/reload.rs @@ -10,8 +10,12 @@ impl Reload { // workflow_dom::utils::window().location().reload().ok(); let ctx = ctx.clone().downcast_arc::()?; + + let guard = ctx.wallet().guard(); + let guard = guard.lock().await; + tprintln!(ctx, "{}", style("reloading wallet ...").magenta()); - ctx.wallet().reload(true).await?; + ctx.wallet().reload(true, &guard).await?; Ok(()) } diff --git a/cli/src/modules/rpc.rs b/cli/src/modules/rpc.rs index 47cd1a8..a88393c 100644 --- a/cli/src/modules/rpc.rs +++ b/cli/src/modules/rpc.rs @@ -1,6 +1,6 @@ use crate::imports::*; use convert_case::{Case, Casing}; -use spectre_rpc_core::{api::ops::RpcApiOps, *}; +use spectre_rpc_core::api::ops::RpcApiOps; #[derive(Default, Handler)] #[help("Execute RPC commands against the connected Spectre node")] @@ -38,19 +38,27 @@ impl Rpc { tprintln!(ctx, "ok"); } RpcApiOps::GetMetrics => { - let result = rpc.get_metrics(true, true, true, true).await?; + let result = rpc.get_metrics(true, true, true, true, true, true).await?; + self.println(&ctx, result); + } + RpcApiOps::GetSystemInfo => { + let result = rpc.get_system_info().await?; + self.println(&ctx, result); + } + RpcApiOps::GetConnections => { + let result = rpc.get_connections(true).await?; self.println(&ctx, result); } RpcApiOps::GetServerInfo => { - let result = rpc.get_server_info_call(GetServerInfoRequest {}).await?; + let result = rpc.get_server_info_call(None, GetServerInfoRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetSyncStatus => { - let result = rpc.get_sync_status_call(GetSyncStatusRequest {}).await?; + let result = rpc.get_sync_status_call(None, GetSyncStatusRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetCurrentNetwork => { - let result = rpc.get_current_network_call(GetCurrentNetworkRequest {}).await?; + let result = rpc.get_current_network_call(None, GetCurrentNetworkRequest {}).await?; self.println(&ctx, result); } // RpcApiOps::SubmitBlock => { @@ -62,11 +70,11 @@ impl Rpc { // self.println(&ctx, result); // } RpcApiOps::GetPeerAddresses => { - let result = rpc.get_peer_addresses_call(GetPeerAddressesRequest {}).await?; + let result = rpc.get_peer_addresses_call(None, GetPeerAddressesRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetSink => { - let result = rpc.get_sink_call(GetSinkRequest {}).await?; + let result = rpc.get_sink_call(None, GetSinkRequest {}).await?; self.println(&ctx, result); } // RpcApiOps::GetMempoolEntry => { @@ -76,12 +84,15 @@ impl Rpc { RpcApiOps::GetMempoolEntries => { // TODO let result = rpc - .get_mempool_entries_call(GetMempoolEntriesRequest { include_orphan_pool: true, filter_transaction_pool: true }) + .get_mempool_entries_call( + None, + GetMempoolEntriesRequest { include_orphan_pool: true, filter_transaction_pool: true }, + ) .await?; self.println(&ctx, result); } RpcApiOps::GetConnectedPeerInfo => { - let result = rpc.get_connected_peer_info_call(GetConnectedPeerInfoRequest {}).await?; + let result = rpc.get_connected_peer_info_call(None, GetConnectedPeerInfoRequest {}).await?; self.println(&ctx, result); } RpcApiOps::AddPeer => { @@ -90,7 +101,7 @@ impl Rpc { } let peer_address = argv.remove(0).parse::()?; let is_permanent = argv.remove(0).parse::().unwrap_or(false); - let result = rpc.add_peer_call(AddPeerRequest { peer_address, is_permanent }).await?; + let result = rpc.add_peer_call(None, AddPeerRequest { peer_address, is_permanent }).await?; self.println(&ctx, result); } // RpcApiOps::SubmitTransaction => { @@ -103,7 +114,7 @@ impl Rpc { } let hash = argv.remove(0); let hash = RpcHash::from_hex(hash.as_str())?; - let result = rpc.get_block_call(GetBlockRequest { hash, include_transactions: true }).await?; + let result = rpc.get_block_call(None, GetBlockRequest { hash, include_transactions: true }).await?; self.println(&ctx, result); } // RpcApiOps::GetSubnetwork => { @@ -119,11 +130,11 @@ impl Rpc { // self.println(&ctx, result); // } RpcApiOps::GetBlockCount => { - let result = rpc.get_block_count_call(GetBlockCountRequest {}).await?; + let result = rpc.get_block_count_call(None, GetBlockCountRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetBlockDagInfo => { - let result = rpc.get_block_dag_info_call(GetBlockDagInfoRequest {}).await?; + let result = rpc.get_block_dag_info_call(None, GetBlockDagInfoRequest {}).await?; self.println(&ctx, result); } // RpcApiOps::ResolveFinalityConflict => { @@ -131,7 +142,7 @@ impl Rpc { // self.println(&ctx, result); // } RpcApiOps::Shutdown => { - let result = rpc.shutdown_call(ShutdownRequest {}).await?; + let result = rpc.shutdown_call(None, ShutdownRequest {}).await?; self.println(&ctx, result); } // RpcApiOps::GetHeaders => { @@ -143,7 +154,7 @@ impl Rpc { return Err(Error::custom("Please specify at least one address")); } let addresses = argv.iter().map(|s| Address::try_from(s.as_str())).collect::, _>>()?; - let result = rpc.get_utxos_by_addresses_call(GetUtxosByAddressesRequest { addresses }).await?; + let result = rpc.get_utxos_by_addresses_call(None, GetUtxosByAddressesRequest { addresses }).await?; self.println(&ctx, result); } RpcApiOps::GetBalanceByAddress => { @@ -152,7 +163,7 @@ impl Rpc { } let addresses = argv.iter().map(|s| Address::try_from(s.as_str())).collect::, _>>()?; for address in addresses { - let result = rpc.get_balance_by_address_call(GetBalanceByAddressRequest { address }).await?; + let result = rpc.get_balance_by_address_call(None, GetBalanceByAddressRequest { address }).await?; self.println(&ctx, sompi_to_spectre(result.balance)); } } @@ -161,11 +172,11 @@ impl Rpc { return Err(Error::custom("Please specify at least one address")); } let addresses = argv.iter().map(|s| Address::try_from(s.as_str())).collect::, _>>()?; - let result = rpc.get_balances_by_addresses_call(GetBalancesByAddressesRequest { addresses }).await?; + let result = rpc.get_balances_by_addresses_call(None, GetBalancesByAddressesRequest { addresses }).await?; self.println(&ctx, result); } RpcApiOps::GetSinkBlueScore => { - let result = rpc.get_sink_blue_score_call(GetSinkBlueScoreRequest {}).await?; + let result = rpc.get_sink_blue_score_call(None, GetSinkBlueScoreRequest {}).await?; self.println(&ctx, result); } RpcApiOps::Ban => { @@ -173,7 +184,7 @@ impl Rpc { return Err(Error::custom("Please specify peer IP address")); } let ip: RpcIpAddress = argv.remove(0).parse()?; - let result = rpc.ban_call(BanRequest { ip }).await?; + let result = rpc.ban_call(None, BanRequest { ip }).await?; self.println(&ctx, result); } RpcApiOps::Unban => { @@ -181,11 +192,11 @@ impl Rpc { return Err(Error::custom("Please specify peer IP address")); } let ip: RpcIpAddress = argv.remove(0).parse()?; - let result = rpc.unban_call(UnbanRequest { ip }).await?; + let result = rpc.unban_call(None, UnbanRequest { ip }).await?; self.println(&ctx, result); } RpcApiOps::GetInfo => { - let result = rpc.get_info_call(GetInfoRequest {}).await?; + let result = rpc.get_info_call(None, GetInfoRequest {}).await?; self.println(&ctx, result); } // RpcApiOps::EstimateNetworkHashesPerSecond => { @@ -200,16 +211,15 @@ impl Rpc { let include_orphan_pool = true; let filter_transaction_pool = true; let result = rpc - .get_mempool_entries_by_addresses_call(GetMempoolEntriesByAddressesRequest { - addresses, - include_orphan_pool, - filter_transaction_pool, - }) + .get_mempool_entries_by_addresses_call( + None, + GetMempoolEntriesByAddressesRequest { addresses, include_orphan_pool, filter_transaction_pool }, + ) .await?; self.println(&ctx, result); } RpcApiOps::GetCoinSupply => { - let result = rpc.get_coin_supply_call(GetCoinSupplyRequest {}).await?; + let result = rpc.get_coin_supply_call(None, GetCoinSupplyRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetDaaScoreTimestampEstimate => { @@ -220,8 +230,9 @@ impl Rpc { match daa_score_result { Ok(daa_scores) => { - let result = - rpc.get_daa_score_timestamp_estimate_call(GetDaaScoreTimestampEstimateRequest { daa_scores }).await?; + let result = rpc + .get_daa_score_timestamp_estimate_call(None, GetDaaScoreTimestampEstimateRequest { daa_scores }) + .await?; self.println(&ctx, result); } Err(_err) => { @@ -230,12 +241,12 @@ impl Rpc { } } RpcApiOps::GetFeeEstimate => { - let result = rpc.get_fee_estimate_call(GetFeeEstimateRequest {}).await?; + let result = rpc.get_fee_estimate_call(None, GetFeeEstimateRequest {}).await?; self.println(&ctx, result); } RpcApiOps::GetFeeEstimateExperimental => { let verbose = if argv.is_empty() { false } else { argv.remove(0).parse().unwrap_or(false) }; - let result = rpc.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await?; + let result = rpc.get_fee_estimate_experimental_call(None, GetFeeEstimateExperimentalRequest { verbose }).await?; self.println(&ctx, result); } _ => { @@ -252,9 +263,8 @@ impl Rpc { async fn display_help(self: Arc, ctx: Arc, _argv: Vec) -> Result<()> { // RpcApiOps that do not contain docs are not displayed - let help = RpcApiOps::list() - .iter() - .filter_map(|op| op.doc().is_not_empty().then_some((op.as_str().to_case(Case::Kebab).to_string(), op.doc()))) + let help = RpcApiOps::into_iter() + .filter_map(|op| op.rustdoc().is_not_empty().then_some((op.as_str().to_case(Case::Kebab).to_string(), op.rustdoc()))) .collect::>(); ctx.term().help(&help, None)?; diff --git a/cli/src/modules/send.rs b/cli/src/modules/send.rs index 525b293..2247f8d 100644 --- a/cli/src/modules/send.rs +++ b/cli/src/modules/send.rs @@ -39,7 +39,7 @@ impl Send { .await?; tprintln!(ctx, "Transaction sent - {summary}"); - // tprintln!(ctx, "\nSending {} SPR to {address}, transaction IDs:", sompi_to_spectre_string(amount_sompi)); + tprintln!(ctx, "\nSending {} SPR to {address}, transaction IDs:", sompi_to_spectre_string(amount_sompi)); // tprintln!(ctx, "{}\n", ids.into_iter().map(|a| a.to_string()).collect::>().join("\n")); Ok(()) diff --git a/cli/src/modules/settings.rs b/cli/src/modules/settings.rs index d78cc91..d82c377 100644 --- a/cli/src/modules/settings.rs +++ b/cli/src/modules/settings.rs @@ -9,12 +9,11 @@ impl Settings { let ctx = ctx.clone().downcast_arc::()?; tprintln!(ctx, "\nSettings:\n"); - let list = WalletSettings::list(); - let list = list - .iter() + // let list = WalletSettings::list(); + let list = WalletSettings::into_iter() .map(|setting| { let value: String = ctx.wallet().settings().get(setting.clone()).unwrap_or_else(|| "-".to_string()); - let descr = setting.descr(); + let descr = setting.describe(); (setting.as_str().to_lowercase(), value, descr) }) .collect::>(); diff --git a/cli/src/modules/wallet.rs b/cli/src/modules/wallet.rs index ff4f174..91ea18f 100644 --- a/cli/src/modules/wallet.rs +++ b/cli/src/modules/wallet.rs @@ -9,6 +9,9 @@ impl Wallet { async fn main(self: Arc, ctx: &Arc, mut argv: Vec, cmd: &str) -> Result<()> { let ctx = ctx.clone().downcast_arc::()?; + let guard = ctx.wallet().guard(); + let guard = guard.lock().await; + if argv.is_empty() { return self.display_help(ctx, argv).await; } @@ -43,8 +46,8 @@ impl Wallet { }; let wallet_name = wallet_name.as_deref(); - let import_with_mnemonic = op == "import"; - wizards::wallet::create(&ctx, wallet_name, import_with_mnemonic).await?; + let import_with_mnemonic = op.as_str() == "import"; + wizards::wallet::create(&ctx, guard.into(), wallet_name, import_with_mnemonic).await?; } "open" => { let name = if let Some(name) = argv.first().cloned() { @@ -61,8 +64,8 @@ impl Wallet { let (wallet_secret, _) = ctx.ask_wallet_secret(None).await?; let _ = ctx.notifier().show(Notification::Processing).await; let args = WalletOpenArgs::default_with_legacy_accounts(); - ctx.wallet().open(&wallet_secret, name, args).await?; - ctx.wallet().activate_accounts(None).await?; + ctx.wallet().open(&wallet_secret, name, args, &guard).await?; + ctx.wallet().activate_accounts(None, &guard).await?; } "close" => { ctx.wallet().close().await?; @@ -70,13 +73,14 @@ impl Wallet { "hint" => { if !argv.is_empty() { let re = regex::Regex::new(r"wallet\s+hint\s+").unwrap(); - let hint = re.replace(cmd, "").trim().to_string(); + let hint = re.replace(cmd, ""); + let hint = hint.trim(); let store = ctx.store(); if hint == "remove" { tprintln!(ctx, "Hint is empty - removing wallet hint"); store.set_user_hint(None).await?; } else { - store.set_user_hint(Some(spectre_wallet_core::storage::Hint { text: hint })).await?; + store.set_user_hint(Some(hint.into())).await?; } } else { tprintln!(ctx, "Usage:\n'wallet hint ' or 'wallet hint remove' to remove the hint"); diff --git a/cli/src/wizards/wallet.rs b/cli/src/wizards/wallet.rs index 28e61ef..9e4ce77 100644 --- a/cli/src/wizards/wallet.rs +++ b/cli/src/wizards/wallet.rs @@ -2,20 +2,34 @@ use crate::cli::SpectreCli; use crate::imports::*; use crate::result::Result; use spectre_bip32::{Language, Mnemonic, WordCount}; -use spectre_wallet_core::storage::{make_filename, Hint}; - -pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_with_mnemonic: bool) -> Result<()> { +use spectre_wallet_core::{ + storage::{make_filename, Hint}, + wallet::WalletGuard, +}; + +pub(crate) async fn create( + ctx: &Arc, + wallet_guard: Option>, + name: Option<&str>, + import_with_mnemonic: bool, +) -> Result<()> { let term = ctx.term(); let wallet = ctx.wallet(); + let local_guard = ctx.wallet().guard(); + + let guard = match wallet_guard { + Some(locked_guard) => locked_guard, + None => local_guard.lock().await, + }; - // TODO + // TODO @aspect let word_count = WordCount::Words12; if let Err(err) = wallet.network_id() { tprintln!(ctx); tprintln!(ctx, "Before creating a wallet, you need to select a Spectre network."); tprintln!(ctx, "Please use the 'network ' command to select a network."); - tprintln!(ctx, "Currently available networks: 'mainnet'"); + tprintln!(ctx, "Currently available networks: 'mainnet' and 'testnet'"); tprintln!(ctx); return Err(err.into()); } @@ -37,11 +51,19 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit tpara!( ctx, - "\n\"Phishing hint\" is a secret word or phrase displayed when you open your wallet. If you do not see the hint when opening your wallet, you may be accessing a fake wallet designed to steal your private key. If this occurs, stop using the wallet immediately, check the browser URL domain name, and seek help on social networks (Spectre Discord or Telegram).\n" + "\n\ + \"Phishing hint\" is a secret word or a phrase that is displayed \ + when you open your wallet. If you do not see the hint when opening \ + your wallet, you may be accessing a fake wallet designed to steal \ + your private key. If this occurs, stop using the wallet immediately, \ + check the browser URL domain name and seek help on social networks \ + (Spectre Discord or Telegram). \ + \n\ + ", ); let hint = term.ask(false, "Create phishing hint (optional, press to skip): ").await?.trim().to_string(); - let hint = if hint.is_empty() { None } else { Some(Hint::from(hint)) }; + let hint = hint.is_not_empty().then_some(hint).map(Hint::from); let wallet_secret = Secret::new(term.ask(true, "Enter wallet encryption password: ").await?.trim().as_bytes().to_vec()); if wallet_secret.as_ref().is_empty() { @@ -54,15 +76,30 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit } tprintln!(ctx, ""); + if import_with_mnemonic { tpara!( ctx, - "If your original wallet has a bip39 recovery passphrase, please enter it now. This is not a wallet password. This is a secondary mnemonic passphrase used to encrypt your mnemonic. This is known as a 'payment passphrase', 'mnemonic passphrase', or 'recovery passphrase'. If your mnemonic was created with a payment passphrase and you do not enter it now, the import process will generate a different private key. If you do not have a bip39 recovery passphrase, press ENTER." + "\ + If your original wallet has a bip39 recovery passphrase, please enter it now.\ + Specifically, this is not a wallet password. This is a secondary mnemonic passphrase\ + used to encrypt your mnemonic. This is known as a 'payment passphrase',\ + 'mnemonic passphrase', or a 'recovery passphrase'. If your mnemonic was created\ + with a payment passphrase and you do not enter it now, the import process\ + will generate a different private key.\ + If you do not have a bip39 recovery passphrase, press ENTER.\ + ", ); } else { tpara!( ctx, - "PLEASE NOTE: The optional bip39 mnemonic passphrase, if provided, will be required to issue transactions. This passphrase will also be required when recovering your wallet in addition to your private key or mnemonic. If you lose this passphrase, you will not be able to use or recover your wallet! If you do not want to use a bip39 recovery passphrase, press ENTER." + "\ + PLEASE NOTE: The optional bip39 mnemonic passphrase, if provided, will be required to \ + issue transactions. This passphrase will also be required when recovering your wallet \ + in addition to your private key or mnemonic. If you lose this passphrase, you will not \ + be able to use or recover your wallet! \ + If you do not want to use bip39 recovery passphrase, press ENTER.\ + ", ); } @@ -92,7 +129,6 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit }; let mnemonic_phrase = prv_key_data_args.mnemonic.clone(); - let notifier = ctx.notifier().show(Notification::Processing).await; // suspend commits for multiple operations @@ -107,7 +143,6 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit // flush data to storage wallet.store().flush(&wallet_secret).await?; - notifier.hide(); if !import_with_mnemonic { @@ -119,7 +154,14 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit tpara!( ctx, - "Your mnemonic phrase allows you to recreate your private key. The person who has access to this mnemonic will have full control of the Spectre stored in it. Keep your mnemonic safe. Write it down and store it in a safe, preferably in a fire-resistant location. Do not store your mnemonic on this computer or a mobile device. This wallet will never ask you for this mnemonic phrase unless you manually initiate a private key recovery." + "Your mnemonic phrase allows you to re-create your private key. \ + The person who has access to this mnemonic will have full control of \ + the Spectre stored in it. Keep your mnemonic safe. Write it down and \ + store it in a safe, preferably in a fire-resistant location. Do not \ + store your mnemonic on this computer or a mobile device. This wallet \ + will never ask you for this mnemonic phrase unless you manually \ + initiate a private key recovery. \ + ", ); ["", "Never share your mnemonic with anyone!", "---", "", "Your default wallet account mnemonic:", mnemonic_phrase.as_str()?] @@ -136,8 +178,8 @@ pub(crate) async fn create(ctx: &Arc, name: Option<&str>, import_wit term.writeln(style(receive_address).blue().to_string()); term.writeln(""); - wallet.open(&wallet_secret, name.map(String::from), WalletOpenArgs::default_with_legacy_accounts()).await?; - wallet.activate_accounts(None).await?; + wallet.open(&wallet_secret, name.map(String::from), WalletOpenArgs::default_with_legacy_accounts(), &guard).await?; + wallet.activate_accounts(None, &guard).await?; Ok(()) } diff --git a/consensus/client/Cargo.toml b/consensus/client/Cargo.toml index cb30a93..0ab7564 100644 --- a/consensus/client/Cargo.toml +++ b/consensus/client/Cargo.toml @@ -38,5 +38,5 @@ itertools.workspace = true workflow-wasm.workspace = true workflow-log.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/consensus/client/src/header.rs b/consensus/client/src/header.rs index 72b9a11..122207d 100644 --- a/consensus/client/src/header.rs +++ b/consensus/client/src/header.rs @@ -37,7 +37,7 @@ export interface IHeader { #[wasm_bindgen] extern "C" { #[wasm_bindgen(typescript_type = "IHeader | Header")] - pub type IHeader; + pub type HeaderT; } /// @category Consensus @@ -64,7 +64,7 @@ impl Header { #[wasm_bindgen] impl Header { #[wasm_bindgen(constructor)] - pub fn constructor(js_value: IHeader) -> std::result::Result { + pub fn constructor(js_value: HeaderT) -> std::result::Result { Ok(js_value.try_into_owned()?) } @@ -232,8 +232,11 @@ impl Header { impl TryCastFromJs for Header { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(object) = Object::try_from(value.as_ref()) { let parents_by_level = object .get_vec("parentsByLevel")? diff --git a/consensus/client/src/input.rs b/consensus/client/src/input.rs index 32e00a8..2295427 100644 --- a/consensus/client/src/input.rs +++ b/consensus/client/src/input.rs @@ -13,7 +13,7 @@ const TS_TRANSACTION: &'static str = r#" */ export interface ITransactionInput { previousOutpoint: ITransactionOutpoint; - signatureScript: HexString; + signatureScript?: HexString; sequence: bigint; sigOpCount: number; utxo?: UtxoEntryReference; @@ -33,15 +33,19 @@ export interface ITransactionInputVerboseData { } #[wasm_bindgen] extern "C" { - #[wasm_bindgen(typescript_type = "ITransactionInput")] - pub type ITransactionInput; + #[wasm_bindgen(typescript_type = "ITransactionInput | TransactionInput")] + pub type TransactionInputT; + #[wasm_bindgen(typescript_type = "(ITransactionInput | TransactionInput)[]")] + pub type TransactionInputArrayAsArgT; + #[wasm_bindgen(typescript_type = "TransactionInput[]")] + pub type TransactionInputArrayAsResultT; } #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransactionInputInner { pub previous_outpoint: TransactionOutpoint, - pub signature_script: Vec, + pub signature_script: Option>, pub sequence: u64, pub sig_op_count: u8, pub utxo: Option, @@ -50,7 +54,7 @@ pub struct TransactionInputInner { impl TransactionInputInner { pub fn new( previous_outpoint: TransactionOutpoint, - signature_script: Vec, + signature_script: Option>, sequence: u64, sig_op_count: u8, utxo: Option, @@ -70,7 +74,7 @@ pub struct TransactionInput { impl TransactionInput { pub fn new( previous_outpoint: TransactionOutpoint, - signature_script: Vec, + signature_script: Option>, sequence: u64, sig_op_count: u8, utxo: Option, @@ -91,6 +95,10 @@ impl TransactionInput { self.inner().sig_op_count } + pub fn signature_script_length(&self) -> usize { + self.inner().signature_script.as_ref().map(|signature_script| signature_script.len()).unwrap_or_default() + } + pub fn utxo(&self) -> Option { self.inner().utxo.clone() } @@ -99,7 +107,7 @@ impl TransactionInput { #[wasm_bindgen] impl TransactionInput { #[wasm_bindgen(constructor)] - pub fn constructor(value: &ITransactionInput) -> Result { + pub fn constructor(value: &TransactionInputT) -> Result { Self::try_owned_from(value) } @@ -120,8 +128,8 @@ impl TransactionInput { } #[wasm_bindgen(getter = signatureScript)] - pub fn get_signature_script_as_hex(&self) -> String { - self.inner().signature_script.to_hex() + pub fn get_signature_script_as_hex(&self) -> Option { + self.inner().signature_script.as_ref().map(|script| script.to_hex()) } #[wasm_bindgen(setter = signatureScript)] @@ -163,7 +171,7 @@ impl TransactionInput { impl TransactionInput { pub fn set_signature_script(&self, signature_script: Vec) { - self.inner().signature_script = signature_script; + self.inner().signature_script.replace(signature_script); } pub fn script_public_key(&self) -> Option { @@ -179,14 +187,17 @@ impl AsRef for TransactionInput { impl TryCastFromJs for TransactionInput { type Error = Error; - fn try_cast_from(value: impl AsRef) -> std::result::Result, Self::Error> { - Self::resolve_cast(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> std::result::Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve_cast(value, || { if let Some(object) = Object::try_from(value.as_ref()) { let previous_outpoint: TransactionOutpoint = object.get_value("previousOutpoint")?.as_ref().try_into()?; - let signature_script = object.get_vec_u8("signatureScript")?; + let signature_script = object.get_vec_u8("signatureScript").ok(); let sequence = object.get_u64("sequence")?; let sig_op_count = object.get_u8("sigOpCount")?; - let utxo = object.try_get_cast::("utxo")?.map(Cast::into_owned); + let utxo = object.try_cast_into::("utxo")?; Ok(TransactionInput::new(previous_outpoint, signature_script, sequence, sig_op_count, utxo).into()) } else { Err("TransactionInput must be an object".into()) @@ -199,7 +210,7 @@ impl From for TransactionInput { fn from(tx_input: cctx::TransactionInput) -> Self { TransactionInput::new( tx_input.previous_outpoint.into(), - tx_input.signature_script, + Some(tx_input.signature_script), tx_input.sequence, tx_input.sig_op_count, None, @@ -212,7 +223,8 @@ impl From<&TransactionInput> for cctx::TransactionInput { let inner = tx_input.inner(); cctx::TransactionInput::new( inner.previous_outpoint.clone().into(), - inner.signature_script.clone(), + // TODO - discuss: should this unwrap_or_default or return an error? + inner.signature_script.clone().unwrap_or_default(), inner.sequence, inner.sig_op_count, ) diff --git a/consensus/client/src/lib.rs b/consensus/client/src/lib.rs index 4935b16..eb482ea 100644 --- a/consensus/client/src/lib.rs +++ b/consensus/client/src/lib.rs @@ -1,33 +1,29 @@ pub mod error; mod imports; +mod input; mod outpoint; mod output; pub mod result; +mod serializable; +mod transaction; mod utxo; +pub use input::*; pub use outpoint::*; pub use output::*; +pub use serializable::*; +pub use transaction::*; pub use utxo::*; cfg_if::cfg_if! { if #[cfg(feature = "wasm32-sdk")] { mod header; - mod input; - mod transaction; - mod vtx; + mod utils; mod hash; mod sign; - mod script; - mod serializable; - pub use header::*; - pub use input::*; - pub use transaction::*; - pub use serializable::*; - pub use vtx::*; + pub use utils::*; pub use hash::*; - // pub use signing::*; - pub use script::*; pub use sign::sign_with_multiple_v3; } } diff --git a/consensus/client/src/outpoint.rs b/consensus/client/src/outpoint.rs index 725ed83..e265604 100644 --- a/consensus/client/src/outpoint.rs +++ b/consensus/client/src/outpoint.rs @@ -165,6 +165,15 @@ impl From for cctx::TransactionOutpoint { } } +impl From<&TransactionOutpoint> for cctx::TransactionOutpoint { + fn from(outpoint: &TransactionOutpoint) -> Self { + let inner = outpoint.inner(); + let transaction_id = inner.transaction_id; + let index = inner.index; + cctx::TransactionOutpoint::new(transaction_id, index) + } +} + impl TransactionOutpoint { pub fn simulated() -> Self { Self::new(TransactionId::from_slice(&rand::random::<[u8; spectre_hashes::HASH_SIZE]>()), 0) diff --git a/consensus/client/src/output.rs b/consensus/client/src/output.rs index cc9b541..b9bf5fe 100644 --- a/consensus/client/src/output.rs +++ b/consensus/client/src/output.rs @@ -9,7 +9,7 @@ const TS_TRANSACTION_OUTPUT: &'static str = r#" */ export interface ITransactionOutput { value: bigint; - scriptPublicKey: IScriptPublicKey; + scriptPublicKey: IScriptPublicKey | HexString; /** Optional verbose data provided by RPC */ verboseData?: ITransactionOutputVerboseData; @@ -26,6 +26,16 @@ export interface ITransactionOutputVerboseData { } "#; +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "ITransactionOutput | TransactionOutput")] + pub type TransactionOutputT; + #[wasm_bindgen(typescript_type = "(ITransactionOutput | TransactionOutput)[]")] + pub type TransactionOutputArrayAsArgT; + #[wasm_bindgen(typescript_type = "TransactionOutput[]")] + pub type TransactionOutputArrayAsResultT; +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TransactionOutputInner { @@ -55,7 +65,7 @@ impl TransactionOutput { self.inner.lock().unwrap() } - pub fn script_length(&self) -> usize { + pub fn script_public_key_length(&self) -> usize { self.inner().script_public_key.script().len() } } @@ -69,7 +79,7 @@ impl TransactionOutput { } #[wasm_bindgen(getter, js_name = value)] - pub fn get_value(&self) -> u64 { + pub fn value(&self) -> u64 { self.inner().value } @@ -114,25 +124,20 @@ impl From<&TransactionOutput> for cctx::TransactionOutput { } } -impl TryFrom<&JsValue> for TransactionOutput { - type Error = Error; - fn try_from(js_value: &JsValue) -> Result { - // workflow_log::log_trace!("js_value->TransactionOutput: {js_value:?}"); - if let Some(object) = Object::try_from(js_value) { - let has_address = Object::has_own(object, &JsValue::from("address")); - workflow_log::log_trace!("js_value->TransactionOutput: has_address:{has_address:?}"); - let value = object.get_u64("value")?; - let script_public_key = ScriptPublicKey::try_cast_from(object.get_value("scriptPublicKey")?)?; - Ok(TransactionOutput::new(value, script_public_key.into_owned())) - } else { - Err("TransactionInput must be an object".into()) - } - } -} - -impl TryFrom for TransactionOutput { +impl TryCastFromJs for TransactionOutput { type Error = Error; - fn try_from(js_value: JsValue) -> Result { - Self::try_from(&js_value) + fn try_cast_from<'a, R>(value: &'a R) -> std::result::Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve_cast(value, || { + if let Some(object) = Object::try_from(value.as_ref()) { + let value = object.get_u64("value")?; + let script_public_key = ScriptPublicKey::try_owned_from(object.get_value("scriptPublicKey")?)?; + Ok(TransactionOutput::new(value, script_public_key).into()) + } else { + Err("TransactionInput must be an object".into()) + } + }) } } diff --git a/consensus/client/src/serializable/mod.rs b/consensus/client/src/serializable/mod.rs index 5855e26..a590ab2 100644 --- a/consensus/client/src/serializable/mod.rs +++ b/consensus/client/src/serializable/mod.rs @@ -30,7 +30,7 @@ export interface ISerializableTransactionInput { index: number; sequence: bigint; sigOpCount: number; - signatureScript: HexString; + signatureScript?: HexString; utxo: ISerializableUtxoEntry; } @@ -77,3 +77,9 @@ export interface ISerializableTransaction { } "#; + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(extends = js_sys::Array, typescript_type = "ISerializableTransaction")] + pub type SerializableTransactionT; +} diff --git a/consensus/client/src/serializable/numeric.rs b/consensus/client/src/serializable/numeric.rs index 76624db..8a475ee 100644 --- a/consensus/client/src/serializable/numeric.rs +++ b/consensus/client/src/serializable/numeric.rs @@ -80,6 +80,7 @@ pub struct SerializableTransactionInput { pub sequence: u64, pub sig_op_count: u8, #[serde(with = "hex::serde")] + // TODO - convert to Option> and use hex serialization over Option pub signature_script: Vec, pub utxo: SerializableUtxoEntry, } @@ -91,6 +92,8 @@ impl SerializableTransactionInput { Self { transaction_id: input.previous_outpoint.transaction_id, index: input.previous_outpoint.index, + // TODO - convert signature_script to Option> + // signature_script: (!input.signature_script.is_empty()).then_some(input.signature_script.clone()), signature_script: input.signature_script.clone(), sequence: input.sequence, sig_op_count: input.sig_op_count, @@ -134,15 +137,16 @@ impl TryFrom for cctx::TransactionInput { impl TryFrom<&SerializableTransactionInput> for TransactionInput { type Error = Error; - fn try_from(signable_input: &SerializableTransactionInput) -> Result { - let utxo = UtxoEntryReference::try_from(signable_input)?; + fn try_from(serializable_input: &SerializableTransactionInput) -> Result { + let utxo = UtxoEntryReference::try_from(serializable_input)?; - let previous_outpoint = TransactionOutpoint::new(signable_input.transaction_id, signable_input.index); + let previous_outpoint = TransactionOutpoint::new(serializable_input.transaction_id, serializable_input.index); let inner = TransactionInputInner { previous_outpoint, - signature_script: signable_input.signature_script.clone(), - sequence: signable_input.sequence, - sig_op_count: signable_input.sig_op_count, + // TODO - convert to Option> and use hex serialization over Option + signature_script: (!serializable_input.signature_script.is_empty()).then_some(serializable_input.signature_script.clone()), + sequence: serializable_input.sequence, + sig_op_count: serializable_input.sig_op_count, utxo: Some(utxo), }; @@ -159,7 +163,8 @@ impl TryFrom<&TransactionInput> for SerializableTransactionInput { Ok(Self { transaction_id: inner.previous_outpoint.transaction_id(), index: inner.previous_outpoint.index(), - signature_script: inner.signature_script.clone(), + // TODO - convert to Option> and use hex serialization over Option + signature_script: inner.signature_script.clone().unwrap_or_default(), sequence: inner.sequence, sig_op_count: inner.sig_op_count, utxo, diff --git a/consensus/client/src/serializable/string.rs b/consensus/client/src/serializable/string.rs index 7ecf606..dfe1f16 100644 --- a/consensus/client/src/serializable/string.rs +++ b/consensus/client/src/serializable/string.rs @@ -139,7 +139,8 @@ impl TryFrom<&SerializableTransactionInput> for TransactionInput { let previous_outpoint = TransactionOutpoint::new(serializable_input.transaction_id, serializable_input.index); let inner = TransactionInputInner { previous_outpoint, - signature_script: serializable_input.signature_script.clone(), + // TODO - convert to Option> and use hex serialization over Option + signature_script: (!serializable_input.signature_script.is_empty()).then_some(serializable_input.signature_script.clone()), sequence: serializable_input.sequence.parse()?, sig_op_count: serializable_input.sig_op_count, utxo: Some(utxo), @@ -158,7 +159,8 @@ impl TryFrom<&TransactionInput> for SerializableTransactionInput { Ok(Self { transaction_id: inner.previous_outpoint.transaction_id(), index: inner.previous_outpoint.index(), - signature_script: inner.signature_script.clone(), + // TODO - convert to Option> and use hex serialization over Option + signature_script: inner.signature_script.clone().unwrap_or_default(), sequence: inner.sequence.to_string(), sig_op_count: inner.sig_op_count, utxo, diff --git a/consensus/client/src/sign.rs b/consensus/client/src/sign.rs index 9fb9b1c..95fa9c3 100644 --- a/consensus/client/src/sign.rs +++ b/consensus/client/src/sign.rs @@ -13,14 +13,14 @@ use std::collections::BTreeMap; /// A wrapper enum that represents the transaction signed state. A transaction /// contained by this enum can be either fully signed or partially signed. -pub enum Signed { - Fully(Transaction), - Partially(Transaction), +pub enum Signed<'a> { + Fully(&'a Transaction), + Partially(&'a Transaction), } -impl Signed { +impl<'a> Signed<'a> { /// Returns the transaction regardless of whether it is fully or partially signed - pub fn unwrap(self) -> Transaction { + pub fn unwrap(self) -> &'a Transaction { match self { Signed::Fully(tx) => tx, Signed::Partially(tx) => tx, @@ -31,7 +31,7 @@ impl Signed { /// TODO (aspect) - merge this with `v1` fn above or refactor wallet core to use the script engine. /// Sign a transaction using schnorr #[allow(clippy::result_large_err)] -pub fn sign_with_multiple_v3(tx: Transaction, privkeys: &[[u8; 32]]) -> crate::result::Result { +pub fn sign_with_multiple_v3<'a>(tx: &'a Transaction, privkeys: &[[u8; 32]]) -> crate::result::Result> { let mut map = BTreeMap::new(); for privkey in privkeys { let schnorr_key = secp256k1::Keypair::from_seckey_slice(secp256k1::SECP256K1, privkey).unwrap(); @@ -44,7 +44,7 @@ pub fn sign_with_multiple_v3(tx: Transaction, privkeys: &[[u8; 32]]) -> crate::r let mut additional_signatures_required = false; { let input_len = tx.inner().inputs.len(); - let (cctx, utxos) = tx.tx_and_utxos(); + let (cctx, utxos) = tx.tx_and_utxos()?; let populated_transaction = PopulatedTransaction::new(&cctx, utxos); for i in 0..input_len { let script_pub_key = match tx.inner().inputs[i].script_public_key() { diff --git a/consensus/client/src/transaction.rs b/consensus/client/src/transaction.rs index 2df7258..84b9f4c 100644 --- a/consensus/client/src/transaction.rs +++ b/consensus/client/src/transaction.rs @@ -1,11 +1,11 @@ #![allow(non_snake_case)] use crate::imports::*; -use crate::input::TransactionInput; +use crate::input::{TransactionInput, TransactionInputArrayAsArgT, TransactionInputArrayAsResultT}; use crate::outpoint::TransactionOutpoint; -use crate::output::TransactionOutput; +use crate::output::{TransactionOutput, TransactionOutputArrayAsArgT, TransactionOutputArrayAsResultT}; use crate::result::Result; -use crate::serializable::{numeric, string}; +use crate::serializable::{numeric, string, SerializableTransactionT}; use crate::utxo::{UtxoEntryId, UtxoEntryReference}; use ahash::AHashMap; use spectre_consensus_core::network::NetworkType; @@ -51,8 +51,8 @@ export interface ITransactionVerboseData { #[wasm_bindgen] extern "C" { - #[wasm_bindgen(typescript_type = "ITransaction")] - pub type ITransaction; + #[wasm_bindgen(typescript_type = "ITransaction | Transaction")] + pub type TransactionT; } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -149,14 +149,14 @@ impl Transaction { } #[wasm_bindgen(constructor)] - pub fn constructor(js_value: &ITransaction) -> std::result::Result { + pub fn constructor(js_value: &TransactionT) -> std::result::Result { Ok(js_value.try_into_owned()?) } #[wasm_bindgen(getter = inputs)] - pub fn get_inputs_as_js_array(&self) -> Array { + pub fn get_inputs_as_js_array(&self) -> TransactionInputArrayAsResultT { let inputs = self.inner.lock().unwrap().inputs.clone().into_iter().map(JsValue::from); - Array::from_iter(inputs) + Array::from_iter(inputs).unchecked_into() } /// Returns a list of unique addresses used by transaction inputs. @@ -179,7 +179,7 @@ impl Transaction { } #[wasm_bindgen(setter = inputs)] - pub fn set_inputs_from_js_array(&mut self, js_value: &JsValue) { + pub fn set_inputs_from_js_array(&mut self, js_value: &TransactionInputArrayAsArgT) { let inputs = Array::from(js_value) .iter() .map(|js_value| { @@ -190,16 +190,16 @@ impl Transaction { } #[wasm_bindgen(getter = outputs)] - pub fn get_outputs_as_js_array(&self) -> Array { + pub fn get_outputs_as_js_array(&self) -> TransactionOutputArrayAsResultT { let outputs = self.inner.lock().unwrap().outputs.clone().into_iter().map(JsValue::from); - Array::from_iter(outputs) + Array::from_iter(outputs).unchecked_into() } #[wasm_bindgen(setter = outputs)] - pub fn set_outputs_from_js_array(&mut self, js_value: &JsValue) { + pub fn set_outputs_from_js_array(&mut self, js_value: &TransactionOutputArrayAsArgT) { let outputs = Array::from(js_value) .iter() - .map(|js_value| TransactionOutput::try_from(&js_value).unwrap_or_else(|err| panic!("invalid transaction output: {err}"))) + .map(|js_value| TryCastFromJs::try_owned_from(&js_value).unwrap_or_else(|err| panic!("invalid transaction output: {err}"))) .collect::>(); self.inner().outputs = outputs; } @@ -214,12 +214,12 @@ impl Transaction { self.inner().version = v; } - #[wasm_bindgen(getter, js_name = lock_time)] + #[wasm_bindgen(getter, js_name = lockTime)] pub fn get_lock_time(&self) -> u64 { self.inner().lock_time } - #[wasm_bindgen(setter, js_name = lock_time)] + #[wasm_bindgen(setter, js_name = lockTime)] pub fn set_lock_time(&self, v: u64) { self.inner().lock_time = v; } @@ -258,13 +258,18 @@ impl Transaction { impl TryCastFromJs for Transaction { type Error = Error; - fn try_cast_from(value: impl AsRef) -> std::result::Result, Self::Error> { - Self::resolve_cast(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> std::result::Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve_cast(value, || { if let Some(object) = Object::try_from(value.as_ref()) { if let Some(tx) = object.try_get_value("tx")? { - Transaction::try_cast_from(&tx) + // TODO - optimize to use ref anchor + Transaction::try_captured_cast_from(tx) + // Ok(Cast::value(Transaction::try_owned_from(tx)?)) } else { - let id = object.try_get_cast::("id")?.map(|id| id.into_owned()); + let id = object.try_cast_into::("id")?; let version = object.get_u16("version")?; let lock_time = object.get_u64("lockTime")?; let gas = object.get_u64("gas")?; @@ -285,7 +290,7 @@ impl TryCastFromJs for Transaction { let outputs: Vec = object .get_vec("outputs")? .iter() - .map(|jsv| jsv.try_into()) + .map(TryCastFromJs::try_owned_from) .collect::, Error>>()?; Transaction::new(id, version, inputs, outputs, lock_time, subnetwork_id, gas, payload).map(Into::into) } @@ -342,7 +347,13 @@ impl Transaction { .map(|input| { let previous_outpoint: TransactionOutpoint = input.previous_outpoint.into(); let utxo = utxos.get(previous_outpoint.id()).cloned(); - TransactionInput::new(previous_outpoint, input.signature_script.clone(), input.sequence, input.sig_op_count, utxo) + TransactionInput::new( + previous_outpoint, + Some(input.signature_script.clone()), + input.sequence, + input.sig_op_count, + utxo, + ) }) .collect::>(); let outputs: Vec = tx.outputs.iter().map(|output| output.into()).collect::>(); @@ -359,18 +370,18 @@ impl Transaction { }) } - pub fn tx_and_utxos(&self) -> (cctx::Transaction, Vec) { - let mut utxos = vec![]; + pub fn tx_and_utxos(&self) -> Result<(cctx::Transaction, Vec)> { + let mut inputs = vec![]; let inner = self.inner(); - let inputs: Vec = inner + let utxos: Vec = inner .inputs .clone() .into_iter() .map(|input| { - utxos.push((&input.get_utxo().unwrap().entry()).into()); - input.as_ref().into() + inputs.push(input.as_ref().into()); + Ok(input.get_utxo().ok_or(Error::MissingUtxoEntry)?.entry().as_ref().into()) }) - .collect::>(); + .collect::>>()?; let outputs: Vec = inner.outputs.clone().into_iter().map(|output| output.as_ref().into()).collect::>(); let tx = cctx::Transaction::new( @@ -383,7 +394,37 @@ impl Transaction { inner.payload.clone(), ); - (tx, utxos) + Ok((tx, utxos)) + } + + pub fn utxo_entry_references(&self) -> Result> { + let inner = self.inner(); + let utxo_entry_references = inner + .inputs + .clone() + .into_iter() + .map(|input| input.get_utxo().ok_or(Error::MissingUtxoEntry)) + .collect::>>()?; + Ok(utxo_entry_references) + } + + pub fn outputs(&self) -> Vec { + let inner = self.inner(); + let outputs = inner.outputs.iter().map(|output| output.into()).collect::>(); + outputs + } + + pub fn inputs(&self) -> Vec { + let inner = self.inner(); + let inputs = inner.inputs.iter().map(Into::into).collect::>(); + inputs + } + + pub fn inputs_outputs(&self) -> (Vec, Vec) { + let inner = self.inner(); + let inputs = inner.inputs.iter().map(Into::into).collect::>(); + let outputs = inner.outputs.iter().map(Into::into).collect::>(); + (inputs, outputs) } pub fn set_signature_script(&self, input_index: usize, signature_script: Vec) -> Result<()> { @@ -393,6 +434,14 @@ impl Transaction { self.inner().inputs[input_index].set_signature_script(signature_script); Ok(()) } + + pub fn payload(&self) -> Vec { + self.inner().payload.clone() + } + + pub fn payload_len(&self) -> usize { + self.inner().payload.len() + } } #[wasm_bindgen] @@ -401,7 +450,7 @@ impl Transaction { /// The schema of the JavaScript object is defined by {@link ISerializableTransaction}. /// @see {@link ISerializableTransaction} #[wasm_bindgen(js_name = "serializeToObject")] - pub fn serialize_to_object(&self) -> Result { + pub fn serialize_to_object(&self) -> Result { Ok(numeric::SerializableTransaction::from_client_transaction(self)?.serialize_to_object()?.into()) } diff --git a/consensus/client/src/utils.rs b/consensus/client/src/utils.rs new file mode 100644 index 0000000..7d26339 --- /dev/null +++ b/consensus/client/src/utils.rs @@ -0,0 +1,81 @@ +use crate::imports::*; +use crate::result::Result; +use spectre_addresses::*; +use spectre_consensus_core::{ + network::{NetworkType, NetworkTypeT}, + tx::ScriptPublicKeyT, +}; +use spectre_txscript::{script_class::ScriptClass, standard}; +use spectre_utils::hex::ToHex; +use spectre_wasm_core::types::{BinaryT, HexString}; + +/// Creates a new script to pay a transaction output to the specified address. +/// @category Wallet SDK +#[wasm_bindgen(js_name = payToAddressScript)] +pub fn pay_to_address_script(address: &AddressT) -> Result { + let address = Address::try_cast_from(address)?; + Ok(standard::pay_to_address_script(address.as_ref())) +} + +/// Takes a script and returns an equivalent pay-to-script-hash script. +/// @param redeem_script - The redeem script ({@link HexString} or Uint8Array). +/// @category Wallet SDK +#[wasm_bindgen(js_name = payToScriptHashScript)] +pub fn pay_to_script_hash_script(redeem_script: BinaryT) -> Result { + let redeem_script = redeem_script.try_as_vec_u8()?; + Ok(standard::pay_to_script_hash_script(redeem_script.as_slice())) +} + +/// Generates a signature script that fits a pay-to-script-hash script. +/// @param redeem_script - The redeem script ({@link HexString} or Uint8Array). +/// @param signature - The signature ({@link HexString} or Uint8Array). +/// @category Wallet SDK +#[wasm_bindgen(js_name = payToScriptHashSignatureScript)] +pub fn pay_to_script_hash_signature_script(redeem_script: BinaryT, signature: BinaryT) -> Result { + let redeem_script = redeem_script.try_as_vec_u8()?; + let signature = signature.try_as_vec_u8()?; + let script = standard::pay_to_script_hash_signature_script(redeem_script, signature)?; + Ok(script.to_hex().into()) +} + +/// Returns the address encoded in a script public key. +/// @param script_public_key - The script public key ({@link ScriptPublicKey}). +/// @param network - The network type. +/// @category Wallet SDK +#[wasm_bindgen(js_name = addressFromScriptPublicKey)] +pub fn address_from_script_public_key(script_public_key: &ScriptPublicKeyT, network: &NetworkTypeT) -> Result { + let script_public_key = ScriptPublicKey::try_cast_from(script_public_key)?; + let network_type = NetworkType::try_from(network)?; + + match standard::extract_script_pub_key_address(script_public_key.as_ref(), network_type.into()) { + Ok(address) => Ok(AddressOrUndefinedT::from(JsValue::from(address))), + Err(_) => Ok(AddressOrUndefinedT::from(JsValue::UNDEFINED)), + } +} + +/// Returns true if the script passed is a pay-to-pubkey. +/// @param script - The script ({@link HexString} or Uint8Array). +/// @category Wallet SDK +#[wasm_bindgen(js_name = isScriptPayToPubkey)] +pub fn is_script_pay_to_pubkey(script: BinaryT) -> Result { + let script = script.try_as_vec_u8()?; + Ok(ScriptClass::is_pay_to_pubkey(script.as_slice())) +} + +/// Returns returns true if the script passed is an ECDSA pay-to-pubkey. +/// @param script - The script ({@link HexString} or Uint8Array). +/// @category Wallet SDK +#[wasm_bindgen(js_name = isScriptPayToPubkeyECDSA)] +pub fn is_script_pay_to_pubkey_ecdsa(script: BinaryT) -> Result { + let script = script.try_as_vec_u8()?; + Ok(ScriptClass::is_pay_to_pubkey_ecdsa(script.as_slice())) +} + +/// Returns true if the script passed is a pay-to-script-hash (P2SH) format, false otherwise. +/// @param script - The script ({@link HexString} or Uint8Array). +/// @category Wallet SDK +#[wasm_bindgen(js_name = isScriptPayToScriptHash)] +pub fn is_script_pay_to_script_hash(script: BinaryT) -> Result { + let script = script.try_as_vec_u8()?; + Ok(ScriptClass::is_pay_to_script_hash(script.as_slice())) +} diff --git a/consensus/client/src/utxo.rs b/consensus/client/src/utxo.rs index 293c73b..c910cc4 100644 --- a/consensus/client/src/utxo.rs +++ b/consensus/client/src/utxo.rs @@ -101,6 +101,12 @@ impl UtxoEntry { } } +impl AsRef for UtxoEntry { + fn as_ref(&self) -> &UtxoEntry { + self + } +} + impl From<&UtxoEntry> for cctx::UtxoEntry { fn from(utxo: &UtxoEntry) -> Self { cctx::UtxoEntry { @@ -136,14 +142,14 @@ impl UtxoEntryReference { self.as_ref().clone() } - #[wasm_bindgen(js_name = "getTransactionId")] - pub fn transaction_id_as_string(&self) -> String { - self.utxo.outpoint.get_transaction_id_as_string() + #[wasm_bindgen(getter)] + pub fn outpoint(&self) -> TransactionOutpoint { + self.utxo.outpoint.clone() } - #[wasm_bindgen(js_name = "getId")] - pub fn id_string(&self) -> String { - self.utxo.outpoint.id_string() + #[wasm_bindgen(getter)] + pub fn address(&self) -> Option
{ + self.utxo.address.clone() } #[wasm_bindgen(getter)] @@ -160,6 +166,11 @@ impl UtxoEntryReference { pub fn block_daa_score(&self) -> u64 { self.utxo.block_daa_score } + + #[wasm_bindgen(getter, js_name = "scriptPublicKey")] + pub fn script_public_key(&self) -> ScriptPublicKey { + self.utxo.script_public_key.clone() + } } impl UtxoEntryReference { @@ -252,7 +263,10 @@ impl TryIntoUtxoEntryReferences for JsValue { impl TryCastFromJs for UtxoEntry { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { Ok(Self::try_ref_from_js_value_as_cast(value)?) } } @@ -372,21 +386,45 @@ impl TryFrom for UtxoEntries { impl TryCastFromJs for UtxoEntryReference { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Ok(utxo_entry) = UtxoEntry::try_ref_from_js_value(&value) { Ok(Self::from(utxo_entry.clone())) } else if let Some(object) = Object::try_from(value.as_ref()) { - let address = object.get_cast::
("address")?.into_owned(); + let address = object.try_cast_into::
("address")?; let outpoint = TransactionOutpoint::try_from(object.get_value("outpoint")?.as_ref())?; let utxo_entry = Object::from(object.get_value("utxoEntry")?); - let amount = utxo_entry.get_u64("amount")?; - let script_public_key = ScriptPublicKey::try_owned_from(utxo_entry.get_value("scriptPublicKey")?)?; - let block_daa_score = utxo_entry.get_u64("blockDaaScore")?; - let is_coinbase = utxo_entry.get_bool("isCoinbase")?; - let utxo_entry = - UtxoEntry { address: Some(address), outpoint, amount, script_public_key, block_daa_score, is_coinbase }; + let utxo_entry = if !utxo_entry.is_undefined() { + let amount = utxo_entry.get_u64("amount").map_err(|_| { + Error::custom("Supplied object does not contain `utxoEntry.amount` property (or it is not a numerical value)") + })?; + let script_public_key = ScriptPublicKey::try_owned_from(utxo_entry.get_value("scriptPublicKey")?) + .map_err(|_|Error::custom("Supplied object does not contain `utxoEntry.scriptPublicKey` property (or it is not a hex string or a ScriptPublicKey class)"))?; + let block_daa_score = utxo_entry.get_u64("blockDaaScore").map_err(|_| { + Error::custom( + "Supplied object does not contain `utxoEntry.blockDaaScore` property (or it is not a numerical value)", + ) + })?; + let is_coinbase = utxo_entry.get_bool("isCoinbase")?; + + UtxoEntry { address, outpoint, amount, script_public_key, block_daa_score, is_coinbase } + } else { + let amount = object.get_u64("amount").map_err(|_| { + Error::custom("Supplied object does not contain `amount` property (or it is not a numerical value)") + })?; + let script_public_key = ScriptPublicKey::try_owned_from(object.get_value("scriptPublicKey")?) + .map_err(|_|Error::custom("Supplied object does not contain `scriptPublicKey` property (or it is not a hex string or a ScriptPublicKey class)"))?; + let block_daa_score = object.get_u64("blockDaaScore").map_err(|_| { + Error::custom("Supplied object does not contain `blockDaaScore` property (or it is not a numerical value)") + })?; + let is_coinbase = object.try_get_bool("isCoinbase")?.unwrap_or(false); + + UtxoEntry { address, outpoint, amount, script_public_key, block_daa_score, is_coinbase } + }; Ok(UtxoEntryReference::from(utxo_entry)) } else { diff --git a/consensus/client/src/vtx.rs b/consensus/client/src/vtx.rs deleted file mode 100644 index bcad1c2..0000000 --- a/consensus/client/src/vtx.rs +++ /dev/null @@ -1,35 +0,0 @@ -use crate::imports::*; -// use crate::serializable::{numeric,string}; -use crate::result::Result; -use serde::de::DeserializeOwned; -use spectre_addresses::Address; -// use serde::de::DeserializeOwned; - -#[derive(Debug, Serialize, Deserialize, Clone)] -#[serde(rename_all = "camelCase")] -pub struct VirtualTransactionT -where - T: Clone + serde::Serialize, -{ - //} + Deserialize { - pub version: u32, - pub generator: Option, - pub transactions: Vec, - pub addresses: Option>, -} - -impl VirtualTransactionT -where - T: Clone + Serialize, -{ - pub fn deserialize(json: &str) -> Result - where - T: DeserializeOwned, - { - Ok(serde_json::from_str(json)?) - } - - pub fn serialize(&self) -> String { - serde_json::to_string(self).unwrap() - } -} diff --git a/consensus/core/Cargo.toml b/consensus/core/Cargo.toml index 5daf73b..b0beb64 100644 --- a/consensus/core/Cargo.toml +++ b/consensus/core/Cargo.toml @@ -41,6 +41,7 @@ thiserror.workspace = true wasm-bindgen.workspace = true workflow-core.workspace = true workflow-log.workspace = true +workflow-serializer.workspace = true workflow-wasm.workspace = true [dev-dependencies] @@ -53,5 +54,5 @@ web-sys.workspace = true name = "serde_benchmark" harness = false -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/consensus/core/src/api/stats.rs b/consensus/core/src/api/stats.rs index fd59f09..c2fea48 100644 --- a/consensus/core/src/api/stats.rs +++ b/consensus/core/src/api/stats.rs @@ -1,7 +1,7 @@ -use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use workflow_serializer::prelude::*; -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize, Default)] +#[derive(Clone, Debug, Serialize, Deserialize, Default)] #[serde(rename_all = "camelCase")] pub struct BlockCount { pub header_count: u64, @@ -14,6 +14,26 @@ impl BlockCount { } } +impl Serializer for BlockCount { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.header_count, writer)?; + store!(u64, &self.block_count, writer)?; + + Ok(()) + } +} + +impl Deserializer for BlockCount { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let header_count = load!(u64, reader)?; + let block_count = load!(u64, reader)?; + + Ok(Self { header_count, block_count }) + } +} + #[derive(Clone, Default)] pub struct VirtualStateStats { /// Number of direct parents of virtual diff --git a/consensus/core/src/hashing/mod.rs b/consensus/core/src/hashing/mod.rs index ae85c7c..1f32466 100644 --- a/consensus/core/src/hashing/mod.rs +++ b/consensus/core/src/hashing/mod.rs @@ -5,6 +5,8 @@ pub mod header; pub mod sighash; pub mod sighash_type; pub mod tx; +#[cfg(feature = "wasm32-sdk")] +pub mod wasm; pub trait HasherExtensions { /// Writes the len as u64 little endian bytes diff --git a/consensus/core/src/hashing/wasm.rs b/consensus/core/src/hashing/wasm.rs new file mode 100644 index 0000000..92caa31 --- /dev/null +++ b/consensus/core/src/hashing/wasm.rs @@ -0,0 +1,27 @@ +use super::sighash_type::{self, SigHashType}; +use wasm_bindgen::prelude::*; + +/// Spectre Sighash types allowed by consensus +/// @category Consensus +#[wasm_bindgen] +pub enum SighashType { + All, + None, + Single, + AllAnyOneCanPay, + NoneAnyOneCanPay, + SingleAnyOneCanPay, +} + +impl From for SigHashType { + fn from(sighash_type: SighashType) -> SigHashType { + match sighash_type { + SighashType::All => sighash_type::SIG_HASH_ALL, + SighashType::None => sighash_type::SIG_HASH_NONE, + SighashType::Single => sighash_type::SIG_HASH_SINGLE, + SighashType::AllAnyOneCanPay => sighash_type::SIG_HASH_ANY_ONE_CAN_PAY, + SighashType::NoneAnyOneCanPay => SigHashType(sighash_type::SIG_HASH_NONE.0 | sighash_type::SIG_HASH_ANY_ONE_CAN_PAY.0), + SighashType::SingleAnyOneCanPay => SigHashType(sighash_type::SIG_HASH_SINGLE.0 | sighash_type::SIG_HASH_ANY_ONE_CAN_PAY.0), + } + } +} diff --git a/consensus/core/src/header.rs b/consensus/core/src/header.rs index 61ed2e8..6b0d5bd 100644 --- a/consensus/core/src/header.rs +++ b/consensus/core/src/header.rs @@ -93,6 +93,12 @@ impl Header { } } +impl AsRef
for Header { + fn as_ref(&self) -> &Header { + self + } +} + impl MemSizeEstimator for Header { fn estimate_mem_bytes(&self) -> usize { size_of::() + self.parents_by_level.iter().map(|l| l.len()).sum::() * size_of::() diff --git a/consensus/core/src/mass/mod.rs b/consensus/core/src/mass/mod.rs index 251b55b..3bd41b2 100644 --- a/consensus/core/src/mass/mod.rs +++ b/consensus/core/src/mass/mod.rs @@ -4,6 +4,17 @@ use crate::{ }; use spectre_hashes::HASH_SIZE; +/// Temp enum for the transition phases of KIP9 +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum Kip9Version { + /// Initial KIP9 mass calculation, w/o the relaxed formula and summing storage mass and compute mass + Alpha, + + /// Currently proposed KIP9 mass calculation, with the relaxed formula (for the cases `|O| = 1 OR |O| <= |I| <= 2`), + /// and using a maximum operator over storage and compute mass + Beta, +} + // transaction_estimated_serialized_size is the estimated size of a transaction in some // serialization. This has to be deterministic, but not necessarily accurate, since // it's only used as the size component in the transaction and block mass limit diff --git a/consensus/core/src/network.rs b/consensus/core/src/network.rs index a6f6011..620631a 100644 --- a/consensus/core/src/network.rs +++ b/consensus/core/src/network.rs @@ -400,8 +400,11 @@ impl TryFrom for NetworkId { impl TryCastFromJs for NetworkId { type Error = NetworkIdError; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(network_id) = value.as_ref().as_string() { Ok(NetworkId::from_str(&network_id)?) } else { diff --git a/consensus/core/src/sign.rs b/consensus/core/src/sign.rs index dee0d38..a40b949 100644 --- a/consensus/core/src/sign.rs +++ b/consensus/core/src/sign.rs @@ -1,9 +1,9 @@ use crate::{ hashing::{ sighash::{calc_schnorr_signature_hash, SigHashReusedValues}, - sighash_type::SIG_HASH_ALL, + sighash_type::{SigHashType, SIG_HASH_ALL}, }, - tx::SignableTransaction, + tx::{SignableTransaction, VerifiableTransaction}, }; use itertools::Itertools; use std::collections::BTreeMap; @@ -153,7 +153,20 @@ pub fn sign_with_multiple_v2(mut mutable_tx: SignableTransaction, privkeys: &[[u } } -pub fn verify(tx: &impl crate::tx::VerifiableTransaction) -> Result<(), Error> { +/// Sign a transaction input with a sighash_type using schnorr +pub fn sign_input(tx: &impl VerifiableTransaction, input_index: usize, private_key: &[u8; 32], hash_type: SigHashType) -> Vec { + let mut reused_values = SigHashReusedValues::new(); + + let hash = calc_schnorr_signature_hash(tx, input_index, hash_type, &mut reused_values); + let msg = secp256k1::Message::from_digest_slice(hash.as_bytes().as_slice()).unwrap(); + let schnorr_key = secp256k1::Keypair::from_seckey_slice(secp256k1::SECP256K1, private_key).unwrap(); + let sig: [u8; 64] = *schnorr_key.sign_schnorr(msg).as_ref(); + + // This represents OP_DATA_65 (since signature length is 64 bytes and SIGHASH_TYPE is one byte) + std::iter::once(65u8).chain(sig).chain([hash_type.to_u8()]).collect() +} + +pub fn verify(tx: &impl VerifiableTransaction) -> Result<(), Error> { let mut reused_values = SigHashReusedValues::new(); for (i, (input, entry)) in tx.populated_inputs().enumerate() { if input.signature_script.is_empty() { diff --git a/consensus/core/src/tx.rs b/consensus/core/src/tx.rs index 69f9dd2..c60842e 100644 --- a/consensus/core/src/tx.rs +++ b/consensus/core/src/tx.rs @@ -1,7 +1,9 @@ mod script_public_key; use borsh::{BorshDeserialize, BorshSerialize}; -pub use script_public_key::{scriptvec, ScriptPublicKey, ScriptPublicKeyVersion, ScriptPublicKeys, ScriptVec, SCRIPT_VECTOR_SIZE}; +pub use script_public_key::{ + scriptvec, ScriptPublicKey, ScriptPublicKeyT, ScriptPublicKeyVersion, ScriptPublicKeys, ScriptVec, SCRIPT_VECTOR_SIZE, +}; use serde::{Deserialize, Serialize}; use spectre_utils::hex::ToHex; use spectre_utils::mem_size::MemSizeEstimator; @@ -137,8 +139,8 @@ impl Clone for TransactionMass { } impl BorshDeserialize for TransactionMass { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { - let mass: u64 = borsh::BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let mass: u64 = borsh::BorshDeserialize::deserialize_reader(reader)?; Ok(Self(AtomicU64::new(mass))) } } @@ -163,7 +165,6 @@ pub struct Transaction { pub payload: Vec, #[serde(default)] - #[borsh_skip] // TODO: skipped for now as it is only required for consensus storage and miner grpc mass: TransactionMass, // A field that is used to cache the transaction ID. @@ -634,12 +635,12 @@ mod tests { fn test_spk_borsh() { // Tests for ScriptPublicKey Borsh ser/deser since we manually implemented them let spk = ScriptPublicKey::from_vec(12, vec![32; 20]); - let bin = spk.try_to_vec().unwrap(); + let bin = borsh::to_vec(&spk).unwrap(); let spk2: ScriptPublicKey = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(spk, spk2); let spk = ScriptPublicKey::from_vec(55455, vec![11; 200]); - let bin = spk.try_to_vec().unwrap(); + let bin = borsh::to_vec(&spk).unwrap(); let spk2: ScriptPublicKey = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(spk, spk2); } diff --git a/consensus/core/src/tx/script_public_key.rs b/consensus/core/src/tx/script_public_key.rs index a722809..10f63ab 100644 --- a/consensus/core/src/tx/script_public_key.rs +++ b/consensus/core/src/tx/script_public_key.rs @@ -1,6 +1,7 @@ use alloc::borrow::Cow; use borsh::{BorshDeserialize, BorshSerialize}; use core::fmt::Formatter; +use js_sys::Object; use serde::{ de::{Error, Visitor}, Deserialize, Deserializer, Serialize, Serializer, @@ -41,6 +42,7 @@ const TS_SCRIPT_PUBLIC_KEY: &'static str = r#" * @category Consensus */ export interface IScriptPublicKey { + version : number; script: HexString; } "#; @@ -328,6 +330,12 @@ impl ScriptPublicKey { } } +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "ScriptPublicKey | HexString")] + pub type ScriptPublicKeyT; +} + #[wasm_bindgen] impl ScriptPublicKey { #[wasm_bindgen(constructor)] @@ -357,19 +365,36 @@ impl BorshSerialize for ScriptPublicKey { } impl BorshDeserialize for ScriptPublicKey { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { + fn deserialize_reader(reader: &mut R) -> std::io::Result { // Deserialize into vec first since we have no custom smallvec support - Ok(Self::from_vec(borsh::BorshDeserialize::deserialize(buf)?, borsh::BorshDeserialize::deserialize(buf)?)) + Ok(Self::from_vec(borsh::BorshDeserialize::deserialize_reader(reader)?, borsh::BorshDeserialize::deserialize_reader(reader)?)) } } type CastError = workflow_wasm::error::Error; impl TryCastFromJs for ScriptPublicKey { type Error = workflow_wasm::error::Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(hex_str) = value.as_ref().as_string() { Ok(Self::from_str(&hex_str).map_err(CastError::custom)?) + } else if let Some(object) = Object::try_from(value.as_ref()) { + let version = object.try_get_value("version")?.ok_or(CastError::custom( + "ScriptPublicKey must be a hex string or an object with 'version' and 'script' properties", + ))?; + + let version = if let Ok(version) = version.try_as_u16() { + version + } else { + return Err(CastError::custom("Invalid version value '{version:?}'")); + }; + + let script = object.get_vec_u8("script")?; + + Ok(ScriptPublicKey::from_vec(version, script)) } else { Err(CastError::custom(format!("Unable to convert ScriptPublicKey from: {:?}", value.as_ref()))) } @@ -403,12 +428,12 @@ mod tests { fn test_spk_borsh() { // Tests for ScriptPublicKey Borsh ser/deser since we manually implemented them let spk = ScriptPublicKey::from_vec(12, vec![32; 20]); - let bin = spk.try_to_vec().unwrap(); + let bin = borsh::to_vec(&spk).unwrap(); let spk2: ScriptPublicKey = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(spk, spk2); let spk = ScriptPublicKey::from_vec(55455, vec![11; 200]); - let bin = spk.try_to_vec().unwrap(); + let bin = borsh::to_vec(&spk).unwrap(); let spk2: ScriptPublicKey = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(spk, spk2); } diff --git a/consensus/pow/Cargo.toml b/consensus/pow/Cargo.toml index 6e50e8c..717f786 100644 --- a/consensus/pow/Cargo.toml +++ b/consensus/pow/Cargo.toml @@ -30,3 +30,6 @@ criterion.workspace = true [[bench]] name = "bench" harness = false + +[lints] +workspace = true diff --git a/consensus/pow/src/wasm.rs b/consensus/pow/src/wasm.rs index 621af4e..faa76ac 100644 --- a/consensus/pow/src/wasm.rs +++ b/consensus/pow/src/wasm.rs @@ -2,73 +2,103 @@ use crate::matrix::Matrix; use js_sys::BigInt; use num::Float; use spectre_consensus_client::Header; +use spectre_consensus_client::HeaderT; use spectre_consensus_core::hashing; use spectre_hashes::Hash; use spectre_hashes::PowHash; use spectre_math::Uint256; +use spectre_utils::hex::FromHex; use spectre_utils::hex::ToHex; use wasm_bindgen::prelude::*; +use workflow_wasm::convert::TryCastFromJs; use workflow_wasm::error::Error; -use workflow_wasm::prelude::*; use workflow_wasm::result::Result; -/// @category PoW +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(extends = js_sys::Array, typescript_type = "[boolean, bigint]")] + pub type WorkT; +} + +/// Represents a Spectre header PoW manager +/// @category Mining #[wasm_bindgen(inspectable)] -pub struct State { +pub struct PoW { inner: crate::State, pre_pow_hash: Hash, } #[wasm_bindgen] -impl State { +impl PoW { #[wasm_bindgen(constructor)] - pub fn new(header: &Header) -> Self { + pub fn new(header: &HeaderT, timestamp: Option) -> Result { // this function replicates crate::State::new() but caches // the pre_pow_hash value internally, making it available // via the `pre_pow_hash` property getter. - - // obtain locked inner + let header = Header::try_cast_from(header).map_err(Error::custom)?; + let header = header.as_ref(); let header = header.inner(); + // Get required target from header bits. let target = Uint256::from_compact_target_bits(header.bits); // Zero out the time and nonce. let pre_pow_hash = hashing::header::hash_override_nonce_time(header, 0, 0); // PRE_POW_HASH || TIME || 32 zero byte padding || NONCE - let hasher = PowHash::new(pre_pow_hash, header.timestamp); + let hasher = PowHash::new(pre_pow_hash, timestamp.unwrap_or(header.timestamp)); let matrix = Matrix::generate(pre_pow_hash); - Self { inner: crate::State { matrix, target, hasher }, pre_pow_hash } + Ok(Self { inner: crate::State { matrix, target, hasher }, pre_pow_hash }) } + /// The target based on the provided bits. #[wasm_bindgen(getter)] pub fn target(&self) -> Result { - self.inner.target.try_into().map_err(|err| Error::Custom(format!("{err:?}"))) + self.inner.target.try_into().map_err(|err| Error::custom(format!("{err:?}"))) } - #[wasm_bindgen(js_name=checkPow)] - pub fn check_pow(&self, nonce_jsv: JsValue) -> Result { - let nonce = nonce_jsv.try_as_u64()?; + /// Checks if the computed target meets or exceeds the difficulty specified in the template. + /// @returns A boolean indicating if it reached the target and a bigint representing the reached target. + #[wasm_bindgen(js_name=checkWork)] + pub fn check_work(&self, nonce: u64) -> Result { let (c, v) = self.inner.check_pow(nonce); let array = js_sys::Array::new(); array.push(&JsValue::from(c)); - array.push(&v.to_bigint().map_err(|err| Error::Custom(format!("{err:?}")))?.into()); + array.push(&v.to_bigint().map_err(|err| Error::custom(format!("{err:?}")))?.into()); - Ok(array) + Ok(array.unchecked_into()) } - #[wasm_bindgen(getter = prePowHash)] + /// Hash of the header without timestamp and nonce. + #[wasm_bindgen(getter = prePoWHash)] pub fn get_pre_pow_hash(&self) -> String { self.pre_pow_hash.to_hex() } + + /// Can be used for parsing Stratum templates. + #[wasm_bindgen(js_name=fromRaw)] + pub fn from_raw(pre_pow_hash: &str, timestamp: u64, target_bits: Option) -> Result { + // Convert the pre_pow_hash from hex string to Hash + let pre_pow_hash = Hash::from_hex(pre_pow_hash).map_err(|err| Error::custom(format!("{err:?}")))?; + + // Generate the target from compact target bits if provided + let target = Uint256::from_compact_target_bits(target_bits.unwrap_or_default()); + + // Initialize the matrix and hasher using pre_pow_hash and timestamp + let matrix = Matrix::generate(pre_pow_hash); + let hasher = PowHash::new(pre_pow_hash, timestamp); + + Ok(PoW { inner: crate::State { matrix, target, hasher }, pre_pow_hash }) + } } // https://github.com/tmrlvi/spectre-miner/blob/bf361d02a46c580f55f46b5dfa773477634a5753/src/client/stratum.rs#L36 const DIFFICULTY_1_TARGET: (u64, i16) = (0xffffu64, 208); // 0xffff 2^208 -/// `calculate_difficulty` is based on set_difficulty function: -/// @category PoW -#[wasm_bindgen(js_name = calculateDifficulty)] -pub fn calculate_difficulty(difficulty: f32) -> std::result::Result { +/// Calculates target from difficulty, based on set_difficulty function on +/// +/// @category Mining +#[wasm_bindgen(js_name = calculateTarget)] +pub fn calculate_target(difficulty: f32) -> Result { let mut buf = [0u64, 0u64, 0u64, 0u64]; let (mantissa, exponent, _) = difficulty.recip().integer_decode(); let new_mantissa = mantissa * DIFFICULTY_1_TARGET.0; @@ -80,10 +110,8 @@ pub fn calculate_difficulty(difficulty: f32) -> std::result::Result> (64 - remainder); // top } else if new_mantissa.leading_zeros() < remainder as u32 { - return Err(JsError::new("Target is too big")); + return Err(Error::custom("Target is too big")); } - // let target_pool = Uint256(buf); - // workflow_log::log_info!("Difficulty: {:?}, Target: 0x{}", difficulty, target_pool.to_hex()); - Ok(Uint256(buf).try_into()?) + Uint256(buf).try_into().map_err(Error::custom) } diff --git a/consensus/src/consensus/storage.rs b/consensus/src/consensus/storage.rs index bd7f665..341963e 100644 --- a/consensus/src/consensus/storage.rs +++ b/consensus/src/consensus/storage.rs @@ -31,7 +31,7 @@ use parking_lot::RwLock; use spectre_consensus_core::{blockstatus::BlockStatus, BlockHashSet}; use spectre_database::registry::DatabaseStorePrefixes; use spectre_hashes::Hash; -use std::{mem::size_of, ops::DerefMut, sync::Arc}; +use std::{ops::DerefMut, sync::Arc}; pub struct ConsensusStorage { // DB diff --git a/consensus/src/model/stores/acceptance_data.rs b/consensus/src/model/stores/acceptance_data.rs index c0a42b0..fca8d7b 100644 --- a/consensus/src/model/stores/acceptance_data.rs +++ b/consensus/src/model/stores/acceptance_data.rs @@ -12,7 +12,6 @@ use spectre_database::prelude::{BatchDbWriter, CachedDbAccess, DirectDbWriter}; use spectre_database::registry::DatabaseStorePrefixes; use spectre_hashes::Hash; use spectre_utils::mem_size::MemSizeEstimator; -use std::mem::size_of; use std::sync::Arc; pub trait AcceptanceDataStoreReader { diff --git a/consensus/src/model/stores/block_transactions.rs b/consensus/src/model/stores/block_transactions.rs index 20fb08b..f43f4dd 100644 --- a/consensus/src/model/stores/block_transactions.rs +++ b/consensus/src/model/stores/block_transactions.rs @@ -9,7 +9,6 @@ use spectre_database::prelude::{BatchDbWriter, CachedDbAccess, DirectDbWriter}; use spectre_database::registry::DatabaseStorePrefixes; use spectre_hashes::Hash; use spectre_utils::mem_size::MemSizeEstimator; -use std::mem::size_of; use std::sync::Arc; pub trait BlockTransactionsStoreReader { diff --git a/consensus/src/model/stores/ghostdag.rs b/consensus/src/model/stores/ghostdag.rs index 67dcc06..9f1b39a 100644 --- a/consensus/src/model/stores/ghostdag.rs +++ b/consensus/src/model/stores/ghostdag.rs @@ -14,7 +14,6 @@ use rocksdb::WriteBatch; use serde::{Deserialize, Serialize}; use spectre_utils::mem_size::MemSizeEstimator; use std::iter::once; -use std::mem::size_of; use std::{cell::RefCell, sync::Arc}; /// Re-export for convenience diff --git a/consensus/src/model/stores/headers.rs b/consensus/src/model/stores/headers.rs index 9834c37..8fe98d8 100644 --- a/consensus/src/model/stores/headers.rs +++ b/consensus/src/model/stores/headers.rs @@ -1,4 +1,3 @@ -use std::mem::size_of; use std::sync::Arc; use rocksdb::WriteBatch; diff --git a/consensus/src/model/stores/mod.rs b/consensus/src/model/stores/mod.rs index 457d499..94076fc 100644 --- a/consensus/src/model/stores/mod.rs +++ b/consensus/src/model/stores/mod.rs @@ -3,10 +3,6 @@ pub mod block_transactions; pub mod block_window_cache; pub mod children; pub mod daa; -pub mod selected_chain; -use std::{fmt::Display, mem::size_of}; - -pub use spectre_database; pub mod depth; pub mod ghostdag; pub mod headers; @@ -16,6 +12,7 @@ pub mod pruning; pub mod pruning_utxoset; pub mod reachability; pub mod relations; +pub mod selected_chain; pub mod statuses; pub mod tips; pub mod utxo_diffs; @@ -23,7 +20,9 @@ pub mod utxo_multisets; pub mod utxo_set; pub mod virtual_state; +pub use spectre_database; pub use spectre_database::prelude::DB; +use std::fmt::Display; #[derive(PartialEq, Eq, Clone, Copy, Hash)] pub(crate) struct U64Key([u8; size_of::()]); diff --git a/consensus/src/model/stores/utxo_set.rs b/consensus/src/model/stores/utxo_set.rs index 77d2540..8ebabac 100644 --- a/consensus/src/model/stores/utxo_set.rs +++ b/consensus/src/model/stores/utxo_set.rs @@ -28,7 +28,7 @@ pub trait UtxoSetStore: UtxoSetStoreReader { fn write_many(&mut self, utxos: &[(TransactionOutpoint, UtxoEntry)]) -> Result<(), StoreError>; } -pub const UTXO_KEY_SIZE: usize = spectre_hashes::HASH_SIZE + std::mem::size_of::(); +pub const UTXO_KEY_SIZE: usize = spectre_hashes::HASH_SIZE + size_of::(); #[derive(Eq, Hash, PartialEq, Debug, Copy, Clone)] struct UtxoKey([u8; UTXO_KEY_SIZE]); @@ -81,8 +81,7 @@ impl From for TransactionOutpoint { fn from(k: UtxoKey) -> Self { let transaction_id = Hash::from_slice(&k.0[..spectre_hashes::HASH_SIZE]); let index = TransactionIndexType::from_le_bytes( - <[u8; std::mem::size_of::()]>::try_from(&k.0[spectre_hashes::HASH_SIZE..]) - .expect("expecting index size"), + <[u8; size_of::()]>::try_from(&k.0[spectre_hashes::HASH_SIZE..]).expect("expecting index size"), ); Self::new(transaction_id, index) } diff --git a/consensus/src/pipeline/body_processor/body_validation_in_context.rs b/consensus/src/pipeline/body_processor/body_validation_in_context.rs index e7e9e63..f615643 100644 --- a/consensus/src/pipeline/body_processor/body_validation_in_context.rs +++ b/consensus/src/pipeline/body_processor/body_validation_in_context.rs @@ -64,7 +64,7 @@ impl BlockBodyProcessor { .copied() .filter(|parent| { let status_option = statuses_read_guard.get(*parent).unwrap_option(); - status_option.is_none_or(|s| !s.has_block_body()) + status_option.is_none_or_ex(|s| !s.has_block_body()) }) .collect(); if !missing.is_empty() { diff --git a/consensus/src/processes/coinbase.rs b/consensus/src/processes/coinbase.rs index 1b87690..ca1d563 100644 --- a/consensus/src/processes/coinbase.rs +++ b/consensus/src/processes/coinbase.rs @@ -5,7 +5,7 @@ use spectre_consensus_core::{ tx::{ScriptPublicKey, ScriptVec, Transaction, TransactionOutput}, BlockHashMap, BlockHashSet, }; -use std::{convert::TryInto, mem::size_of}; +use std::convert::TryInto; use crate::{constants, model::stores::ghostdag::GhostdagData}; diff --git a/consensus/src/processes/mass.rs b/consensus/src/processes/mass.rs index 18451fe..6c44774 100644 --- a/consensus/src/processes/mass.rs +++ b/consensus/src/processes/mass.rs @@ -1,19 +1,9 @@ +pub use spectre_consensus_core::mass::Kip9Version; use spectre_consensus_core::{ mass::transaction_estimated_serialized_size, tx::{Transaction, VerifiableTransaction}, }; -/// Temp enum for the transition phases of KIP9 -#[derive(Copy, Clone, PartialEq, Eq)] -pub enum Kip9Version { - /// Initial KIP9 mass calculation, w/o the relaxed formula and summing storage mass and compute mass - Alpha, - - /// Currently proposed KIP9 mass calculation, with the relaxed formula (for the cases `|O| = 1 OR |O| <= |I| <= 2`), - /// and using a maximum operator over storage and compute mass - Beta, -} - // TODO (aspect) - review and potentially merge this with the new MassCalculator currently located in the wallet core // (i.e. migrate mass calculator from wallet core here or to consensus core) #[derive(Clone)] diff --git a/consensus/src/processes/pruning.rs b/consensus/src/processes/pruning.rs index 4fc92ef..595c1fe 100644 --- a/consensus/src/processes/pruning.rs +++ b/consensus/src/processes/pruning.rs @@ -213,7 +213,7 @@ impl< let mut expected_pps_queue = VecDeque::new(); for current in self.reachability_service.backward_chain_iterator(hst, pruning_info.pruning_point, false) { let current_header = self.headers_store.get_header(current).unwrap(); - if expected_pps_queue.back().is_none_or(|&&h| h != current_header.pruning_point) { + if expected_pps_queue.back().is_none_or_ex(|&&h| h != current_header.pruning_point) { expected_pps_queue.push_back(current_header.pruning_point); } } diff --git a/consensus/src/processes/sync/mod.rs b/consensus/src/processes/sync/mod.rs index 3cbdcb7..ac75a1d 100644 --- a/consensus/src/processes/sync/mod.rs +++ b/consensus/src/processes/sync/mod.rs @@ -191,7 +191,7 @@ impl< } } - if highest_with_body.is_none_or(|&h| h == high) { + if highest_with_body.is_none_or_ex(|&h| h == high) { return Ok(vec![]); }; diff --git a/consensus/wasm/Cargo.toml b/consensus/wasm/Cargo.toml index 83c333c..cd4de62 100644 --- a/consensus/wasm/Cargo.toml +++ b/consensus/wasm/Cargo.toml @@ -35,5 +35,5 @@ wasm-bindgen.workspace = true workflow-wasm.workspace = true workflow-log.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/crypto/addresses/Cargo.toml b/crypto/addresses/Cargo.toml index 63f13e2..04a4a57 100644 --- a/crypto/addresses/Cargo.toml +++ b/crypto/addresses/Cargo.toml @@ -28,5 +28,5 @@ web-sys.workspace = true name = "bench" harness = false -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/crypto/addresses/src/bech32.rs b/crypto/addresses/src/bech32.rs index 496e2a5..32b0253 100644 --- a/crypto/addresses/src/bech32.rs +++ b/crypto/addresses/src/bech32.rs @@ -122,11 +122,16 @@ impl Address { }) .collect::>(); err?; + if address.len() < 8 { + return Err(AddressError::BadPayload); + } + let (payload_u5, checksum_u5) = address_u5.split_at(address.len() - 8); let fivebit_prefix = prefix.as_str().as_bytes().iter().copied().map(|c| c & 0x1fu8); // Convert to number - let checksum_ = u64::from_be_bytes([vec![0u8; 3], conv5to8(checksum_u5)].concat().try_into().expect("Is exactly 8 bytes")); + let checksum_ = + u64::from_be_bytes([vec![0u8; 3], conv5to8(checksum_u5)].concat().try_into().map_err(|_| AddressError::BadChecksumSize)?); if checksum(payload_u5, fivebit_prefix) != checksum_ { return Err(AddressError::BadChecksum); diff --git a/crypto/addresses/src/lib.rs b/crypto/addresses/src/lib.rs index 0c486fb..61b6634 100644 --- a/crypto/addresses/src/lib.rs +++ b/crypto/addresses/src/lib.rs @@ -28,9 +28,15 @@ pub enum AddressError { #[error("The address contains an invalid character {0}")] DecodingError(char), + #[error("The address checksum is invalid (must be exactly 8 bytes)")] + BadChecksumSize, + #[error("The address checksum is invalid")] BadChecksum, + #[error("The address payload is invalid")] + BadPayload, + #[error("The address is invalid")] InvalidAddress, @@ -49,6 +55,7 @@ impl From for AddressError { /// Address prefix identifying the network type this address belongs to (such as `spectre`, `spectretest`, `spectresim`, `spectredev`). #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Hash, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[borsh(use_discriminant = true)] pub enum Prefix { #[serde(rename = "spectre")] Mainnet, @@ -117,6 +124,7 @@ impl TryFrom<&str> for Prefix { /// @category Address #[derive(PartialEq, Eq, PartialOrd, Ord, Clone, Copy, Debug, Hash, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[repr(u8)] +#[borsh(use_discriminant = true)] #[wasm_bindgen(js_name = "AddressVersion")] pub enum Version { /// PubKey addresses always have the version byte set to 0 @@ -281,11 +289,10 @@ impl BorshSerialize for Address { } impl BorshDeserialize for Address { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { - // Deserialize into vec first since we have no custom smallvec support - let prefix: Prefix = borsh::BorshDeserialize::deserialize(buf)?; - let version: Version = borsh::BorshDeserialize::deserialize(buf)?; - let payload: Vec = borsh::BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let prefix: Prefix = borsh::BorshDeserialize::deserialize_reader(reader)?; + let version: Version = borsh::BorshDeserialize::deserialize_reader(reader)?; + let payload: Vec = borsh::BorshDeserialize::deserialize_reader(reader)?; Ok(Self::new(prefix, version, &payload)) } } @@ -489,8 +496,11 @@ impl<'de> Deserialize<'de> for Address { impl TryCastFromJs for Address { type Error = AddressError; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(string) = value.as_ref().as_string() { Address::try_from(string) } else if let Some(object) = js_sys::Object::try_from(value.as_ref()) { @@ -512,6 +522,8 @@ extern "C" { pub type AddressOrStringArrayT; #[wasm_bindgen(extends = js_sys::Array, typescript_type = "Address[]")] pub type AddressArrayT; + #[wasm_bindgen(typescript_type = "Address | undefined")] + pub type AddressOrUndefinedT; } impl TryFrom for Vec
{ diff --git a/crypto/hashes/src/lib.rs b/crypto/hashes/src/lib.rs index e53c95e..afc0db9 100644 --- a/crypto/hashes/src/lib.rs +++ b/crypto/hashes/src/lib.rs @@ -187,8 +187,11 @@ impl Hash { type TryFromError = workflow_wasm::error::Error; impl TryCastFromJs for Hash { type Error = TryFromError; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let bytes = value.as_ref().try_as_vec_u8()?; Ok(Hash( <[u8; HASH_SIZE]>::try_from(bytes) diff --git a/crypto/muhash/fuzz/fuzz_targets/u3072.rs b/crypto/muhash/fuzz/fuzz_targets/u3072.rs index c4b1b07..99d2e18 100644 --- a/crypto/muhash/fuzz/fuzz_targets/u3072.rs +++ b/crypto/muhash/fuzz/fuzz_targets/u3072.rs @@ -4,7 +4,6 @@ use spectre_muhash::u3072::{self, U3072}; use num_bigint::BigInt; use num_integer::Integer; use num_traits::{One, Signed}; -use std::mem::size_of; fuzz_target!(|data: &[u8]| { if data.len() < muhash::SERIALIZED_MUHASH_SIZE { diff --git a/crypto/muhash/src/u3072.rs b/crypto/muhash/src/u3072.rs index 86560de..85a0d5c 100644 --- a/crypto/muhash/src/u3072.rs +++ b/crypto/muhash/src/u3072.rs @@ -15,8 +15,8 @@ pub(crate) type DoubleLimb = u128; //#[cfg(target_pointer_width = "32")] //pub(crate) type DoubleLimb = u64; -const LIMB_SIZE_BYTES: usize = std::mem::size_of::(); -const LIMB_SIZE: usize = std::mem::size_of::() * 8; +const LIMB_SIZE_BYTES: usize = size_of::(); +const LIMB_SIZE: usize = Limb::BITS as usize; pub const LIMBS: usize = crate::ELEMENT_BYTE_SIZE / LIMB_SIZE_BYTES; pub const PRIME_DIFF: Limb = 1103717; diff --git a/crypto/txscript/Cargo.toml b/crypto/txscript/Cargo.toml index 337ab68..f68e0bb 100644 --- a/crypto/txscript/Cargo.toml +++ b/crypto/txscript/Cargo.toml @@ -9,24 +9,35 @@ include.workspace = true license.workspace = true repository.workspace = true +[features] +wasm32-core = [] +wasm32-sdk = [] + [dependencies] blake2b_simd.workspace = true borsh.workspace = true +cfg-if.workspace = true +hexplay.workspace = true indexmap.workspace = true itertools.workspace = true spectre-addresses.workspace = true spectre-consensus-core.workspace = true spectre-hashes.workspace = true spectre-txscript-errors.workspace = true +spectre-utils.workspace = true +spectre-wasm-core.workspace = true log.workspace = true parking_lot.workspace = true rand.workspace = true secp256k1.workspace = true +serde_json.workspace = true +serde-wasm-bindgen.workspace = true serde.workspace = true sha2.workspace = true smallvec.workspace = true thiserror.workspace = true wasm-bindgen.workspace = true +workflow-wasm.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/crypto/txscript/src/caches.rs b/crypto/txscript/src/caches.rs index 3f76b25..46ec663 100644 --- a/crypto/txscript/src/caches.rs +++ b/crypto/txscript/src/caches.rs @@ -32,9 +32,8 @@ impl Option { - self.map.read().get(key).cloned().map(|data| { + self.map.read().get(key).cloned().inspect(|_data| { self.counters.get_counts.fetch_add(1, Ordering::Relaxed); - data }) } diff --git a/crypto/txscript/src/data_stack.rs b/crypto/txscript/src/data_stack.rs index 0bb0130..f24c646 100644 --- a/crypto/txscript/src/data_stack.rs +++ b/crypto/txscript/src/data_stack.rs @@ -1,7 +1,6 @@ use crate::TxScriptError; use core::fmt::Debug; use core::iter; -use core::mem::size_of; const DEFAULT_SCRIPT_NUM_LEN: usize = 4; diff --git a/crypto/txscript/src/error.rs b/crypto/txscript/src/error.rs new file mode 100644 index 0000000..a90d35e --- /dev/null +++ b/crypto/txscript/src/error.rs @@ -0,0 +1,89 @@ +use crate::script_builder; +use thiserror::Error; +use wasm_bindgen::{JsError, JsValue}; +use workflow_wasm::jserror::JsErrorData; + +#[derive(Debug, Error, Clone)] +pub enum Error { + #[error("{0}")] + Custom(String), + + #[error(transparent)] + JsValue(JsErrorData), + + #[error(transparent)] + Wasm(#[from] workflow_wasm::error::Error), + + #[error(transparent)] + ScriptBuilder(#[from] script_builder::ScriptBuilderError), + + #[error("{0}")] + ParseInt(#[from] std::num::ParseIntError), + + #[error(transparent)] + SerdeWasmBindgen(JsErrorData), + + #[error(transparent)] + NetworkType(#[from] spectre_consensus_core::network::NetworkTypeError), + + #[error("Error converting property `{0}`: {1}")] + Convert(&'static str, String), + + #[error("Error processing JSON: {0}")] + SerdeJson(String), +} + +impl Error { + pub fn custom>(msg: T) -> Self { + Error::Custom(msg.into()) + } + + pub fn convert(prop: &'static str, msg: S) -> Self { + Self::Convert(prop, msg.to_string()) + } +} + +impl From for Error { + fn from(err: String) -> Self { + Self::Custom(err) + } +} + +impl From<&str> for Error { + fn from(err: &str) -> Self { + Self::Custom(err.to_string()) + } +} + +impl From for JsValue { + fn from(value: Error) -> Self { + match value { + Error::JsValue(js_error_data) => js_error_data.into(), + _ => JsValue::from(value.to_string()), + } + } +} + +impl From for Error { + fn from(err: JsValue) -> Self { + Self::JsValue(err.into()) + } +} + +impl From for Error { + fn from(err: JsError) -> Self { + Self::JsValue(err.into()) + } +} + +impl From for Error { + fn from(err: serde_json::Error) -> Self { + Self::SerdeJson(err.to_string()) + } +} + +impl From for Error { + fn from(err: serde_wasm_bindgen::Error) -> Self { + Self::SerdeWasmBindgen(JsValue::from(err).into()) + } +} diff --git a/crypto/txscript/src/lib.rs b/crypto/txscript/src/lib.rs index fb53f43..31bdcd4 100644 --- a/crypto/txscript/src/lib.rs +++ b/crypto/txscript/src/lib.rs @@ -3,10 +3,14 @@ extern crate core; pub mod caches; mod data_stack; +pub mod error; pub mod opcodes; +pub mod result; pub mod script_builder; pub mod script_class; pub mod standard; +#[cfg(feature = "wasm32-sdk")] +pub mod wasm; use crate::caches::Cache; use crate::data_stack::{DataStack, Stack}; diff --git a/crypto/txscript/src/opcodes/mod.rs b/crypto/txscript/src/opcodes/mod.rs index 312ffc9..40b7db7 100644 --- a/crypto/txscript/src/opcodes/mod.rs +++ b/crypto/txscript/src/opcodes/mod.rs @@ -1,5 +1,3 @@ -use core::mem::size_of; - #[macro_use] mod macros; diff --git a/crypto/txscript/src/result.rs b/crypto/txscript/src/result.rs new file mode 100644 index 0000000..4c8cb83 --- /dev/null +++ b/crypto/txscript/src/result.rs @@ -0,0 +1 @@ +pub type Result = std::result::Result; diff --git a/crypto/txscript/src/script_builder.rs b/crypto/txscript/src/script_builder.rs index e3e08d6..acb0f6a 100644 --- a/crypto/txscript/src/script_builder.rs +++ b/crypto/txscript/src/script_builder.rs @@ -5,6 +5,7 @@ use crate::{ opcodes::{codes::*, OP_1_NEGATE_VAL, OP_DATA_MAX_VAL, OP_DATA_MIN_VAL, OP_SMALL_INT_MAX_VAL}, MAX_SCRIPTS_SIZE, MAX_SCRIPT_ELEMENT_SIZE, }; +use hexplay::{HexView, HexViewBuilder}; use thiserror::Error; /// DEFAULT_SCRIPT_ALLOC is the default size used for the backing array @@ -69,7 +70,7 @@ impl ScriptBuilder { &self.script } - #[cfg(test)] + #[cfg(any(test, target_arch = "wasm32"))] pub fn extend(&mut self, data: &[u8]) { self.script.extend(data); } @@ -248,6 +249,16 @@ impl ScriptBuilder { let trimmed = &buffer[0..trimmed_size]; self.add_data(trimmed) } + + /// Return [`HexViewBuilder`] for the script + pub fn hex_view_builder(&self) -> HexViewBuilder<'_> { + HexViewBuilder::new(&self.script) + } + + /// Return ready to use [`HexView`] for the script + pub fn hex_view(&self, offset: usize, width: usize) -> HexView<'_> { + HexViewBuilder::new(&self.script).address_offset(offset).row_width(width).finish() + } } impl Default for ScriptBuilder { diff --git a/crypto/txscript/src/script_class.rs b/crypto/txscript/src/script_class.rs index 37534c0..6ace437 100644 --- a/crypto/txscript/src/script_class.rs +++ b/crypto/txscript/src/script_class.rs @@ -17,6 +17,7 @@ pub enum Error { /// Standard classes of script payment in the blockDAG #[derive(PartialEq, Eq, Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[borsh(use_discriminant = true)] #[repr(u8)] pub enum ScriptClass { /// None of the recognized forms diff --git a/crypto/txscript/src/standard.rs b/crypto/txscript/src/standard.rs index bdb011d..df9b33c 100644 --- a/crypto/txscript/src/standard.rs +++ b/crypto/txscript/src/standard.rs @@ -215,7 +215,6 @@ mod tests { for test in tests { let extracted = extract_script_pub_key_address(&test.script_pub_key, test.prefix); - eprintln!("extracted: {:#?} - test.expected_address: {:#?}", extracted, test.expected_address); assert_eq!(extracted, test.expected_address, "extract address test failed for '{}'", test.name); if let Ok(ref address) = extracted { let encoded = pay_to_address_script(address); diff --git a/crypto/txscript/src/wasm/builder.rs b/crypto/txscript/src/wasm/builder.rs new file mode 100644 index 0000000..54ba608 --- /dev/null +++ b/crypto/txscript/src/wasm/builder.rs @@ -0,0 +1,179 @@ +use crate::result::Result; +use crate::{script_builder as native, standard}; +use spectre_consensus_core::tx::ScriptPublicKey; +use spectre_utils::hex::ToHex; +use spectre_wasm_core::hex::{HexViewConfig, HexViewConfigT}; +use spectre_wasm_core::types::{BinaryT, HexString}; +use std::cell::{Ref, RefCell, RefMut}; +use std::rc::Rc; +use wasm_bindgen::prelude::wasm_bindgen; +use workflow_wasm::prelude::*; + +/// ScriptBuilder provides a facility for building custom scripts. It allows +/// you to push opcodes, ints, and data while respecting canonical encoding. In +/// general it does not ensure the script will execute correctly, however any +/// data pushes which would exceed the maximum allowed script engine limits and +/// are therefore guaranteed not to execute will not be pushed and will result in +/// the Script function returning an error. +/// @category Consensus +#[derive(Clone)] +#[wasm_bindgen(inspectable)] +pub struct ScriptBuilder { + script_builder: Rc>, +} + +impl ScriptBuilder { + #[inline] + pub fn inner(&self) -> Ref<'_, native::ScriptBuilder> { + self.script_builder.borrow() + } + + #[inline] + pub fn inner_mut(&self) -> RefMut<'_, native::ScriptBuilder> { + self.script_builder.borrow_mut() + } +} + +impl Default for ScriptBuilder { + fn default() -> Self { + Self { script_builder: Rc::new(RefCell::new(native::ScriptBuilder::new())) } + } +} + +#[wasm_bindgen] +impl ScriptBuilder { + #[wasm_bindgen(constructor)] + pub fn new() -> Self { + Self::default() + } + + /// Creates a new ScriptBuilder over an existing script. + /// Supplied script can be represented as an `Uint8Array` or a `HexString`. + #[wasm_bindgen(js_name = "fromScript")] + pub fn from_script(script: BinaryT) -> Result { + let builder = ScriptBuilder::default(); + let script = script.try_as_vec_u8()?; + builder.inner_mut().extend(&script); + + Ok(builder) + } + + /// Pushes the passed opcode to the end of the script. The script will not + /// be modified if pushing the opcode would cause the script to exceed the + /// maximum allowed script engine size. + #[wasm_bindgen(js_name = "addOp")] + pub fn add_op(&self, op: u8) -> Result { + let mut inner = self.inner_mut(); + inner.add_op(op)?; + + Ok(self.clone()) + } + + /// Adds the passed opcodes to the end of the script. + /// Supplied opcodes can be represented as an `Uint8Array` or a `HexString`. + #[wasm_bindgen(js_name = "addOps")] + pub fn add_ops(&self, opcodes: BinaryT) -> Result { + let opcodes = opcodes.try_as_vec_u8()?; + self.inner_mut().add_ops(&opcodes)?; + + Ok(self.clone()) + } + + /// AddData pushes the passed data to the end of the script. It automatically + /// chooses canonical opcodes depending on the length of the data. + /// + /// A zero length buffer will lead to a push of empty data onto the stack (Op0 = OpFalse) + /// and any push of data greater than [`MAX_SCRIPT_ELEMENT_SIZE`](spectre_txscript::MAX_SCRIPT_ELEMENT_SIZE) will not modify + /// the script since that is not allowed by the script engine. + /// + /// Also, the script will not be modified if pushing the data would cause the script to + /// exceed the maximum allowed script engine size [`MAX_SCRIPTS_SIZE`](spectre_txscript::MAX_SCRIPTS_SIZE). + #[wasm_bindgen(js_name = "addData")] + pub fn add_data(&self, data: BinaryT) -> Result { + let data = data.try_as_vec_u8()?; + + let mut inner = self.inner_mut(); + inner.add_data(&data)?; + + Ok(self.clone()) + } + + #[wasm_bindgen(js_name = "addI64")] + pub fn add_i64(&self, value: i64) -> Result { + let mut inner = self.inner_mut(); + inner.add_i64(value)?; + + Ok(self.clone()) + } + + #[wasm_bindgen(js_name = "addLockTime")] + pub fn add_lock_time(&self, lock_time: u64) -> Result { + let mut inner = self.inner_mut(); + inner.add_lock_time(lock_time)?; + + Ok(self.clone()) + } + + #[wasm_bindgen(js_name = "addSequence")] + pub fn add_sequence(&self, sequence: u64) -> Result { + let mut inner = self.inner_mut(); + inner.add_sequence(sequence)?; + + Ok(self.clone()) + } + + #[wasm_bindgen(js_name = "canonicalDataSize")] + pub fn canonical_data_size(data: BinaryT) -> Result { + let data = data.try_as_vec_u8()?; + let size = native::ScriptBuilder::canonical_data_size(&data) as u32; + + Ok(size) + } + + /// Get script bytes represented by a hex string. + #[wasm_bindgen(js_name = "toString")] + pub fn to_string_js(&self) -> HexString { + let inner = self.inner(); + + HexString::from(inner.script()) + } + + /// Drains (empties) the script builder, returning the + /// script bytes represented by a hex string. + pub fn drain(&self) -> HexString { + let mut inner = self.inner_mut(); + + HexString::from(inner.drain().as_slice()) + } + + /// Creates an equivalent pay-to-script-hash script. + /// Can be used to create an P2SH address. + /// @see {@link addressFromScriptPublicKey} + #[wasm_bindgen(js_name = "createPayToScriptHashScript")] + pub fn pay_to_script_hash_script(&self) -> ScriptPublicKey { + let inner = self.inner(); + let script = inner.script(); + + standard::pay_to_script_hash_script(script) + } + + /// Generates a signature script that fits a pay-to-script-hash script. + #[wasm_bindgen(js_name = "encodePayToScriptHashSignatureScript")] + pub fn pay_to_script_hash_signature_script(&self, signature: BinaryT) -> Result { + let inner = self.inner(); + let script = inner.script(); + let signature = signature.try_as_vec_u8()?; + let generated_script = standard::pay_to_script_hash_signature_script(script.into(), signature)?; + + Ok(generated_script.to_hex().into()) + } + + #[wasm_bindgen(js_name = "hexView")] + pub fn hex_view(&self, args: Option) -> Result { + let inner = self.inner(); + let script = inner.script(); + + let config = args.map(HexViewConfig::try_from).transpose()?.unwrap_or_default(); + Ok(config.build(script).to_string()) + } +} diff --git a/crypto/txscript/src/wasm/mod.rs b/crypto/txscript/src/wasm/mod.rs new file mode 100644 index 0000000..e88e580 --- /dev/null +++ b/crypto/txscript/src/wasm/mod.rs @@ -0,0 +1,15 @@ +//! +//! WASM32 bindings for the txscript framework components. +//! + +use cfg_if::cfg_if; + +cfg_if! { + if #[cfg(any(feature = "wasm32-sdk", feature = "wasm32-core"))] { + pub mod opcodes; + pub mod builder; + + pub use self::opcodes::*; + pub use self::builder::*; + } +} diff --git a/consensus/client/src/script.rs b/crypto/txscript/src/wasm/opcodes.rs similarity index 51% rename from consensus/client/src/script.rs rename to crypto/txscript/src/wasm/opcodes.rs index cf9932f..643ede8 100644 --- a/consensus/client/src/script.rs +++ b/crypto/txscript/src/wasm/opcodes.rs @@ -1,20 +1,12 @@ -use std::cell::{Ref, RefCell, RefMut}; -use std::rc::Rc; +pub use wasm_bindgen::prelude::*; -use spectre_wasm_core::types::{BinaryT, HexString}; - -use crate::imports::*; -use crate::result::Result; -use spectre_txscript::script_builder as native; +/// Spectre Transaction Script Opcodes +/// @see {@link ScriptBuilder} +/// @category Consensus +#[wasm_bindgen] +pub enum Opcodes { + OpFalse = 0x00, -#[wasm_bindgen(typescript_custom_section)] -const TS_SCRIPT_OPCODES: &'static str = r#" -/** - * Spectre Transaction Script Opcodes - * @see {@link ScriptBuilder} - * @category Consensus - */ -export enum Opcode { OpData1 = 0x01, OpData2 = 0x02, OpData3 = 0x03, @@ -90,15 +82,17 @@ export enum Opcode { OpData73 = 0x49, OpData74 = 0x4a, OpData75 = 0x4b, + OpPushData1 = 0x4c, OpPushData2 = 0x4d, OpPushData4 = 0x4e, + Op1Negate = 0x4f, - /** - * Reserved - */ + OpReserved = 0x50, - Op1 = 0x51, + + OpTrue = 0x51, + Op2 = 0x52, Op3 = 0x53, Op4 = 0x54, @@ -114,27 +108,21 @@ export enum Opcode { Op14 = 0x5e, Op15 = 0x5f, Op16 = 0x60, + OpNop = 0x61, - /** - * Reserved - */ OpVer = 0x62, OpIf = 0x63, OpNotIf = 0x64, - /** - * Reserved - */ OpVerIf = 0x65, - /** - * Reserved - */ OpVerNotIf = 0x66, + OpElse = 0x67, OpEndIf = 0x68, OpVerify = 0x69, OpReturn = 0x6a, OpToAltStack = 0x6b, OpFromAltStack = 0x6c, + Op2Drop = 0x6d, Op2Dup = 0x6e, Op3Dup = 0x6f, @@ -148,88 +136,57 @@ export enum Opcode { OpNip = 0x77, OpOver = 0x78, OpPick = 0x79, + OpRoll = 0x7a, OpRot = 0x7b, OpSwap = 0x7c, OpTuck = 0x7d, - /** - * Disabled - */ + + /// Splice opcodes. OpCat = 0x7e, - /** - * Disabled - */ OpSubStr = 0x7f, - /** - * Disabled - */ OpLeft = 0x80, - /** - * Disabled - */ OpRight = 0x81, + OpSize = 0x82, - /** - * Disabled - */ + + /// Bitwise logic opcodes. OpInvert = 0x83, - /** - * Disabled - */ OpAnd = 0x84, - /** - * Disabled - */ OpOr = 0x85, - /** - * Disabled - */ OpXor = 0x86, + OpEqual = 0x87, OpEqualVerify = 0x88, + OpReserved1 = 0x89, OpReserved2 = 0x8a, + + /// Numeric related opcodes. Op1Add = 0x8b, Op1Sub = 0x8c, - /** - * Disabled - */ Op2Mul = 0x8d, - /** - * Disabled - */ Op2Div = 0x8e, OpNegate = 0x8f, OpAbs = 0x90, OpNot = 0x91, Op0NotEqual = 0x92, + OpAdd = 0x93, OpSub = 0x94, - /** - * Disabled - */ OpMul = 0x95, - /** - * Disabled - */ OpDiv = 0x96, - /** - * Disabled - */ OpMod = 0x97, - /** - * Disabled - */ OpLShift = 0x98, - /** - * Disabled - */ OpRShift = 0x99, + OpBoolAnd = 0x9a, OpBoolOr = 0x9b, + OpNumEqual = 0x9c, OpNumEqualVerify = 0x9d, OpNumNotEqual = 0x9e, + OpLessThan = 0x9f, OpGreaterThan = 0xa0, OpLessThanOrEqual = 0xa1, @@ -237,10 +194,16 @@ export enum Opcode { OpMin = 0xa3, OpMax = 0xa4, OpWithin = 0xa5, + + /// Undefined opcodes. OpUnknown166 = 0xa6, OpUnknown167 = 0xa7, - OpSha256 = 0xa8, + + /// Crypto opcodes. + OpSHA256 = 0xa8, + OpCheckMultiSigECDSA = 0xa9, + OpBlake2b = 0xaa, OpCheckSigECDSA = 0xab, OpCheckSig = 0xac, @@ -249,6 +212,8 @@ export enum Opcode { OpCheckMultiSigVerify = 0xaf, OpCheckLockTimeVerify = 0xb0, OpCheckSequenceVerify = 0xb1, + + /// Undefined opcodes. OpUnknown178 = 0xb2, OpUnknown179 = 0xb3, OpUnknown180 = 0xb4, @@ -321,6 +286,7 @@ export enum Opcode { OpUnknown247 = 0xf7, OpUnknown248 = 0xf8, OpUnknown249 = 0xf9, + OpSmallInteger = 0xfa, OpPubKeys = 0xfb, OpUnknown252 = 0xfc, @@ -328,130 +294,3 @@ export enum Opcode { OpPubKey = 0xfe, OpInvalidOpCode = 0xff, } - -"#; - -/// -/// ScriptBuilder provides a facility for building custom scripts. It allows -/// you to push opcodes, ints, and data while respecting canonical encoding. In -/// general it does not ensure the script will execute correctly, however any -/// data pushes which would exceed the maximum allowed script engine limits and -/// are therefore guaranteed not to execute will not be pushed and will result in -/// the Script function returning an error. -/// -/// @see {@link Opcode} -/// @category Consensus -#[derive(Clone)] -#[wasm_bindgen(inspectable)] -pub struct ScriptBuilder { - script_builder: Rc>, -} - -impl ScriptBuilder { - #[inline] - pub fn inner(&self) -> Ref<'_, native::ScriptBuilder> { - self.script_builder.borrow() - } - - #[inline] - pub fn inner_mut(&self) -> RefMut<'_, native::ScriptBuilder> { - self.script_builder.borrow_mut() - } -} - -impl Default for ScriptBuilder { - fn default() -> Self { - Self { script_builder: Rc::new(RefCell::new(spectre_txscript::script_builder::ScriptBuilder::new())) } - } -} - -#[wasm_bindgen] -impl ScriptBuilder { - #[wasm_bindgen(constructor)] - pub fn new() -> Self { - Self::default() - } - - #[wasm_bindgen(getter)] - pub fn data(&self) -> HexString { - self.script() - } - - /// Get script bytes represented by a hex string. - pub fn script(&self) -> HexString { - let inner = self.inner(); - HexString::from(inner.script()) - } - - /// Drains (empties) the script builder, returning the - /// script bytes represented by a hex string. - pub fn drain(&self) -> HexString { - let mut inner = self.inner_mut(); - HexString::from(inner.drain().as_slice()) - } - - #[wasm_bindgen(js_name = canonicalDataSize)] - pub fn canonical_data_size(data: BinaryT) -> Result { - let data = data.try_as_vec_u8()?; - let size = native::ScriptBuilder::canonical_data_size(&data) as u32; - Ok(size) - } - - /// Pushes the passed opcode to the end of the script. The script will not - /// be modified if pushing the opcode would cause the script to exceed the - /// maximum allowed script engine size. - #[wasm_bindgen(js_name = addOp)] - pub fn add_op(&self, op: u8) -> Result { - let mut inner = self.inner_mut(); - inner.add_op(op)?; - Ok(self.clone()) - } - - /// Adds the passed opcodes to the end of the script. - /// Supplied opcodes can be represented as a `Uint8Array` or a `HexString`. - #[wasm_bindgen(js_name = "addOps")] - pub fn add_ops(&self, opcodes: JsValue) -> Result { - let opcodes = opcodes.try_as_vec_u8()?; - self.inner_mut().add_ops(&opcodes)?; - Ok(self.clone()) - } - - /// AddData pushes the passed data to the end of the script. It automatically - /// chooses canonical opcodes depending on the length of the data. - /// - /// A zero length buffer will lead to a push of empty data onto the stack (Op0 = OpFalse) - /// and any push of data greater than [`MAX_SCRIPT_ELEMENT_SIZE`](spectre_txscript::MAX_SCRIPT_ELEMENT_SIZE) will not modify - /// the script since that is not allowed by the script engine. - /// - /// Also, the script will not be modified if pushing the data would cause the script to - /// exceed the maximum allowed script engine size [`MAX_SCRIPTS_SIZE`](spectre_txscript::MAX_SCRIPTS_SIZE). - #[wasm_bindgen(js_name = addData)] - pub fn add_data(&self, data: BinaryT) -> Result { - let data = data.try_as_vec_u8()?; - - let mut inner = self.inner_mut(); - inner.add_data(&data)?; - Ok(self.clone()) - } - - #[wasm_bindgen(js_name = addI64)] - pub fn add_i64(&self, value: i64) -> Result { - let mut inner = self.inner_mut(); - inner.add_i64(value)?; - Ok(self.clone()) - } - - #[wasm_bindgen(js_name = addLockTime)] - pub fn add_lock_time(&self, lock_time: u64) -> Result { - let mut inner = self.inner_mut(); - inner.add_lock_time(lock_time)?; - Ok(self.clone()) - } - - #[wasm_bindgen(js_name = addSequence)] - pub fn add_sequence(&self, sequence: u64) -> Result { - let mut inner = self.inner_mut(); - inner.add_sequence(sequence)?; - Ok(self.clone()) - } -} diff --git a/database/src/registry.rs b/database/src/registry.rs index 9e1b129..752efb9 100644 --- a/database/src/registry.rs +++ b/database/src/registry.rs @@ -95,8 +95,8 @@ mod tests { let prefix = DatabaseStorePrefixes::AcceptanceData; assert_eq!(&[prefix as u8], prefix.as_ref()); assert_eq!( - std::mem::size_of::(), - std::mem::size_of::(), + size_of::(), + size_of::(), "DatabaseStorePrefixes is expected to have the same memory layout of u8" ); } diff --git a/indexes/utxoindex/src/stores/indexed_utxos.rs b/indexes/utxoindex/src/stores/indexed_utxos.rs index 6e890b2..e6ab165 100644 --- a/indexes/utxoindex/src/stores/indexed_utxos.rs +++ b/indexes/utxoindex/src/stores/indexed_utxos.rs @@ -11,7 +11,6 @@ use spectre_hashes::Hash; use spectre_index_core::indexed_utxos::BalanceByScriptPublicKey; use std::collections::HashSet; use std::fmt::Display; -use std::mem::size_of; use std::sync::Arc; pub const VERSION_TYPE_SIZE: usize = size_of::(); // Const since we need to re-use this a few times. @@ -67,8 +66,7 @@ impl From for TransactionOutpoint { fn from(key: TransactionOutpointKey) -> Self { let transaction_id = Hash::from_slice(&key.0[..spectre_hashes::HASH_SIZE]); let index = TransactionIndexType::from_le_bytes( - <[u8; std::mem::size_of::()]>::try_from(&key.0[spectre_hashes::HASH_SIZE..]) - .expect("expected index size"), + <[u8; size_of::()]>::try_from(&key.0[spectre_hashes::HASH_SIZE..]).expect("expected index size"), ); Self::new(transaction_id, index) } diff --git a/math/src/uint.rs b/math/src/uint.rs index 0b5ab7a..379fc18 100644 --- a/math/src/uint.rs +++ b/math/src/uint.rs @@ -15,7 +15,7 @@ macro_rules! construct_uint { pub const MIN: Self = Self::ZERO; pub const MAX: Self = $name([u64::MAX; $n_words]); pub const BITS: u32 = $n_words * u64::BITS; - pub const BYTES: usize = $n_words * core::mem::size_of::(); + pub const BYTES: usize = $n_words * size_of::(); pub const LIMBS: usize = $n_words; #[inline] diff --git a/metrics/core/src/data.rs b/metrics/core/src/data.rs index 82929bf..3079faf 100644 --- a/metrics/core/src/data.rs +++ b/metrics/core/src/data.rs @@ -1,6 +1,9 @@ +use crate::error::Error; +use crate::result::Result; use borsh::{BorshDeserialize, BorshSerialize}; use separator::{separated_float, separated_int, separated_uint_with_output, Separatable}; use serde::{Deserialize, Serialize}; +use spectre_rpc_core::GetMetricsResponse; use workflow_core::enums::Describe; #[derive(Describe, Debug, Clone, Copy, Eq, PartialEq, Hash, BorshDeserialize, BorshSerialize, Serialize, Deserialize)] @@ -37,10 +40,6 @@ impl MetricGroup { } impl MetricGroup { - pub fn iter() -> impl Iterator { - [MetricGroup::System, MetricGroup::Storage, MetricGroup::Connections, MetricGroup::Network].into_iter() - } - pub fn metrics(&self) -> impl Iterator { match self { MetricGroup::System => [ @@ -56,6 +55,7 @@ impl MetricGroup { Metric::NodeDiskIoReadPerSec, Metric::NodeDiskIoWriteBytes, Metric::NodeDiskIoWritePerSec, + Metric::NodeStorageSizeBytes, ] .as_slice() .iter(), @@ -127,7 +127,8 @@ impl From for MetricGroup { | Metric::NodeDiskIoReadBytes | Metric::NodeDiskIoWriteBytes | Metric::NodeDiskIoReadPerSec - | Metric::NodeDiskIoWritePerSec => MetricGroup::Storage, + | Metric::NodeDiskIoWritePerSec + | Metric::NodeStorageSizeBytes => MetricGroup::Storage, // -- Metric::NodeBorshLiveConnections | Metric::NodeBorshConnectionAttempts @@ -194,6 +195,7 @@ pub enum Metric { NodeDiskIoWriteBytes, NodeDiskIoReadPerSec, NodeDiskIoWritePerSec, + NodeStorageSizeBytes, // --- NodeActivePeers, NodeBorshLiveConnections, @@ -252,62 +254,62 @@ pub enum Metric { impl Metric { // TODO - this will be refactored at a later date // as this requires changes and testing in /kos - pub fn group(&self) -> &'static str { - match self { - Metric::NodeCpuUsage - | Metric::NodeResidentSetSizeBytes - | Metric::NodeVirtualMemorySizeBytes - | Metric::NodeFileHandlesCount - | Metric::NodeDiskIoReadBytes - | Metric::NodeDiskIoWriteBytes - | Metric::NodeDiskIoReadPerSec - | Metric::NodeDiskIoWritePerSec - | Metric::NodeBorshLiveConnections - | Metric::NodeBorshConnectionAttempts - | Metric::NodeBorshHandshakeFailures - | Metric::NodeJsonLiveConnections - | Metric::NodeJsonConnectionAttempts - | Metric::NodeJsonHandshakeFailures - | Metric::NodeBorshBytesTx - | Metric::NodeBorshBytesRx - | Metric::NodeJsonBytesTx - | Metric::NodeJsonBytesRx - | Metric::NodeP2pBytesTx - | Metric::NodeP2pBytesRx - | Metric::NodeGrpcUserBytesTx - | Metric::NodeGrpcUserBytesRx - | Metric::NodeTotalBytesTx - | Metric::NodeTotalBytesRx - | Metric::NodeBorshBytesTxPerSecond - | Metric::NodeBorshBytesRxPerSecond - | Metric::NodeJsonBytesTxPerSecond - | Metric::NodeJsonBytesRxPerSecond - | Metric::NodeP2pBytesTxPerSecond - | Metric::NodeP2pBytesRxPerSecond - | Metric::NodeGrpcUserBytesTxPerSecond - | Metric::NodeGrpcUserBytesRxPerSecond - | Metric::NodeTotalBytesTxPerSecond - | Metric::NodeTotalBytesRxPerSecond - | Metric::NodeActivePeers => "system", - // -- - Metric::NodeBlocksSubmittedCount - | Metric::NodeHeadersProcessedCount - | Metric::NodeDependenciesProcessedCount - | Metric::NodeBodiesProcessedCount - | Metric::NodeTransactionsProcessedCount - | Metric::NodeChainBlocksProcessedCount - | Metric::NodeMassProcessedCount - | Metric::NodeDatabaseBlocksCount - | Metric::NodeDatabaseHeadersCount - | Metric::NetworkMempoolSize - | Metric::NetworkTransactionsPerSecond - | Metric::NetworkTipHashesCount - | Metric::NetworkDifficulty - | Metric::NetworkPastMedianTime - | Metric::NetworkVirtualParentHashesCount - | Metric::NetworkVirtualDaaScore => "spectre", - } - } + // pub fn group(&self) -> &'static str { + // match self { + // Metric::NodeCpuUsage + // | Metric::NodeResidentSetSizeBytes + // | Metric::NodeVirtualMemorySizeBytes + // | Metric::NodeFileHandlesCount + // | Metric::NodeDiskIoReadBytes + // | Metric::NodeDiskIoWriteBytes + // | Metric::NodeDiskIoReadPerSec + // | Metric::NodeDiskIoWritePerSec + // | Metric::NodeBorshLiveConnections + // | Metric::NodeBorshConnectionAttempts + // | Metric::NodeBorshHandshakeFailures + // | Metric::NodeJsonLiveConnections + // | Metric::NodeJsonConnectionAttempts + // | Metric::NodeJsonHandshakeFailures + // | Metric::NodeBorshBytesTx + // | Metric::NodeBorshBytesRx + // | Metric::NodeJsonBytesTx + // | Metric::NodeJsonBytesRx + // | Metric::NodeP2pBytesTx + // | Metric::NodeP2pBytesRx + // | Metric::NodeGrpcUserBytesTx + // | Metric::NodeGrpcUserBytesRx + // | Metric::NodeTotalBytesTx + // | Metric::NodeTotalBytesRx + // | Metric::NodeBorshBytesTxPerSecond + // | Metric::NodeBorshBytesRxPerSecond + // | Metric::NodeJsonBytesTxPerSecond + // | Metric::NodeJsonBytesRxPerSecond + // | Metric::NodeP2pBytesTxPerSecond + // | Metric::NodeP2pBytesRxPerSecond + // | Metric::NodeGrpcUserBytesTxPerSecond + // | Metric::NodeGrpcUserBytesRxPerSecond + // | Metric::NodeTotalBytesTxPerSecond + // | Metric::NodeTotalBytesRxPerSecond + // | Metric::NodeActivePeers => "system", + // // -- + // Metric::NodeBlocksSubmittedCount + // | Metric::NodeHeadersProcessedCount + // | Metric::NodeDependenciesProcessedCount + // | Metric::NodeBodiesProcessedCount + // | Metric::NodeTransactionsProcessedCount + // | Metric::NodeChainBlocksProcessedCount + // | Metric::NodeMassProcessedCount + // | Metric::NodeDatabaseBlocksCount + // | Metric::NodeDatabaseHeadersCount + // | Metric::NetworkMempoolSize + // | Metric::NetworkTransactionsPerSecond + // | Metric::NetworkTipHashesCount + // | Metric::NetworkDifficulty + // | Metric::NetworkPastMedianTime + // | Metric::NetworkVirtualParentHashesCount + // | Metric::NetworkVirtualDaaScore => "spectre", + // } + // } pub fn is_key_performance_metric(&self) -> bool { matches!( @@ -362,6 +364,7 @@ impl Metric { Metric::NodeDiskIoWriteBytes => as_mb(f, si, short), Metric::NodeDiskIoReadPerSec => format!("{}/s", as_data_size(f, si)), Metric::NodeDiskIoWritePerSec => format!("{}/s", as_data_size(f, si)), + Metric::NodeStorageSizeBytes => as_gb(f, si, short), // -- Metric::NodeBorshLiveConnections => f.trunc().separated_string(), Metric::NodeBorshConnectionAttempts => f.trunc().separated_string(), @@ -425,6 +428,7 @@ impl Metric { Metric::NodeDiskIoWriteBytes => ("Storage Write", "Stor Write"), Metric::NodeDiskIoReadPerSec => ("Storage Read/s", "Stor Read"), Metric::NodeDiskIoWritePerSec => ("Storage Write/s", "Stor Write"), + Metric::NodeStorageSizeBytes => ("Storage Size", "Stor Size"), // -- Metric::NodeActivePeers => ("Active p2p Peers", "Peers"), Metric::NodeBorshLiveConnections => ("Borsh Active Connections", "Borsh Conn"), @@ -493,6 +497,7 @@ pub struct MetricsData { pub node_disk_io_write_bytes: u64, pub node_disk_io_read_per_sec: f32, pub node_disk_io_write_per_sec: f32, + pub node_storage_size_bytes: u64, // --- pub node_borsh_live_connections: u32, pub node_borsh_connection_attempts: u64, @@ -512,17 +517,6 @@ pub struct MetricsData { pub node_grpc_user_bytes_rx: u64, pub node_total_bytes_tx: u64, pub node_total_bytes_rx: u64, - - pub node_borsh_bytes_tx_per_second: u64, - pub node_borsh_bytes_rx_per_second: u64, - pub node_json_bytes_tx_per_second: u64, - pub node_json_bytes_rx_per_second: u64, - pub node_p2p_bytes_tx_per_second: u64, - pub node_p2p_bytes_rx_per_second: u64, - pub node_grpc_user_bytes_tx_per_second: u64, - pub node_grpc_user_bytes_rx_per_second: u64, - pub node_total_bytes_tx_per_second: u64, - pub node_total_bytes_rx_per_second: u64, // --- pub node_blocks_submitted_count: u64, pub node_headers_processed_count: u64, @@ -549,6 +543,87 @@ impl MetricsData { } } +impl TryFrom for MetricsData { + type Error = Error; + fn try_from(response: GetMetricsResponse) -> Result { + let GetMetricsResponse { + server_time, + consensus_metrics, + connection_metrics, + bandwidth_metrics, + process_metrics, + storage_metrics, + custom_metrics: _, + } = response; //rpc.get_metrics(true, true, true, true, true, false).await?; + + let consensus_metrics = consensus_metrics.ok_or(Error::MissingData("Consensus Metrics"))?; + let connection_metrics = connection_metrics.ok_or(Error::MissingData("Connection Metrics"))?; + let bandwidth_metrics = bandwidth_metrics.ok_or(Error::MissingData("Bandwidth Metrics"))?; + let process_metrics = process_metrics.ok_or(Error::MissingData("Process Metrics"))?; + let storage_metrics = storage_metrics.ok_or(Error::MissingData("Storage Metrics"))?; + + Ok(MetricsData { + unixtime_millis: server_time as f64, + + node_blocks_submitted_count: consensus_metrics.node_blocks_submitted_count, + node_headers_processed_count: consensus_metrics.node_headers_processed_count, + node_dependencies_processed_count: consensus_metrics.node_dependencies_processed_count, + node_bodies_processed_count: consensus_metrics.node_bodies_processed_count, + node_transactions_processed_count: consensus_metrics.node_transactions_processed_count, + node_chain_blocks_processed_count: consensus_metrics.node_chain_blocks_processed_count, + node_mass_processed_count: consensus_metrics.node_mass_processed_count, + // -- + node_database_blocks_count: consensus_metrics.node_database_blocks_count, + node_database_headers_count: consensus_metrics.node_database_headers_count, + network_mempool_size: consensus_metrics.network_mempool_size, + network_tip_hashes_count: consensus_metrics.network_tip_hashes_count, + network_difficulty: consensus_metrics.network_difficulty, + network_past_median_time: consensus_metrics.network_past_median_time, + network_virtual_parent_hashes_count: consensus_metrics.network_virtual_parent_hashes_count, + network_virtual_daa_score: consensus_metrics.network_virtual_daa_score, + + node_borsh_live_connections: connection_metrics.borsh_live_connections, + node_borsh_connection_attempts: connection_metrics.borsh_connection_attempts, + node_borsh_handshake_failures: connection_metrics.borsh_handshake_failures, + node_json_live_connections: connection_metrics.json_live_connections, + node_json_connection_attempts: connection_metrics.json_connection_attempts, + node_json_handshake_failures: connection_metrics.json_handshake_failures, + node_active_peers: connection_metrics.active_peers, + + node_borsh_bytes_tx: bandwidth_metrics.borsh_bytes_tx, + node_borsh_bytes_rx: bandwidth_metrics.borsh_bytes_rx, + node_json_bytes_tx: bandwidth_metrics.json_bytes_tx, + node_json_bytes_rx: bandwidth_metrics.json_bytes_rx, + node_p2p_bytes_tx: bandwidth_metrics.p2p_bytes_tx, + node_p2p_bytes_rx: bandwidth_metrics.p2p_bytes_rx, + node_grpc_user_bytes_tx: bandwidth_metrics.grpc_bytes_tx, + node_grpc_user_bytes_rx: bandwidth_metrics.grpc_bytes_rx, + + node_total_bytes_tx: bandwidth_metrics.borsh_bytes_tx + + bandwidth_metrics.json_bytes_tx + + bandwidth_metrics.p2p_bytes_tx + + bandwidth_metrics.grpc_bytes_tx, + + node_total_bytes_rx: bandwidth_metrics.borsh_bytes_rx + + bandwidth_metrics.json_bytes_rx + + bandwidth_metrics.p2p_bytes_rx + + bandwidth_metrics.grpc_bytes_rx, + + node_resident_set_size_bytes: process_metrics.resident_set_size, + node_virtual_memory_size_bytes: process_metrics.virtual_memory_size, + node_cpu_cores: process_metrics.core_num, + node_cpu_usage: process_metrics.cpu_usage, + node_file_handles: process_metrics.fd_num, + node_disk_io_read_bytes: process_metrics.disk_io_read_bytes, + node_disk_io_write_bytes: process_metrics.disk_io_write_bytes, + node_disk_io_read_per_sec: process_metrics.disk_io_read_per_sec, + node_disk_io_write_per_sec: process_metrics.disk_io_write_per_sec, + + node_storage_size_bytes: storage_metrics.storage_size_bytes, + }) + } +} + #[derive(Default, Debug, Clone, BorshDeserialize, BorshSerialize, Serialize, Deserialize)] pub struct MetricsSnapshot { pub data: MetricsData, @@ -615,6 +690,8 @@ pub struct MetricsSnapshot { pub network_past_median_time: f64, pub network_virtual_parent_hashes_count: f64, pub network_virtual_daa_score: f64, + // --- + pub node_storage_size_bytes: f64, } impl MetricsSnapshot { @@ -629,6 +706,7 @@ impl MetricsSnapshot { Metric::NodeDiskIoWriteBytes => self.node_disk_io_write_bytes, Metric::NodeDiskIoReadPerSec => self.node_disk_io_read_per_sec, Metric::NodeDiskIoWritePerSec => self.node_disk_io_write_per_sec, + Metric::NodeStorageSizeBytes => self.node_storage_size_bytes, // --- Metric::NodeActivePeers => self.node_active_peers, Metric::NodeBorshLiveConnections => self.node_borsh_active_connections, @@ -725,6 +803,7 @@ impl From<(&MetricsData, &MetricsData)> for MetricsSnapshot { node_disk_io_write_bytes: b.node_disk_io_write_bytes as f64, node_disk_io_read_per_sec: b.node_disk_io_read_per_sec as f64, node_disk_io_write_per_sec: b.node_disk_io_write_per_sec as f64, + node_storage_size_bytes: b.node_storage_size_bytes as f64, // --- node_borsh_active_connections: b.node_borsh_live_connections as f64, node_borsh_connection_attempts: b.node_borsh_connection_attempts as f64, diff --git a/metrics/core/src/error.rs b/metrics/core/src/error.rs index d72fb8f..8bef8eb 100644 --- a/metrics/core/src/error.rs +++ b/metrics/core/src/error.rs @@ -6,6 +6,9 @@ pub enum Error { #[error("{0}")] Custom(String), + #[error("Missing metrics data `{0}`")] + MissingData(&'static str), + #[error(transparent)] RpcError(#[from] RpcError), } diff --git a/metrics/core/src/lib.rs b/metrics/core/src/lib.rs index fce7cd2..bcd4b44 100644 --- a/metrics/core/src/lib.rs +++ b/metrics/core/src/lib.rs @@ -6,7 +6,7 @@ pub use data::{Metric, MetricGroup, MetricsData, MetricsSnapshot}; use crate::result::Result; use futures::{pin_mut, select, FutureExt, StreamExt}; -use spectre_rpc_core::{api::rpc::RpcApi, GetMetricsResponse}; +use spectre_rpc_core::api::rpc::RpcApi; use std::{ future::Future, pin::Pin, @@ -81,23 +81,31 @@ impl Metrics { }, _ = interval.next().fuse() => { - let last_metrics_data = current_metrics_data; - current_metrics_data = MetricsData::new(unixtime_as_millis_f64()); + // current_metrics_data = MetricsData::new(unixtime_as_millis_f64()); if let Some(rpc) = this.rpc() { - if let Err(err) = this.sample_metrics(rpc.clone(), &mut current_metrics_data).await { - log_trace!("Metrics::sample_metrics() error: {}", err); + // if let Err(err) = this.sample_metrics(rpc.clone(), &mut current_metrics_data).await { + match this.sample_metrics(rpc.clone()).await { + Ok(incoming_data) => { + let last_metrics_data = current_metrics_data; + current_metrics_data = incoming_data; + this.data.lock().unwrap().replace(current_metrics_data.clone()); + + if let Some(sink) = this.sink() { + let snapshot = MetricsSnapshot::from((&last_metrics_data, ¤t_metrics_data)); + if let Some(future) = sink(snapshot) { + future.await.ok(); + } + } + + } + Err(err) => { + // current_metrics_data = last_metrics_data.clone(); + log_trace!("Metrics::sample_metrics() error: {}", err); + } } } - this.data.lock().unwrap().replace(current_metrics_data.clone()); - - if let Some(sink) = this.sink() { - let snapshot = MetricsSnapshot::from((&last_metrics_data, ¤t_metrics_data)); - if let Some(future) = sink(snapshot) { - future.await.ok(); - } - } } } } @@ -114,72 +122,85 @@ impl Metrics { // --- samplers - async fn sample_metrics(self: &Arc, rpc: Arc, data: &mut MetricsData) -> Result<()> { - let GetMetricsResponse { server_time: _, consensus_metrics, connection_metrics, bandwidth_metrics, process_metrics } = - rpc.get_metrics(true, true, true, true).await?; - - if let Some(consensus_metrics) = consensus_metrics { - data.node_blocks_submitted_count = consensus_metrics.node_blocks_submitted_count; - data.node_headers_processed_count = consensus_metrics.node_headers_processed_count; - data.node_dependencies_processed_count = consensus_metrics.node_dependencies_processed_count; - data.node_bodies_processed_count = consensus_metrics.node_bodies_processed_count; - data.node_transactions_processed_count = consensus_metrics.node_transactions_processed_count; - data.node_chain_blocks_processed_count = consensus_metrics.node_chain_blocks_processed_count; - data.node_mass_processed_count = consensus_metrics.node_mass_processed_count; - // -- - data.node_database_blocks_count = consensus_metrics.node_database_blocks_count; - data.node_database_headers_count = consensus_metrics.node_database_headers_count; - data.network_mempool_size = consensus_metrics.network_mempool_size; - data.network_tip_hashes_count = consensus_metrics.network_tip_hashes_count; - data.network_difficulty = consensus_metrics.network_difficulty; - data.network_past_median_time = consensus_metrics.network_past_median_time; - data.network_virtual_parent_hashes_count = consensus_metrics.network_virtual_parent_hashes_count; - data.network_virtual_daa_score = consensus_metrics.network_virtual_daa_score; - } - - if let Some(connection_metrics) = connection_metrics { - data.node_borsh_live_connections = connection_metrics.borsh_live_connections; - data.node_borsh_connection_attempts = connection_metrics.borsh_connection_attempts; - data.node_borsh_handshake_failures = connection_metrics.borsh_handshake_failures; - data.node_json_live_connections = connection_metrics.json_live_connections; - data.node_json_connection_attempts = connection_metrics.json_connection_attempts; - data.node_json_handshake_failures = connection_metrics.json_handshake_failures; - data.node_active_peers = connection_metrics.active_peers; - } - - if let Some(bandwidth_metrics) = bandwidth_metrics { - data.node_borsh_bytes_tx = bandwidth_metrics.borsh_bytes_tx; - data.node_borsh_bytes_rx = bandwidth_metrics.borsh_bytes_rx; - data.node_json_bytes_tx = bandwidth_metrics.json_bytes_tx; - data.node_json_bytes_rx = bandwidth_metrics.json_bytes_rx; - data.node_p2p_bytes_tx = bandwidth_metrics.p2p_bytes_tx; - data.node_p2p_bytes_rx = bandwidth_metrics.p2p_bytes_rx; - data.node_grpc_user_bytes_tx = bandwidth_metrics.grpc_bytes_tx; - data.node_grpc_user_bytes_rx = bandwidth_metrics.grpc_bytes_rx; - - data.node_total_bytes_tx = bandwidth_metrics.borsh_bytes_tx - + bandwidth_metrics.json_bytes_tx - + bandwidth_metrics.p2p_bytes_tx - + bandwidth_metrics.grpc_bytes_tx; - - data.node_total_bytes_rx = bandwidth_metrics.borsh_bytes_rx - + bandwidth_metrics.json_bytes_rx - + bandwidth_metrics.p2p_bytes_rx - + bandwidth_metrics.grpc_bytes_rx; - } - - if let Some(process_metrics) = process_metrics { - data.node_resident_set_size_bytes = process_metrics.resident_set_size; - data.node_virtual_memory_size_bytes = process_metrics.virtual_memory_size; - data.node_cpu_cores = process_metrics.core_num; - data.node_cpu_usage = process_metrics.cpu_usage; - data.node_file_handles = process_metrics.fd_num; - data.node_disk_io_read_bytes = process_metrics.disk_io_read_bytes; - data.node_disk_io_write_bytes = process_metrics.disk_io_write_bytes; - data.node_disk_io_read_per_sec = process_metrics.disk_io_read_per_sec; - data.node_disk_io_write_per_sec = process_metrics.disk_io_write_per_sec; - } - - Ok(()) + async fn sample_metrics(self: &Arc, rpc: Arc) -> Result { + // let GetMetricsResponse { + // server_time: _, + // consensus_metrics, + // connection_metrics, + // bandwidth_metrics, + // process_metrics, + // storage_metrics, + // custom_metrics: _, + // } = + let response = rpc.get_metrics(true, true, true, true, true, false).await?; + + MetricsData::try_from(response) + + // if let Some(consensus_metrics) = consensus_metrics { + // data.node_blocks_submitted_count = consensus_metrics.node_blocks_submitted_count; + // data.node_headers_processed_count = consensus_metrics.node_headers_processed_count; + // data.node_dependencies_processed_count = consensus_metrics.node_dependencies_processed_count; + // data.node_bodies_processed_count = consensus_metrics.node_bodies_processed_count; + // data.node_transactions_processed_count = consensus_metrics.node_transactions_processed_count; + // data.node_chain_blocks_processed_count = consensus_metrics.node_chain_blocks_processed_count; + // data.node_mass_processed_count = consensus_metrics.node_mass_processed_count; + // // -- + // data.node_database_blocks_count = consensus_metrics.node_database_blocks_count; + // data.node_database_headers_count = consensus_metrics.node_database_headers_count; + // data.network_mempool_size = consensus_metrics.network_mempool_size; + // data.network_tip_hashes_count = consensus_metrics.network_tip_hashes_count; + // data.network_difficulty = consensus_metrics.network_difficulty; + // data.network_past_median_time = consensus_metrics.network_past_median_time; + // data.network_virtual_parent_hashes_count = consensus_metrics.network_virtual_parent_hashes_count; + // data.network_virtual_daa_score = consensus_metrics.network_virtual_daa_score; + // } + + // if let Some(connection_metrics) = connection_metrics { + // data.node_borsh_live_connections = connection_metrics.borsh_live_connections; + // data.node_borsh_connection_attempts = connection_metrics.borsh_connection_attempts; + // data.node_borsh_handshake_failures = connection_metrics.borsh_handshake_failures; + // data.node_json_live_connections = connection_metrics.json_live_connections; + // data.node_json_connection_attempts = connection_metrics.json_connection_attempts; + // data.node_json_handshake_failures = connection_metrics.json_handshake_failures; + // data.node_active_peers = connection_metrics.active_peers; + // } + + // if let Some(bandwidth_metrics) = bandwidth_metrics { + // data.node_borsh_bytes_tx = bandwidth_metrics.borsh_bytes_tx; + // data.node_borsh_bytes_rx = bandwidth_metrics.borsh_bytes_rx; + // data.node_json_bytes_tx = bandwidth_metrics.json_bytes_tx; + // data.node_json_bytes_rx = bandwidth_metrics.json_bytes_rx; + // data.node_p2p_bytes_tx = bandwidth_metrics.p2p_bytes_tx; + // data.node_p2p_bytes_rx = bandwidth_metrics.p2p_bytes_rx; + // data.node_grpc_user_bytes_tx = bandwidth_metrics.grpc_bytes_tx; + // data.node_grpc_user_bytes_rx = bandwidth_metrics.grpc_bytes_rx; + + // data.node_total_bytes_tx = bandwidth_metrics.borsh_bytes_tx + // + bandwidth_metrics.json_bytes_tx + // + bandwidth_metrics.p2p_bytes_tx + // + bandwidth_metrics.grpc_bytes_tx; + + // data.node_total_bytes_rx = bandwidth_metrics.borsh_bytes_rx + // + bandwidth_metrics.json_bytes_rx + // + bandwidth_metrics.p2p_bytes_rx + // + bandwidth_metrics.grpc_bytes_rx; + // } + + // if let Some(process_metrics) = process_metrics { + // data.node_resident_set_size_bytes = process_metrics.resident_set_size; + // data.node_virtual_memory_size_bytes = process_metrics.virtual_memory_size; + // data.node_cpu_cores = process_metrics.core_num; + // data.node_cpu_usage = process_metrics.cpu_usage; + // data.node_file_handles = process_metrics.fd_num; + // data.node_disk_io_read_bytes = process_metrics.disk_io_read_bytes; + // data.node_disk_io_write_bytes = process_metrics.disk_io_write_bytes; + // data.node_disk_io_read_per_sec = process_metrics.disk_io_read_per_sec; + // data.node_disk_io_write_per_sec = process_metrics.disk_io_write_per_sec; + // } + + // if let Some(storage_metrics) = storage_metrics { + // data.node_storage_size_bytes = storage_metrics.storage_size_bytes; + // } + // Ok(()) } } diff --git a/mining/src/testutils/coinbase_mock.rs b/mining/src/testutils/coinbase_mock.rs index 1f7a4bd..37fa645 100644 --- a/mining/src/testutils/coinbase_mock.rs +++ b/mining/src/testutils/coinbase_mock.rs @@ -4,7 +4,6 @@ use spectre_consensus_core::{ subnets::SUBNETWORK_ID_COINBASE, tx::{Transaction, TransactionOutput}, }; -use std::mem::size_of; const LENGTH_OF_BLUE_SCORE: usize = size_of::(); const LENGTH_OF_SUBSIDY: usize = size_of::(); diff --git a/notify/Cargo.toml b/notify/Cargo.toml index 2e676bf..2583b3c 100644 --- a/notify/Cargo.toml +++ b/notify/Cargo.toml @@ -34,6 +34,7 @@ thiserror.workspace = true triggered.workspace = true workflow-core.workspace = true workflow-log.workspace = true +workflow-serializer.workspace = true [dev-dependencies] criterion.workspace = true diff --git a/notify/src/scope.rs b/notify/src/scope.rs index f993004..010d517 100644 --- a/notify/src/scope.rs +++ b/notify/src/scope.rs @@ -3,6 +3,7 @@ use borsh::{BorshDeserialize, BorshSerialize}; use derive_more::Display; use serde::{Deserialize, Serialize}; use spectre_addresses::Address; +use workflow_serializer::prelude::*; macro_rules! scope_enum { ($(#[$meta:meta])* $vis:vis enum $name:ident { @@ -53,9 +54,38 @@ impl Scope { } } +impl Serializer for Scope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Scope, self, writer)?; + Ok(()) + } +} + +impl Deserializer for Scope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + load!(Scope, reader) + } +} + #[derive(Clone, Display, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct BlockAddedScope {} +impl Serializer for BlockAddedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for BlockAddedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct VirtualChainChangedScope { pub include_accepted_transaction_ids: bool, @@ -73,12 +103,56 @@ impl std::fmt::Display for VirtualChainChangedScope { } } +impl Serializer for VirtualChainChangedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.include_accepted_transaction_ids, writer)?; + Ok(()) + } +} + +impl Deserializer for VirtualChainChangedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let include_accepted_transaction_ids = load!(bool, reader)?; + Ok(Self { include_accepted_transaction_ids }) + } +} + #[derive(Clone, Display, Debug, PartialEq, Eq, Default, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct FinalityConflictScope {} +impl Serializer for FinalityConflictScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for FinalityConflictScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Display, Debug, PartialEq, Eq, Default, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct FinalityConflictResolvedScope {} +impl Serializer for FinalityConflictResolvedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for FinalityConflictResolvedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Debug, Default, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct UtxosChangedScope { pub addresses: Vec
, @@ -109,14 +183,86 @@ impl UtxosChangedScope { } } +impl Serializer for UtxosChangedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec
, &self.addresses, writer)?; + Ok(()) + } +} + +impl Deserializer for UtxosChangedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let addresses = load!(Vec
, reader)?; + Ok(Self { addresses }) + } +} + #[derive(Clone, Display, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct SinkBlueScoreChangedScope {} +impl Serializer for SinkBlueScoreChangedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for SinkBlueScoreChangedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Display, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct VirtualDaaScoreChangedScope {} +impl Serializer for VirtualDaaScoreChangedScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for VirtualDaaScoreChangedScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Display, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct PruningPointUtxoSetOverrideScope {} +impl Serializer for PruningPointUtxoSetOverrideScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for PruningPointUtxoSetOverrideScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + #[derive(Clone, Display, Debug, Default, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] pub struct NewBlockTemplateScope {} + +impl Serializer for NewBlockTemplateScope { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NewBlockTemplateScope { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} diff --git a/notify/src/subscription/mod.rs b/notify/src/subscription/mod.rs index 6cded47..ff468be 100644 --- a/notify/src/subscription/mod.rs +++ b/notify/src/subscription/mod.rs @@ -16,6 +16,7 @@ pub mod context; pub mod single; #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[borsh(use_discriminant = true)] pub enum Command { Start = 0, Stop = 1, diff --git a/protocol/flows/src/flowcontext/orphans.rs b/protocol/flows/src/flowcontext/orphans.rs index 6e067b0..3bf1a7f 100644 --- a/protocol/flows/src/flowcontext/orphans.rs +++ b/protocol/flows/src/flowcontext/orphans.rs @@ -166,7 +166,7 @@ impl OrphanBlocksPool { } } else { let status = consensus.async_get_block_status(current).await; - if status.is_none_or(|s| s.is_header_only()) { + if status.is_none_or_ex(|s| s.is_header_only()) { // Block is not in the orphan pool nor does its body exist consensus-wise, so it is a root roots.push(current); } @@ -193,7 +193,8 @@ impl OrphanBlocksPool { if let Occupied(entry) = self.orphans.entry(orphan_hash) { let mut processable = true; for p in entry.get().block.header.direct_parents().iter().copied() { - if !processing.contains_key(&p) && consensus.async_get_block_status(p).await.is_none_or(|s| s.is_header_only()) { + if !processing.contains_key(&p) && consensus.async_get_block_status(p).await.is_none_or_ex(|s| s.is_header_only()) + { processable = false; break; } @@ -249,7 +250,7 @@ impl OrphanBlocksPool { let mut processable = true; for parent in block.block.header.direct_parents().iter().copied() { if self.orphans.contains_key(&parent) - || consensus.async_get_block_status(parent).await.is_none_or(|status| status.is_header_only()) + || consensus.async_get_block_status(parent).await.is_none_or_ex(|status| status.is_header_only()) { processable = false; break; diff --git a/protocol/p2p/src/convert/net_address.rs b/protocol/p2p/src/convert/net_address.rs index 2cf2d03..885fb17 100644 --- a/protocol/p2p/src/convert/net_address.rs +++ b/protocol/p2p/src/convert/net_address.rs @@ -1,7 +1,4 @@ -use std::{ - mem::size_of, - net::{IpAddr, Ipv4Addr, Ipv6Addr}, -}; +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr}; use super::error::ConversionError; use crate::pb as protowire; diff --git a/rothschild/src/main.rs b/rothschild/src/main.rs index 6f9256f..e76fff4 100644 --- a/rothschild/src/main.rs +++ b/rothschild/src/main.rs @@ -16,7 +16,7 @@ use spectre_consensus_core::{ use spectre_core::{info, spectred_env::version, time::unix_now, warn}; use spectre_grpc_client::{ClientPool, GrpcClient}; use spectre_notify::subscription::context::SubscriptionContext; -use spectre_rpc_core::{api::rpc::RpcApi, notify::mode::NotificationMode}; +use spectre_rpc_core::{api::rpc::RpcApi, notify::mode::NotificationMode, RpcUtxoEntry}; use spectre_txscript::pay_to_address_script; use tokio::time::{interval, MissedTickBehavior}; @@ -48,7 +48,7 @@ impl Args { Args { private_key: m.get_one::("private-key").cloned(), tps: m.get_one::("tps").cloned().unwrap(), - rpc_server: m.get_one::("rpcserver").cloned().unwrap_or("localhost:18210".to_owned()), + rpc_server: m.get_one::("rpcserver").cloned().unwrap_or("localhost:16210".to_owned()), threads: m.get_one::("threads").cloned().unwrap(), unleashed: m.get_one::("unleashed").cloned().unwrap_or(false), } @@ -74,7 +74,7 @@ pub fn cli() -> Command { .long("rpcserver") .short('s') .value_name("rpcserver") - .default_value("localhost:18210") + .default_value("localhost:16210") .help("RPC server"), ) .arg( @@ -323,7 +323,7 @@ async fn populate_pending_outpoints_from_mempool( for entry in entries { for entry in entry.sending { for input in entry.transaction.inputs { - pending_outpoints.insert(input.previous_outpoint, now); + pending_outpoints.insert(input.previous_outpoint.into(), now); } } } @@ -337,20 +337,20 @@ async fn fetch_spendable_utxos( ) -> Vec<(TransactionOutpoint, UtxoEntry)> { let resp = rpc_client.get_utxos_by_addresses(vec![spectre_addr]).await.unwrap(); let dag_info = rpc_client.get_block_dag_info().await.unwrap(); - let mut utxos = Vec::with_capacity(resp.len()); - for resp_entry in resp - .into_iter() - .filter(|resp_entry| is_utxo_spendable(&resp_entry.utxo_entry, dag_info.virtual_daa_score, coinbase_maturity)) + + let mut utxos = resp.into_iter() + .filter(|entry| { + is_utxo_spendable(&entry.utxo_entry, dag_info.virtual_daa_score, coinbase_maturity) + }) + .map(|entry| (TransactionOutpoint::from(entry.outpoint), UtxoEntry::from(entry.utxo_entry))) // Eliminates UTXOs we already tried to spend so we don't try to spend them again in this period - .filter(|utxo| !pending.contains_key(&utxo.outpoint)) - { - utxos.push((resp_entry.outpoint, resp_entry.utxo_entry)); - } + .filter(|(outpoint,_)| !pending.contains_key(outpoint)) + .collect::>(); utxos.sort_by(|a, b| b.1.amount.cmp(&a.1.amount)); utxos } -fn is_utxo_spendable(entry: &UtxoEntry, virtual_daa_score: u64, coinbase_maturity: u64) -> bool { +fn is_utxo_spendable(entry: &RpcUtxoEntry, virtual_daa_score: u64, coinbase_maturity: u64) -> bool { let needed_confs = if !entry.is_coinbase { 10 } else { diff --git a/rpc/core/Cargo.toml b/rpc/core/Cargo.toml index f2724a0..63b9389 100644 --- a/rpc/core/Cargo.toml +++ b/rpc/core/Cargo.toml @@ -42,6 +42,7 @@ hex.workspace = true js-sys.workspace = true log.workspace = true paste.workspace = true +rand.workspace = true serde-wasm-bindgen.workspace = true serde.workspace = true smallvec.workspace = true @@ -49,10 +50,11 @@ thiserror.workspace = true uuid.workspace = true wasm-bindgen.workspace = true workflow-core.workspace = true +workflow-serializer.workspace = true workflow-wasm.workspace = true [dev-dependencies] serde_json.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/rpc/core/src/api/connection.rs b/rpc/core/src/api/connection.rs new file mode 100644 index 0000000..5b42542 --- /dev/null +++ b/rpc/core/src/api/connection.rs @@ -0,0 +1,7 @@ +use std::sync::Arc; + +pub trait RpcConnection: Send + Sync { + fn id(&self) -> u64; +} + +pub type DynRpcConnection = Arc; diff --git a/rpc/core/src/api/mod.rs b/rpc/core/src/api/mod.rs index 6bc968b..1373bd6 100644 --- a/rpc/core/src/api/mod.rs +++ b/rpc/core/src/api/mod.rs @@ -1,3 +1,4 @@ +pub mod connection; pub mod ctl; pub mod notifications; pub mod ops; diff --git a/rpc/core/src/api/notifications.rs b/rpc/core/src/api/notifications.rs index ad5d7e4..cf17df0 100644 --- a/rpc/core/src/api/notifications.rs +++ b/rpc/core/src/api/notifications.rs @@ -1,5 +1,4 @@ use crate::model::message::*; -use borsh::{BorshDeserialize, BorshSerialize}; use derive_more::Display; use serde::{Deserialize, Serialize}; use spectre_notify::{ @@ -13,10 +12,11 @@ use spectre_notify::{ }; use std::sync::Arc; use wasm_bindgen::JsValue; +use workflow_serializer::prelude::*; use workflow_wasm::serde::to_value; full_featured! { -#[derive(Clone, Debug, Display, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Display, Serialize, Deserialize)] pub enum Notification { #[display(fmt = "BlockAdded notification: block hash {}", "_0.block.header.hash")] BlockAdded(BlockAddedNotification), @@ -113,14 +113,92 @@ impl NotificationTrait for Notification { } } -#[cfg(test)] -mod test { - use super::*; +impl Serializer for Notification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + match self { + Notification::BlockAdded(notification) => { + store!(u16, &0, writer)?; + serialize!(BlockAddedNotification, notification, writer)?; + } + Notification::VirtualChainChanged(notification) => { + store!(u16, &1, writer)?; + serialize!(VirtualChainChangedNotification, notification, writer)?; + } + Notification::FinalityConflict(notification) => { + store!(u16, &2, writer)?; + serialize!(FinalityConflictNotification, notification, writer)?; + } + Notification::FinalityConflictResolved(notification) => { + store!(u16, &3, writer)?; + serialize!(FinalityConflictResolvedNotification, notification, writer)?; + } + Notification::UtxosChanged(notification) => { + store!(u16, &4, writer)?; + serialize!(UtxosChangedNotification, notification, writer)?; + } + Notification::SinkBlueScoreChanged(notification) => { + store!(u16, &5, writer)?; + serialize!(SinkBlueScoreChangedNotification, notification, writer)?; + } + Notification::VirtualDaaScoreChanged(notification) => { + store!(u16, &6, writer)?; + serialize!(VirtualDaaScoreChangedNotification, notification, writer)?; + } + Notification::PruningPointUtxoSetOverride(notification) => { + store!(u16, &7, writer)?; + serialize!(PruningPointUtxoSetOverrideNotification, notification, writer)?; + } + Notification::NewBlockTemplate(notification) => { + store!(u16, &8, writer)?; + serialize!(NewBlockTemplateNotification, notification, writer)?; + } + } + Ok(()) + } +} - #[test] - fn test_notification_from_bytes() { - let bytes = &vec![6, 169, 167, 75, 2, 0, 0, 0, 0][..]; - let notification = Notification::try_from_slice(bytes); - println!("notification: {notification:?}"); +impl Deserializer for Notification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + match load!(u16, reader)? { + 0 => { + let notification = deserialize!(BlockAddedNotification, reader)?; + Ok(Notification::BlockAdded(notification)) + } + 1 => { + let notification = deserialize!(VirtualChainChangedNotification, reader)?; + Ok(Notification::VirtualChainChanged(notification)) + } + 2 => { + let notification = deserialize!(FinalityConflictNotification, reader)?; + Ok(Notification::FinalityConflict(notification)) + } + 3 => { + let notification = deserialize!(FinalityConflictResolvedNotification, reader)?; + Ok(Notification::FinalityConflictResolved(notification)) + } + 4 => { + let notification = deserialize!(UtxosChangedNotification, reader)?; + Ok(Notification::UtxosChanged(notification)) + } + 5 => { + let notification = deserialize!(SinkBlueScoreChangedNotification, reader)?; + Ok(Notification::SinkBlueScoreChanged(notification)) + } + 6 => { + let notification = deserialize!(VirtualDaaScoreChangedNotification, reader)?; + Ok(Notification::VirtualDaaScoreChanged(notification)) + } + 7 => { + let notification = deserialize!(PruningPointUtxoSetOverrideNotification, reader)?; + Ok(Notification::PruningPointUtxoSetOverride(notification)) + } + 8 => { + let notification = deserialize!(NewBlockTemplateNotification, reader)?; + Ok(Notification::NewBlockTemplate(notification)) + } + _ => Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid variant")), + } } } diff --git a/rpc/core/src/api/ops.rs b/rpc/core/src/api/ops.rs index 8b26c8c..dd7ff92 100644 --- a/rpc/core/src/api/ops.rs +++ b/rpc/core/src/api/ops.rs @@ -3,124 +3,133 @@ use serde::{Deserialize, Serialize}; use spectre_notify::events::EventType; use workflow_core::enums::Describe; -/// Rpc Api version (4 x short values); First short is reserved. -/// The version format is as follows: `[reserved, major, minor, patch]`. -/// The difference in the major version value indicates breaking binary API changes -/// (i.e. changes in non-versioned model data structures) -/// If such change occurs, BorshRPC-client should refuse to connect to the -/// server and should request a client-side upgrade. JsonRPC-client may opt-in to -/// continue interop, but data structures should handle mutations by pre-filtering -/// or using Serde attributes. This applies only to RPC infrastructure that uses internal -/// data structures and does not affect gRPC. gRPC should issue and handle its -/// own versioning. -pub const RPC_API_VERSION: [u16; 4] = [0, 1, 0, 0]; +/// API version. Change in this value should result +/// in the client refusing to connect. +pub const RPC_API_VERSION: u16 = 1; +/// API revision. Change in this value denotes +/// backwards-compatible changes. +pub const RPC_API_REVISION: u16 = 0; #[derive(Describe, Clone, Copy, Debug, PartialEq, Eq, Hash, BorshSerialize, BorshDeserialize, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] +#[borsh(use_discriminant = true)] pub enum RpcApiOps { + NoOp = 0, + + // connection control (provisional) + Connect = 1, + Disconnect = 2, + + // subscription management + Subscribe = 3, + Unsubscribe = 4, + + // ~~~ + + // Subscription commands for starting/stopping notifications + NotifyBlockAdded = 10, + NotifyNewBlockTemplate = 11, + NotifyUtxosChanged = 12, + NotifyPruningPointUtxoSetOverride = 13, + NotifyFinalityConflict = 14, + NotifyFinalityConflictResolved = 15, // for uniformity purpose only since subscribing to NotifyFinalityConflict means receiving both FinalityConflict and FinalityConflictResolved + NotifyVirtualDaaScoreChanged = 16, + NotifyVirtualChainChanged = 17, + NotifySinkBlueScoreChanged = 18, + + // Notification ops required by wRPC + + // TODO: Remove these ops and use EventType as NotificationOps when workflow_rpc::server::interface::Interface + // will be generic over a MethodOps and NotificationOps instead of a single Ops param. + BlockAddedNotification = 60, + VirtualChainChangedNotification = 61, + FinalityConflictNotification = 62, + FinalityConflictResolvedNotification = 63, + UtxosChangedNotification = 64, + SinkBlueScoreChangedNotification = 65, + VirtualDaaScoreChangedNotification = 66, + PruningPointUtxoSetOverrideNotification = 67, + NewBlockTemplateNotification = 68, + + // RPC methods /// Ping the node to check if connection is alive - Ping = 0, + Ping = 110, /// Get metrics for consensus information and node performance - GetMetrics, + GetMetrics = 111, + /// Get system information (RAM available, number of cores, available file descriptors) + GetSystemInfo = 112, + /// Get current number of active TCP connections + GetConnections = 113, /// Get state information on the node - GetServerInfo, + GetServerInfo = 114, /// Get the current sync status of the node - GetSyncStatus, + GetSyncStatus = 115, /// Returns the network this Spectred is connected to (Mainnet, Testnet) - GetCurrentNetwork, + GetCurrentNetwork = 116, /// Extracts a block out of the request message and attempts to add it to the DAG Returns an empty response or an error message - SubmitBlock, + SubmitBlock = 117, /// Returns a "template" by which a miner can mine a new block - GetBlockTemplate, + GetBlockTemplate = 118, /// Returns a list of all the addresses (IP, port) this Spectred knows and a list of all addresses that are currently banned by this Spectred - GetPeerAddresses, + GetPeerAddresses = 119, /// Returns the hash of the current selected tip block of the DAG - GetSink, + GetSink = 120, /// Get information about an entry in the node's mempool - GetMempoolEntry, + GetMempoolEntry = 121, /// Get a snapshot of the node's mempool - GetMempoolEntries, + GetMempoolEntries = 122, /// Returns a list of the peers currently connected to this Spectred, along with some statistics on them - GetConnectedPeerInfo, + GetConnectedPeerInfo = 123, /// Instructs Spectred to connect to a given IP address. - AddPeer, + AddPeer = 124, /// Extracts a transaction out of the request message and attempts to add it to the mempool Returns an empty response or an error message - SubmitTransaction, + SubmitTransaction = 125, /// Requests info on a block corresponding to a given block hash Returns block info if the block is known. - GetBlock, + GetBlock = 126, // - GetSubnetwork, + GetSubnetwork = 127, // - GetVirtualChainFromBlock, + GetVirtualChainFromBlock = 128, // - GetBlocks, + GetBlocks = 129, /// Returns the amount of blocks in the DAG - GetBlockCount, + GetBlockCount = 130, /// Returns info on the current state of the DAG - GetBlockDagInfo, + GetBlockDagInfo = 131, // - ResolveFinalityConflict, + ResolveFinalityConflict = 132, /// Instructs this node to shut down Returns an empty response or an error message - Shutdown, + Shutdown = 133, // - GetHeaders, + GetHeaders = 134, /// Get a list of available UTXOs for a given address - GetUtxosByAddresses, + GetUtxosByAddresses = 135, /// Get a balance for a given address - GetBalanceByAddress, + GetBalanceByAddress = 136, /// Get a balance for a number of addresses - GetBalancesByAddresses, + GetBalancesByAddresses = 137, // ? - GetSinkBlueScore, + GetSinkBlueScore = 138, /// Ban a specific peer by it's IP address - Ban, + Ban = 139, /// Unban a specific peer by it's IP address - Unban, + Unban = 140, /// Get generic node information - GetInfo, + GetInfo = 141, // - EstimateNetworkHashesPerSecond, + EstimateNetworkHashesPerSecond = 142, /// Get a list of mempool entries that belong to a specific address - GetMempoolEntriesByAddresses, + GetMempoolEntriesByAddresses = 143, /// Get current issuance supply - GetCoinSupply, + GetCoinSupply = 144, /// Get DAA Score timestamp estimate - GetDaaScoreTimestampEstimate, - - // Subscription commands for starting/stopping notifications - NotifyBlockAdded, - NotifyNewBlockTemplate, - NotifyUtxosChanged, - NotifyPruningPointUtxoSetOverride, - NotifyFinalityConflict, - NotifyFinalityConflictResolved, // for uniformity purpose only since subscribing to NotifyFinalityConflict means receiving both FinalityConflict and FinalityConflictResolved - NotifyVirtualDaaScoreChanged, - NotifyVirtualChainChanged, - NotifySinkBlueScoreChanged, - - // ~ - Subscribe, - Unsubscribe, - - // Notification ops required by wRPC - // TODO: Remove these ops and use EventType as NotificationOps when workflow_rpc::server::interface::Interface - // will be generic over a MethodOps and NotificationOps instead of a single Ops param. - BlockAddedNotification, - VirtualChainChangedNotification, - FinalityConflictNotification, - FinalityConflictResolvedNotification, - UtxosChangedNotification, - SinkBlueScoreChangedNotification, - VirtualDaaScoreChangedNotification, - PruningPointUtxoSetOverrideNotification, - NewBlockTemplateNotification, - + GetDaaScoreTimestampEstimate = 145, /// Extracts a transaction out of the request message and attempts to replace a matching transaction in the mempool with it, applying a mandatory Replace by Fee policy - SubmitTransactionReplacement, - - // Fee estimation related commands - GetFeeEstimate, - GetFeeEstimateExperimental, + SubmitTransactionReplacement = 146, + /// Fee estimation + GetFeeEstimate = 147, + /// Fee estimation (experimental) + GetFeeEstimateExperimental = 148, } impl RpcApiOps { diff --git a/rpc/core/src/api/rpc.rs b/rpc/core/src/api/rpc.rs index 0e39073..10d0ca8 100644 --- a/rpc/core/src/api/rpc.rs +++ b/rpc/core/src/api/rpc.rs @@ -4,6 +4,7 @@ //! All data provided by the RCP server can be trusted by the client //! No data submitted by the client to the server can be trusted +use crate::api::connection::DynRpcConnection; use crate::{model::*, notify::connection::ChannelConnection, RpcResult}; use async_trait::async_trait; use downcast::{downcast_sync, AnySync}; @@ -21,10 +22,32 @@ pub const MAX_SAFE_WINDOW_SIZE: u32 = 10_000; pub trait RpcApi: Sync + Send + AnySync { /// async fn ping(&self) -> RpcResult<()> { - self.ping_call(PingRequest {}).await?; + self.ping_call(None, PingRequest {}).await?; Ok(()) } - async fn ping_call(&self, request: PingRequest) -> RpcResult; + async fn ping_call(&self, connection: Option<&DynRpcConnection>, request: PingRequest) -> RpcResult; + + // --- + + async fn get_system_info(&self) -> RpcResult { + Ok(self.get_system_info_call(None, GetSystemInfoRequest {}).await?) + } + async fn get_system_info_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetSystemInfoRequest, + ) -> RpcResult; + + // --- + + async fn get_connections(&self, include_profile_data: bool) -> RpcResult { + self.get_connections_call(None, GetConnectionsRequest { include_profile_data }).await + } + async fn get_connections_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetConnectionsRequest, + ) -> RpcResult; // --- @@ -34,59 +57,100 @@ pub trait RpcApi: Sync + Send + AnySync { connection_metrics: bool, bandwidth_metrics: bool, consensus_metrics: bool, + storage_metrics: bool, + custom_metrics: bool, ) -> RpcResult { - self.get_metrics_call(GetMetricsRequest { process_metrics, connection_metrics, bandwidth_metrics, consensus_metrics }).await - } - async fn get_metrics_call(&self, request: GetMetricsRequest) -> RpcResult; + self.get_metrics_call( + None, + GetMetricsRequest { + process_metrics, + connection_metrics, + bandwidth_metrics, + consensus_metrics, + storage_metrics, + custom_metrics, + }, + ) + .await + } + async fn get_metrics_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetMetricsRequest, + ) -> RpcResult; // get_info alternative that carries only version, network_id (full), is_synced, virtual_daa_score // these are the only variables needed to negotiate a wRPC connection (besides the wRPC handshake) async fn get_server_info(&self) -> RpcResult { - self.get_server_info_call(GetServerInfoRequest {}).await + self.get_server_info_call(None, GetServerInfoRequest {}).await } - async fn get_server_info_call(&self, request: GetServerInfoRequest) -> RpcResult; + async fn get_server_info_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetServerInfoRequest, + ) -> RpcResult; // Get current sync status of the node (should be converted to a notification subscription) async fn get_sync_status(&self) -> RpcResult { - Ok(self.get_sync_status_call(GetSyncStatusRequest {}).await?.is_synced) + Ok(self.get_sync_status_call(None, GetSyncStatusRequest {}).await?.is_synced) } - async fn get_sync_status_call(&self, request: GetSyncStatusRequest) -> RpcResult; + async fn get_sync_status_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetSyncStatusRequest, + ) -> RpcResult; // --- /// Requests the network the node is currently running against. async fn get_current_network(&self) -> RpcResult { - Ok(self.get_current_network_call(GetCurrentNetworkRequest {}).await?.network) + Ok(self.get_current_network_call(None, GetCurrentNetworkRequest {}).await?.network) } - async fn get_current_network_call(&self, request: GetCurrentNetworkRequest) -> RpcResult; + async fn get_current_network_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetCurrentNetworkRequest, + ) -> RpcResult; /// Submit a block into the DAG. /// /// Blocks are generally expected to have been generated using the get_block_template call. - async fn submit_block(&self, block: RpcBlock, allow_non_daa_blocks: bool) -> RpcResult { - self.submit_block_call(SubmitBlockRequest::new(block, allow_non_daa_blocks)).await + async fn submit_block(&self, block: RpcRawBlock, allow_non_daa_blocks: bool) -> RpcResult { + self.submit_block_call(None, SubmitBlockRequest::new(block, allow_non_daa_blocks)).await } - async fn submit_block_call(&self, request: SubmitBlockRequest) -> RpcResult; + async fn submit_block_call( + &self, + connection: Option<&DynRpcConnection>, + request: SubmitBlockRequest, + ) -> RpcResult; /// Request a current block template. /// /// Callers are expected to solve the block template and submit it using the submit_block call. async fn get_block_template(&self, pay_address: RpcAddress, extra_data: RpcExtraData) -> RpcResult { - self.get_block_template_call(GetBlockTemplateRequest::new(pay_address, extra_data)).await + self.get_block_template_call(None, GetBlockTemplateRequest::new(pay_address, extra_data)).await } - async fn get_block_template_call(&self, request: GetBlockTemplateRequest) -> RpcResult; + async fn get_block_template_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetBlockTemplateRequest, + ) -> RpcResult; /// Requests the list of known spectred addresses in the current network (mainnet, testnet, etc.) async fn get_peer_addresses(&self) -> RpcResult { - self.get_peer_addresses_call(GetPeerAddressesRequest {}).await + self.get_peer_addresses_call(None, GetPeerAddressesRequest {}).await } - async fn get_peer_addresses_call(&self, request: GetPeerAddressesRequest) -> RpcResult; + async fn get_peer_addresses_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetPeerAddressesRequest, + ) -> RpcResult; /// requests the hash of the current virtual's selected parent. async fn get_sink(&self) -> RpcResult { - self.get_sink_call(GetSinkRequest {}).await + self.get_sink_call(None, GetSinkRequest {}).await } - async fn get_sink_call(&self, request: GetSinkRequest) -> RpcResult; + async fn get_sink_call(&self, connection: Option<&DynRpcConnection>, request: GetSinkRequest) -> RpcResult; /// Requests information about a specific transaction in the mempool. async fn get_mempool_entry( @@ -96,64 +160,85 @@ pub trait RpcApi: Sync + Send + AnySync { filter_transaction_pool: bool, ) -> RpcResult { Ok(self - .get_mempool_entry_call(GetMempoolEntryRequest::new(transaction_id, include_orphan_pool, filter_transaction_pool)) + .get_mempool_entry_call(None, GetMempoolEntryRequest::new(transaction_id, include_orphan_pool, filter_transaction_pool)) .await? .mempool_entry) } - async fn get_mempool_entry_call(&self, request: GetMempoolEntryRequest) -> RpcResult; + async fn get_mempool_entry_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetMempoolEntryRequest, + ) -> RpcResult; /// Requests information about all the transactions currently in the mempool. async fn get_mempool_entries(&self, include_orphan_pool: bool, filter_transaction_pool: bool) -> RpcResult> { Ok(self - .get_mempool_entries_call(GetMempoolEntriesRequest::new(include_orphan_pool, filter_transaction_pool)) + .get_mempool_entries_call(None, GetMempoolEntriesRequest::new(include_orphan_pool, filter_transaction_pool)) .await? .mempool_entries) } - async fn get_mempool_entries_call(&self, request: GetMempoolEntriesRequest) -> RpcResult; + async fn get_mempool_entries_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetMempoolEntriesRequest, + ) -> RpcResult; /// requests information about all the p2p peers currently connected to this node. async fn get_connected_peer_info(&self) -> RpcResult { - self.get_connected_peer_info_call(GetConnectedPeerInfoRequest {}).await + self.get_connected_peer_info_call(None, GetConnectedPeerInfoRequest {}).await } - async fn get_connected_peer_info_call(&self, request: GetConnectedPeerInfoRequest) -> RpcResult; + async fn get_connected_peer_info_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetConnectedPeerInfoRequest, + ) -> RpcResult; /// Adds a peer to the node's outgoing connection list. /// /// This will, in most cases, result in the node connecting to said peer. async fn add_peer(&self, peer_address: RpcContextualPeerAddress, is_permanent: bool) -> RpcResult<()> { - self.add_peer_call(AddPeerRequest::new(peer_address, is_permanent)).await?; + self.add_peer_call(None, AddPeerRequest::new(peer_address, is_permanent)).await?; Ok(()) } - async fn add_peer_call(&self, request: AddPeerRequest) -> RpcResult; + async fn add_peer_call(&self, connection: Option<&DynRpcConnection>, request: AddPeerRequest) -> RpcResult; /// Submits a transaction to the mempool. async fn submit_transaction(&self, transaction: RpcTransaction, allow_orphan: bool) -> RpcResult { - Ok(self.submit_transaction_call(SubmitTransactionRequest { transaction, allow_orphan }).await?.transaction_id) + Ok(self.submit_transaction_call(None, SubmitTransactionRequest { transaction, allow_orphan }).await?.transaction_id) } - async fn submit_transaction_call(&self, request: SubmitTransactionRequest) -> RpcResult; + async fn submit_transaction_call( + &self, + connection: Option<&DynRpcConnection>, + request: SubmitTransactionRequest, + ) -> RpcResult; /// Submits a transaction replacement to the mempool, applying a mandatory Replace by Fee policy. /// /// Returns the ID of the inserted transaction and the transaction the submission replaced in the mempool. async fn submit_transaction_replacement(&self, transaction: RpcTransaction) -> RpcResult { - self.submit_transaction_replacement_call(SubmitTransactionReplacementRequest { transaction }).await + self.submit_transaction_replacement_call(None, SubmitTransactionReplacementRequest { transaction }).await } async fn submit_transaction_replacement_call( &self, + connection: Option<&DynRpcConnection>, request: SubmitTransactionReplacementRequest, ) -> RpcResult; /// Requests information about a specific block. async fn get_block(&self, hash: RpcHash, include_transactions: bool) -> RpcResult { - Ok(self.get_block_call(GetBlockRequest::new(hash, include_transactions)).await?.block) + Ok(self.get_block_call(None, GetBlockRequest::new(hash, include_transactions)).await?.block) } - async fn get_block_call(&self, request: GetBlockRequest) -> RpcResult; + async fn get_block_call(&self, connection: Option<&DynRpcConnection>, request: GetBlockRequest) -> RpcResult; /// Requests information about a specific subnetwork. async fn get_subnetwork(&self, subnetwork_id: RpcSubnetworkId) -> RpcResult { - self.get_subnetwork_call(GetSubnetworkRequest::new(subnetwork_id)).await + self.get_subnetwork_call(None, GetSubnetworkRequest::new(subnetwork_id)).await } - async fn get_subnetwork_call(&self, request: GetSubnetworkRequest) -> RpcResult; + async fn get_subnetwork_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetSubnetworkRequest, + ) -> RpcResult; /// Requests the virtual selected parent chain from some `start_hash` to this node's current virtual. async fn get_virtual_chain_from_block( @@ -161,11 +246,15 @@ pub trait RpcApi: Sync + Send + AnySync { start_hash: RpcHash, include_accepted_transaction_ids: bool, ) -> RpcResult { - self.get_virtual_chain_from_block_call(GetVirtualChainFromBlockRequest::new(start_hash, include_accepted_transaction_ids)) - .await + self.get_virtual_chain_from_block_call( + None, + GetVirtualChainFromBlockRequest::new(start_hash, include_accepted_transaction_ids), + ) + .await } async fn get_virtual_chain_from_block_call( &self, + connection: Option<&DynRpcConnection>, request: GetVirtualChainFromBlockRequest, ) -> RpcResult; @@ -176,61 +265,79 @@ pub trait RpcApi: Sync + Send + AnySync { include_blocks: bool, include_transactions: bool, ) -> RpcResult { - self.get_blocks_call(GetBlocksRequest::new(low_hash, include_blocks, include_transactions)).await + self.get_blocks_call(None, GetBlocksRequest::new(low_hash, include_blocks, include_transactions)).await } - async fn get_blocks_call(&self, request: GetBlocksRequest) -> RpcResult; + async fn get_blocks_call(&self, connection: Option<&DynRpcConnection>, request: GetBlocksRequest) -> RpcResult; /// Requests the current number of blocks in this node. /// /// Note that this number may decrease as pruning occurs. async fn get_block_count(&self) -> RpcResult { - self.get_block_count_call(GetBlockCountRequest {}).await + self.get_block_count_call(None, GetBlockCountRequest {}).await } - async fn get_block_count_call(&self, request: GetBlockCountRequest) -> RpcResult; + async fn get_block_count_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetBlockCountRequest, + ) -> RpcResult; /// Requests general information about the current state of this node's DAG. async fn get_block_dag_info(&self) -> RpcResult { - self.get_block_dag_info_call(GetBlockDagInfoRequest {}).await + self.get_block_dag_info_call(None, GetBlockDagInfoRequest {}).await } - async fn get_block_dag_info_call(&self, request: GetBlockDagInfoRequest) -> RpcResult; + async fn get_block_dag_info_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetBlockDagInfoRequest, + ) -> RpcResult; /// async fn resolve_finality_conflict(&self, finality_block_hash: RpcHash) -> RpcResult<()> { - self.resolve_finality_conflict_call(ResolveFinalityConflictRequest::new(finality_block_hash)).await?; + self.resolve_finality_conflict_call(None, ResolveFinalityConflictRequest::new(finality_block_hash)).await?; Ok(()) } async fn resolve_finality_conflict_call( &self, + connection: Option<&DynRpcConnection>, request: ResolveFinalityConflictRequest, ) -> RpcResult; /// Shuts down this node. async fn shutdown(&self) -> RpcResult<()> { - self.shutdown_call(ShutdownRequest {}).await?; + self.shutdown_call(None, ShutdownRequest {}).await?; Ok(()) } - async fn shutdown_call(&self, request: ShutdownRequest) -> RpcResult; + async fn shutdown_call(&self, connection: Option<&DynRpcConnection>, request: ShutdownRequest) -> RpcResult; /// Requests headers between the given `start_hash` and the current virtual, up to the given limit. async fn get_headers(&self, start_hash: RpcHash, limit: u64, is_ascending: bool) -> RpcResult> { - Ok(self.get_headers_call(GetHeadersRequest::new(start_hash, limit, is_ascending)).await?.headers) + Ok(self.get_headers_call(None, GetHeadersRequest::new(start_hash, limit, is_ascending)).await?.headers) } - async fn get_headers_call(&self, request: GetHeadersRequest) -> RpcResult; + async fn get_headers_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetHeadersRequest, + ) -> RpcResult; /// Returns the total balance in unspent transactions towards a given address. /// /// This call is only available when this node was started with `--utxoindex`. async fn get_balance_by_address(&self, address: RpcAddress) -> RpcResult { - Ok(self.get_balance_by_address_call(GetBalanceByAddressRequest::new(address)).await?.balance) + Ok(self.get_balance_by_address_call(None, GetBalanceByAddressRequest::new(address)).await?.balance) } - async fn get_balance_by_address_call(&self, request: GetBalanceByAddressRequest) -> RpcResult; + async fn get_balance_by_address_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetBalanceByAddressRequest, + ) -> RpcResult; /// async fn get_balances_by_addresses(&self, addresses: Vec) -> RpcResult> { - Ok(self.get_balances_by_addresses_call(GetBalancesByAddressesRequest::new(addresses)).await?.entries) + Ok(self.get_balances_by_addresses_call(None, GetBalancesByAddressesRequest::new(addresses)).await?.entries) } async fn get_balances_by_addresses_call( &self, + connection: Option<&DynRpcConnection>, request: GetBalancesByAddressesRequest, ) -> RpcResult; @@ -238,45 +345,54 @@ pub trait RpcApi: Sync + Send + AnySync { /// /// This call is only available when this node was started with `--utxoindex`. async fn get_utxos_by_addresses(&self, addresses: Vec) -> RpcResult> { - Ok(self.get_utxos_by_addresses_call(GetUtxosByAddressesRequest::new(addresses)).await?.entries) + Ok(self.get_utxos_by_addresses_call(None, GetUtxosByAddressesRequest::new(addresses)).await?.entries) } - async fn get_utxos_by_addresses_call(&self, request: GetUtxosByAddressesRequest) -> RpcResult; + async fn get_utxos_by_addresses_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetUtxosByAddressesRequest, + ) -> RpcResult; /// Requests the blue score of the current selected parent of the virtual block. async fn get_sink_blue_score(&self) -> RpcResult { - Ok(self.get_sink_blue_score_call(GetSinkBlueScoreRequest {}).await?.blue_score) + Ok(self.get_sink_blue_score_call(None, GetSinkBlueScoreRequest {}).await?.blue_score) } - async fn get_sink_blue_score_call(&self, request: GetSinkBlueScoreRequest) -> RpcResult; + async fn get_sink_blue_score_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetSinkBlueScoreRequest, + ) -> RpcResult; /// Bans the given ip. async fn ban(&self, ip: RpcIpAddress) -> RpcResult<()> { - self.ban_call(BanRequest::new(ip)).await?; + self.ban_call(None, BanRequest::new(ip)).await?; Ok(()) } - async fn ban_call(&self, request: BanRequest) -> RpcResult; + async fn ban_call(&self, connection: Option<&DynRpcConnection>, request: BanRequest) -> RpcResult; /// Unbans the given ip. async fn unban(&self, ip: RpcIpAddress) -> RpcResult<()> { - self.unban_call(UnbanRequest::new(ip)).await?; + self.unban_call(None, UnbanRequest::new(ip)).await?; Ok(()) } - async fn unban_call(&self, request: UnbanRequest) -> RpcResult; + async fn unban_call(&self, connection: Option<&DynRpcConnection>, request: UnbanRequest) -> RpcResult; /// Returns info about the node. - async fn get_info_call(&self, request: GetInfoRequest) -> RpcResult; async fn get_info(&self) -> RpcResult { - self.get_info_call(GetInfoRequest {}).await + self.get_info_call(None, GetInfoRequest {}).await } + async fn get_info_call(&self, connection: Option<&DynRpcConnection>, request: GetInfoRequest) -> RpcResult; /// async fn estimate_network_hashes_per_second(&self, window_size: u32, start_hash: Option) -> RpcResult { Ok(self - .estimate_network_hashes_per_second_call(EstimateNetworkHashesPerSecondRequest::new(window_size, start_hash)) + .estimate_network_hashes_per_second_call(None, EstimateNetworkHashesPerSecondRequest::new(window_size, start_hash)) .await? .network_hashes_per_second) } async fn estimate_network_hashes_per_second_call( &self, + connection: Option<&DynRpcConnection>, request: EstimateNetworkHashesPerSecondRequest, ) -> RpcResult; @@ -288,30 +404,35 @@ pub trait RpcApi: Sync + Send + AnySync { filter_transaction_pool: bool, ) -> RpcResult> { Ok(self - .get_mempool_entries_by_addresses_call(GetMempoolEntriesByAddressesRequest::new( - addresses, - include_orphan_pool, - filter_transaction_pool, - )) + .get_mempool_entries_by_addresses_call( + None, + GetMempoolEntriesByAddressesRequest::new(addresses, include_orphan_pool, filter_transaction_pool), + ) .await? .entries) } async fn get_mempool_entries_by_addresses_call( &self, + connection: Option<&DynRpcConnection>, request: GetMempoolEntriesByAddressesRequest, ) -> RpcResult; /// async fn get_coin_supply(&self) -> RpcResult { - self.get_coin_supply_call(GetCoinSupplyRequest {}).await + self.get_coin_supply_call(None, GetCoinSupplyRequest {}).await } - async fn get_coin_supply_call(&self, request: GetCoinSupplyRequest) -> RpcResult; + async fn get_coin_supply_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetCoinSupplyRequest, + ) -> RpcResult; async fn get_daa_score_timestamp_estimate(&self, daa_scores: Vec) -> RpcResult> { - Ok(self.get_daa_score_timestamp_estimate_call(GetDaaScoreTimestampEstimateRequest { daa_scores }).await?.timestamps) + Ok(self.get_daa_score_timestamp_estimate_call(None, GetDaaScoreTimestampEstimateRequest { daa_scores }).await?.timestamps) } async fn get_daa_score_timestamp_estimate_call( &self, + connection: Option<&DynRpcConnection>, request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult; @@ -319,15 +440,20 @@ pub trait RpcApi: Sync + Send + AnySync { // Fee estimation API async fn get_fee_estimate(&self) -> RpcResult { - Ok(self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate) + Ok(self.get_fee_estimate_call(None, GetFeeEstimateRequest {}).await?.estimate) } - async fn get_fee_estimate_call(&self, request: GetFeeEstimateRequest) -> RpcResult; + async fn get_fee_estimate_call( + &self, + connection: Option<&DynRpcConnection>, + request: GetFeeEstimateRequest, + ) -> RpcResult; async fn get_fee_estimate_experimental(&self, verbose: bool) -> RpcResult { - self.get_fee_estimate_experimental_call(GetFeeEstimateExperimentalRequest { verbose }).await + self.get_fee_estimate_experimental_call(None, GetFeeEstimateExperimentalRequest { verbose }).await } async fn get_fee_estimate_experimental_call( &self, + connection: Option<&DynRpcConnection>, request: GetFeeEstimateExperimentalRequest, ) -> RpcResult; diff --git a/rpc/core/src/convert/block.rs b/rpc/core/src/convert/block.rs index 2b637fc..56e6ca1 100644 --- a/rpc/core/src/convert/block.rs +++ b/rpc/core/src/convert/block.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use crate::{RpcBlock, RpcError, RpcResult, RpcTransaction}; +use crate::{RpcBlock, RpcError, RpcRawBlock, RpcResult, RpcTransaction}; use spectre_consensus_core::block::{Block, MutableBlock}; // ---------------------------------------------------------------------------- @@ -10,7 +10,7 @@ use spectre_consensus_core::block::{Block, MutableBlock}; impl From<&Block> for RpcBlock { fn from(item: &Block) -> Self { Self { - header: (*item.header).clone(), + header: item.header.as_ref().into(), transactions: item.transactions.iter().map(RpcTransaction::from).collect(), // TODO: Implement a populating process inspired from spectred\app\rpc\rpccontext\verbosedata.go verbose_data: None, @@ -18,28 +18,61 @@ impl From<&Block> for RpcBlock { } } +impl From<&Block> for RpcRawBlock { + fn from(item: &Block) -> Self { + Self { header: item.header.as_ref().into(), transactions: item.transactions.iter().map(RpcTransaction::from).collect() } + } +} + impl From<&MutableBlock> for RpcBlock { fn from(item: &MutableBlock) -> Self { Self { - header: item.header.clone(), + header: item.header.as_ref().into(), transactions: item.transactions.iter().map(RpcTransaction::from).collect(), verbose_data: None, } } } +impl From<&MutableBlock> for RpcRawBlock { + fn from(item: &MutableBlock) -> Self { + Self { header: item.header.as_ref().into(), transactions: item.transactions.iter().map(RpcTransaction::from).collect() } + } +} + +impl From for RpcRawBlock { + fn from(item: MutableBlock) -> Self { + Self { header: item.header.into(), transactions: item.transactions.iter().map(RpcTransaction::from).collect() } + } +} + // ---------------------------------------------------------------------------- // rpc_core to consensus_core // ---------------------------------------------------------------------------- -impl TryFrom<&RpcBlock> for Block { +impl TryFrom for Block { + type Error = RpcError; + fn try_from(item: RpcBlock) -> RpcResult { + Ok(Self { + header: Arc::new(item.header.into()), + transactions: Arc::new( + item.transactions + .into_iter() + .map(spectre_consensus_core::tx::Transaction::try_from) + .collect::>>()?, + ), + }) + } +} + +impl TryFrom for Block { type Error = RpcError; - fn try_from(item: &RpcBlock) -> RpcResult { + fn try_from(item: RpcRawBlock) -> RpcResult { Ok(Self { - header: Arc::new(item.header.clone()), + header: Arc::new(item.header.into()), transactions: Arc::new( item.transactions - .iter() + .into_iter() .map(spectre_consensus_core::tx::Transaction::try_from) .collect::>>()?, ), diff --git a/rpc/core/src/convert/tx.rs b/rpc/core/src/convert/tx.rs index 99b116d..77391fa 100644 --- a/rpc/core/src/convert/tx.rs +++ b/rpc/core/src/convert/tx.rs @@ -36,7 +36,7 @@ impl From<&TransactionOutput> for RpcTransactionOutput { impl From<&TransactionInput> for RpcTransactionInput { fn from(item: &TransactionInput) -> Self { Self { - previous_outpoint: item.previous_outpoint, + previous_outpoint: item.previous_outpoint.into(), signature_script: item.signature_script.clone(), sequence: item.sequence, sig_op_count: item.sig_op_count, @@ -50,17 +50,17 @@ impl From<&TransactionInput> for RpcTransactionInput { // rpc_core to consensus_core // ---------------------------------------------------------------------------- -impl TryFrom<&RpcTransaction> for Transaction { +impl TryFrom for Transaction { type Error = RpcError; - fn try_from(item: &RpcTransaction) -> RpcResult { + fn try_from(item: RpcTransaction) -> RpcResult { let transaction = Transaction::new( item.version, item.inputs - .iter() + .into_iter() .map(spectre_consensus_core::tx::TransactionInput::try_from) .collect::>>()?, item.outputs - .iter() + .into_iter() .map(spectre_consensus_core::tx::TransactionOutput::try_from) .collect::>>()?, item.lock_time, @@ -73,16 +73,16 @@ impl TryFrom<&RpcTransaction> for Transaction { } } -impl TryFrom<&RpcTransactionOutput> for TransactionOutput { +impl TryFrom for TransactionOutput { type Error = RpcError; - fn try_from(item: &RpcTransactionOutput) -> RpcResult { - Ok(Self::new(item.value, item.script_public_key.clone())) + fn try_from(item: RpcTransactionOutput) -> RpcResult { + Ok(Self::new(item.value, item.script_public_key)) } } -impl TryFrom<&RpcTransactionInput> for TransactionInput { +impl TryFrom for TransactionInput { type Error = RpcError; - fn try_from(item: &RpcTransactionInput) -> RpcResult { - Ok(Self::new(item.previous_outpoint, item.signature_script.clone(), item.sequence, item.sig_op_count)) + fn try_from(item: RpcTransactionInput) -> RpcResult { + Ok(Self::new(item.previous_outpoint.into(), item.signature_script, item.sequence, item.sig_op_count)) } } diff --git a/rpc/core/src/convert/utxo.rs b/rpc/core/src/convert/utxo.rs index 1c85369..c2e8518 100644 --- a/rpc/core/src/convert/utxo.rs +++ b/rpc/core/src/convert/utxo.rs @@ -1,6 +1,6 @@ +use crate::RpcUtxoEntry; use crate::RpcUtxosByAddressesEntry; use spectre_addresses::Prefix; -use spectre_consensus_core::tx::UtxoEntry; use spectre_index_core::indexed_utxos::UtxoSetByScriptPublicKey; use spectre_txscript::extract_script_pub_key_address; @@ -16,8 +16,8 @@ pub fn utxo_set_into_rpc(item: &UtxoSetByScriptPublicKey, prefix: Option .iter() .map(|(outpoint, entry)| RpcUtxosByAddressesEntry { address: address.clone(), - outpoint: *outpoint, - utxo_entry: UtxoEntry::new(entry.amount, script_public_key.clone(), entry.block_daa_score, entry.is_coinbase), + outpoint: (*outpoint).into(), + utxo_entry: RpcUtxoEntry::new(entry.amount, script_public_key.clone(), entry.block_daa_score, entry.is_coinbase), }) .collect::>() }) diff --git a/rpc/core/src/model/address.rs b/rpc/core/src/model/address.rs index c7b4d65..4526cd1 100644 --- a/rpc/core/src/model/address.rs +++ b/rpc/core/src/model/address.rs @@ -1,11 +1,11 @@ use crate::{RpcTransactionOutpoint, RpcUtxoEntry}; -use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use workflow_serializer::prelude::*; pub type RpcAddress = spectre_addresses::Address; /// Represents a UTXO entry of an address returned by the `GetUtxosByAddresses` RPC. -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcUtxosByAddressesEntry { pub address: Option, @@ -13,8 +13,27 @@ pub struct RpcUtxosByAddressesEntry { pub utxo_entry: RpcUtxoEntry, } +impl Serializer for RpcUtxosByAddressesEntry { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; // version + store!(Option, &self.address, writer)?; + serialize!(RpcTransactionOutpoint, &self.outpoint, writer)?; + serialize!(RpcUtxoEntry, &self.utxo_entry, writer) + } +} + +impl Deserializer for RpcUtxosByAddressesEntry { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version: u8 = load!(u8, reader)?; + let address = load!(Option, reader)?; + let outpoint = deserialize!(RpcTransactionOutpoint, reader)?; + let utxo_entry = deserialize!(RpcUtxoEntry, reader)?; + Ok(Self { address, outpoint, utxo_entry }) + } +} + /// Represents a balance of an address returned by the `GetBalancesByAddresses` RPC. -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcBalancesByAddressesEntry { pub address: RpcAddress, @@ -22,3 +41,20 @@ pub struct RpcBalancesByAddressesEntry { /// Balance of `address` if available pub balance: Option, } + +impl Serializer for RpcBalancesByAddressesEntry { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; // version + store!(RpcAddress, &self.address, writer)?; + store!(Option, &self.balance, writer) + } +} + +impl Deserializer for RpcBalancesByAddressesEntry { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version: u8 = load!(u8, reader)?; + let address = load!(RpcAddress, reader)?; + let balance = load!(Option, reader)?; + Ok(Self { address, balance }) + } +} diff --git a/rpc/core/src/model/block.rs b/rpc/core/src/model/block.rs index c4c501a..96dbcd3 100644 --- a/rpc/core/src/model/block.rs +++ b/rpc/core/src/model/block.rs @@ -1,8 +1,18 @@ +use super::RpcRawHeader; use crate::prelude::{RpcHash, RpcHeader, RpcTransaction}; -use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use workflow_serializer::prelude::*; -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +/// Raw Rpc block type - without a cached header hash and without verbose data. +/// Used for mining APIs (get_block_template & submit_block) +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcRawBlock { + pub header: RpcRawHeader, + pub transactions: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcBlock { pub header: RpcHeader, @@ -10,7 +20,49 @@ pub struct RpcBlock { pub verbose_data: Option, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for RpcBlock { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcHeader, &self.header, writer)?; + serialize!(Vec, &self.transactions, writer)?; + serialize!(Option, &self.verbose_data, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcBlock { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let header = deserialize!(RpcHeader, reader)?; + let transactions = deserialize!(Vec, reader)?; + let verbose_data = deserialize!(Option, reader)?; + + Ok(Self { header, transactions, verbose_data }) + } +} + +impl Serializer for RpcRawBlock { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcRawHeader, &self.header, writer)?; + serialize!(Vec, &self.transactions, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcRawBlock { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let header = deserialize!(RpcRawHeader, reader)?; + let transactions = deserialize!(Vec, reader)?; + + Ok(Self { header, transactions }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcBlockVerboseData { pub hash: RpcHash, @@ -25,6 +77,53 @@ pub struct RpcBlockVerboseData { pub is_chain_block: bool, } +impl Serializer for RpcBlockVerboseData { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(RpcHash, &self.hash, writer)?; + store!(f64, &self.difficulty, writer)?; + store!(RpcHash, &self.selected_parent_hash, writer)?; + store!(Vec, &self.transaction_ids, writer)?; + store!(bool, &self.is_header_only, writer)?; + store!(u64, &self.blue_score, writer)?; + store!(Vec, &self.children_hashes, writer)?; + store!(Vec, &self.merge_set_blues_hashes, writer)?; + store!(Vec, &self.merge_set_reds_hashes, writer)?; + store!(bool, &self.is_chain_block, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcBlockVerboseData { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let hash = load!(RpcHash, reader)?; + let difficulty = load!(f64, reader)?; + let selected_parent_hash = load!(RpcHash, reader)?; + let transaction_ids = load!(Vec, reader)?; + let is_header_only = load!(bool, reader)?; + let blue_score = load!(u64, reader)?; + let children_hashes = load!(Vec, reader)?; + let merge_set_blues_hashes = load!(Vec, reader)?; + let merge_set_reds_hashes = load!(Vec, reader)?; + let is_chain_block = load!(bool, reader)?; + + Ok(Self { + hash, + difficulty, + selected_parent_hash, + transaction_ids, + is_header_only, + blue_score, + children_hashes, + merge_set_blues_hashes, + merge_set_reds_hashes, + is_chain_block, + }) + } +} + cfg_if::cfg_if! { if #[cfg(feature = "wasm32-sdk")] { use wasm_bindgen::prelude::*; diff --git a/rpc/core/src/model/feerate_estimate.rs b/rpc/core/src/model/feerate_estimate.rs index f9de9de..cf97a4b 100644 --- a/rpc/core/src/model/feerate_estimate.rs +++ b/rpc/core/src/model/feerate_estimate.rs @@ -1,5 +1,6 @@ use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use workflow_serializer::prelude::*; #[derive(Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] @@ -11,7 +12,7 @@ pub struct RpcFeerateBucket { pub estimated_seconds: f64, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcFeeEstimate { /// *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. @@ -42,7 +43,27 @@ impl RpcFeeEstimate { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for RpcFeeEstimate { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcFeerateBucket, &self.priority_bucket, writer)?; + store!(Vec, &self.normal_buckets, writer)?; + store!(Vec, &self.low_buckets, writer)?; + Ok(()) + } +} + +impl Deserializer for RpcFeeEstimate { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let priority_bucket = load!(RpcFeerateBucket, reader)?; + let normal_buckets = load!(Vec, reader)?; + let low_buckets = load!(Vec, reader)?; + Ok(Self { priority_bucket, normal_buckets, low_buckets }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcFeeEstimateVerboseExperimentalData { pub mempool_ready_transactions_count: u64, @@ -53,3 +74,36 @@ pub struct RpcFeeEstimateVerboseExperimentalData { pub next_block_template_feerate_median: f64, pub next_block_template_feerate_max: f64, } + +impl Serializer for RpcFeeEstimateVerboseExperimentalData { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.mempool_ready_transactions_count, writer)?; + store!(u64, &self.mempool_ready_transactions_total_mass, writer)?; + store!(u64, &self.network_mass_per_second, writer)?; + store!(f64, &self.next_block_template_feerate_min, writer)?; + store!(f64, &self.next_block_template_feerate_median, writer)?; + store!(f64, &self.next_block_template_feerate_max, writer)?; + Ok(()) + } +} + +impl Deserializer for RpcFeeEstimateVerboseExperimentalData { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let mempool_ready_transactions_count = load!(u64, reader)?; + let mempool_ready_transactions_total_mass = load!(u64, reader)?; + let network_mass_per_second = load!(u64, reader)?; + let next_block_template_feerate_min = load!(f64, reader)?; + let next_block_template_feerate_median = load!(f64, reader)?; + let next_block_template_feerate_max = load!(f64, reader)?; + Ok(Self { + mempool_ready_transactions_count, + mempool_ready_transactions_total_mass, + network_mass_per_second, + next_block_template_feerate_min, + next_block_template_feerate_median, + next_block_template_feerate_max, + }) + } +} diff --git a/rpc/core/src/model/header.rs b/rpc/core/src/model/header.rs index 7193cab..d6f8463 100644 --- a/rpc/core/src/model/header.rs +++ b/rpc/core/src/model/header.rs @@ -1 +1,331 @@ -pub type RpcHeader = spectre_consensus_core::header::Header; +use borsh::{BorshDeserialize, BorshSerialize}; +use serde::{Deserialize, Serialize}; +use spectre_consensus_core::{header::Header, BlueWorkType}; +use spectre_hashes::Hash; +use workflow_serializer::prelude::*; + +/// Raw Rpc header type - without a cached header hash. +/// Used for mining APIs (get_block_template & submit_block) +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] + +pub struct RpcRawHeader { + pub version: u16, + pub parents_by_level: Vec>, + pub hash_merkle_root: Hash, + pub accepted_id_merkle_root: Hash, + pub utxo_commitment: Hash, + /// Timestamp is in milliseconds + pub timestamp: u64, + pub bits: u32, + pub nonce: u64, + pub daa_score: u64, + pub blue_work: BlueWorkType, + pub blue_score: u64, + pub pruning_point: Hash, +} + +#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcHeader { + /// Cached hash + pub hash: Hash, + pub version: u16, + pub parents_by_level: Vec>, + pub hash_merkle_root: Hash, + pub accepted_id_merkle_root: Hash, + pub utxo_commitment: Hash, + /// Timestamp is in milliseconds + pub timestamp: u64, + pub bits: u32, + pub nonce: u64, + pub daa_score: u64, + pub blue_work: BlueWorkType, + pub blue_score: u64, + pub pruning_point: Hash, +} + +impl RpcHeader { + pub fn direct_parents(&self) -> &[Hash] { + if self.parents_by_level.is_empty() { + &[] + } else { + &self.parents_by_level[0] + } + } +} + +impl AsRef for RpcHeader { + fn as_ref(&self) -> &RpcHeader { + self + } +} + +impl From
for RpcHeader { + fn from(header: Header) -> Self { + Self { + hash: header.hash, + version: header.version, + parents_by_level: header.parents_by_level, + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl From<&Header> for RpcHeader { + fn from(header: &Header) -> Self { + Self { + hash: header.hash, + version: header.version, + parents_by_level: header.parents_by_level.clone(), + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl From for Header { + fn from(header: RpcHeader) -> Self { + Self { + hash: header.hash, + version: header.version, + parents_by_level: header.parents_by_level, + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl From<&RpcHeader> for Header { + fn from(header: &RpcHeader) -> Self { + Self { + hash: header.hash, + version: header.version, + parents_by_level: header.parents_by_level.clone(), + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl Serializer for RpcHeader { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + + store!(Hash, &self.hash, writer)?; + store!(u16, &self.version, writer)?; + store!(Vec>, &self.parents_by_level, writer)?; + store!(Hash, &self.hash_merkle_root, writer)?; + store!(Hash, &self.accepted_id_merkle_root, writer)?; + store!(Hash, &self.utxo_commitment, writer)?; + store!(u64, &self.timestamp, writer)?; + store!(u32, &self.bits, writer)?; + store!(u64, &self.nonce, writer)?; + store!(u64, &self.daa_score, writer)?; + store!(BlueWorkType, &self.blue_work, writer)?; + store!(u64, &self.blue_score, writer)?; + store!(Hash, &self.pruning_point, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcHeader { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + + let hash = load!(Hash, reader)?; + let version = load!(u16, reader)?; + let parents_by_level = load!(Vec>, reader)?; + let hash_merkle_root = load!(Hash, reader)?; + let accepted_id_merkle_root = load!(Hash, reader)?; + let utxo_commitment = load!(Hash, reader)?; + let timestamp = load!(u64, reader)?; + let bits = load!(u32, reader)?; + let nonce = load!(u64, reader)?; + let daa_score = load!(u64, reader)?; + let blue_work = load!(BlueWorkType, reader)?; + let blue_score = load!(u64, reader)?; + let pruning_point = load!(Hash, reader)?; + + Ok(Self { + hash, + version, + parents_by_level, + hash_merkle_root, + accepted_id_merkle_root, + utxo_commitment, + timestamp, + bits, + nonce, + daa_score, + blue_work, + blue_score, + pruning_point, + }) + } +} + +impl From for Header { + fn from(header: RpcRawHeader) -> Self { + Self::new_finalized( + header.version, + header.parents_by_level, + header.hash_merkle_root, + header.accepted_id_merkle_root, + header.utxo_commitment, + header.timestamp, + header.bits, + header.nonce, + header.daa_score, + header.blue_work, + header.blue_score, + header.pruning_point, + ) + } +} + +impl From<&RpcRawHeader> for Header { + fn from(header: &RpcRawHeader) -> Self { + Self::new_finalized( + header.version, + header.parents_by_level.clone(), + header.hash_merkle_root, + header.accepted_id_merkle_root, + header.utxo_commitment, + header.timestamp, + header.bits, + header.nonce, + header.daa_score, + header.blue_work, + header.blue_score, + header.pruning_point, + ) + } +} + +impl From<&Header> for RpcRawHeader { + fn from(header: &Header) -> Self { + Self { + version: header.version, + parents_by_level: header.parents_by_level.clone(), + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl From
for RpcRawHeader { + fn from(header: Header) -> Self { + Self { + version: header.version, + parents_by_level: header.parents_by_level, + hash_merkle_root: header.hash_merkle_root, + accepted_id_merkle_root: header.accepted_id_merkle_root, + utxo_commitment: header.utxo_commitment, + timestamp: header.timestamp, + bits: header.bits, + nonce: header.nonce, + daa_score: header.daa_score, + blue_work: header.blue_work, + blue_score: header.blue_score, + pruning_point: header.pruning_point, + } + } +} + +impl Serializer for RpcRawHeader { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + + store!(u16, &self.version, writer)?; + store!(Vec>, &self.parents_by_level, writer)?; + store!(Hash, &self.hash_merkle_root, writer)?; + store!(Hash, &self.accepted_id_merkle_root, writer)?; + store!(Hash, &self.utxo_commitment, writer)?; + store!(u64, &self.timestamp, writer)?; + store!(u32, &self.bits, writer)?; + store!(u64, &self.nonce, writer)?; + store!(u64, &self.daa_score, writer)?; + store!(BlueWorkType, &self.blue_work, writer)?; + store!(u64, &self.blue_score, writer)?; + store!(Hash, &self.pruning_point, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcRawHeader { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + + let version = load!(u16, reader)?; + let parents_by_level = load!(Vec>, reader)?; + let hash_merkle_root = load!(Hash, reader)?; + let accepted_id_merkle_root = load!(Hash, reader)?; + let utxo_commitment = load!(Hash, reader)?; + let timestamp = load!(u64, reader)?; + let bits = load!(u32, reader)?; + let nonce = load!(u64, reader)?; + let daa_score = load!(u64, reader)?; + let blue_work = load!(BlueWorkType, reader)?; + let blue_score = load!(u64, reader)?; + let pruning_point = load!(Hash, reader)?; + + Ok(Self { + version, + parents_by_level, + hash_merkle_root, + accepted_id_merkle_root, + utxo_commitment, + timestamp, + bits, + nonce, + daa_score, + blue_work, + blue_score, + pruning_point, + }) + } +} diff --git a/rpc/core/src/model/mempool.rs b/rpc/core/src/model/mempool.rs index bd08b74..1a04bed 100644 --- a/rpc/core/src/model/mempool.rs +++ b/rpc/core/src/model/mempool.rs @@ -1,9 +1,9 @@ use super::RpcAddress; use super::RpcTransaction; -use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; +use workflow_serializer::prelude::*; -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct RpcMempoolEntry { pub fee: u64, pub transaction: RpcTransaction, @@ -16,7 +16,24 @@ impl RpcMempoolEntry { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for RpcMempoolEntry { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u64, &self.fee, writer)?; + serialize!(RpcTransaction, &self.transaction, writer)?; + store!(bool, &self.is_orphan, writer) + } +} + +impl Deserializer for RpcMempoolEntry { + fn deserialize(reader: &mut R) -> std::io::Result { + let fee = load!(u64, reader)?; + let transaction = deserialize!(RpcTransaction, reader)?; + let is_orphan = load!(bool, reader)?; + Ok(Self { fee, transaction, is_orphan }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct RpcMempoolEntryByAddress { pub address: RpcAddress, pub sending: Vec, @@ -29,6 +46,23 @@ impl RpcMempoolEntryByAddress { } } +impl Serializer for RpcMempoolEntryByAddress { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(RpcAddress, &self.address, writer)?; + serialize!(Vec, &self.sending, writer)?; + serialize!(Vec, &self.receiving, writer) + } +} + +impl Deserializer for RpcMempoolEntryByAddress { + fn deserialize(reader: &mut R) -> std::io::Result { + let address = load!(RpcAddress, reader)?; + let sending = deserialize!(Vec, reader)?; + let receiving = deserialize!(Vec, reader)?; + Ok(Self { address, sending, receiving }) + } +} + cfg_if::cfg_if! { if #[cfg(feature = "wasm32-sdk")] { use wasm_bindgen::prelude::*; diff --git a/rpc/core/src/model/message.rs b/rpc/core/src/model/message.rs index 0adef55..de7c513 100644 --- a/rpc/core/src/model/message.rs +++ b/rpc/core/src/model/message.rs @@ -4,10 +4,13 @@ use serde::{Deserialize, Serialize}; use spectre_consensus_core::api::stats::BlockCount; use spectre_core::debug; use spectre_notify::subscription::{context::SubscriptionContext, single::UtxosChangedSubscription, Command}; +use spectre_utils::hex::ToHex; +use std::collections::HashMap; use std::{ fmt::{Display, Formatter}, sync::Arc, }; +use workflow_serializer::prelude::*; pub type RpcExtraData = Vec; @@ -15,21 +18,42 @@ pub type RpcExtraData = Vec; /// Blocks are generally expected to have been generated using the getBlockTemplate call. /// /// See: [`GetBlockTemplateRequest`] -#[derive(Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitBlockRequest { - pub block: RpcBlock, + pub block: RpcRawBlock, #[serde(alias = "allowNonDAABlocks")] pub allow_non_daa_blocks: bool, } impl SubmitBlockRequest { - pub fn new(block: RpcBlock, allow_non_daa_blocks: bool) -> Self { + pub fn new(block: RpcRawBlock, allow_non_daa_blocks: bool) -> Self { Self { block, allow_non_daa_blocks } } } +impl Serializer for SubmitBlockRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcRawBlock, &self.block, writer)?; + store!(bool, &self.allow_non_daa_blocks, writer)?; + + Ok(()) + } +} + +impl Deserializer for SubmitBlockRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let block = deserialize!(RpcRawBlock, reader)?; + let allow_non_daa_blocks = load!(bool, reader)?; + + Ok(Self { block, allow_non_daa_blocks }) + } +} + #[derive(Clone, Copy, Eq, PartialEq, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] +#[borsh(use_discriminant = true)] pub enum SubmitBlockRejectReason { BlockInvalid = 1, IsInIBD = 2, @@ -54,6 +78,7 @@ impl Display for SubmitBlockRejectReason { #[derive(Eq, PartialEq, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "lowercase")] #[serde(tag = "type", content = "reason")] +#[borsh(use_discriminant = true)] pub enum SubmitBlockReport { Success, Reject(SubmitBlockRejectReason), @@ -64,17 +89,34 @@ impl SubmitBlockReport { } } -#[derive(Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitBlockResponse { pub report: SubmitBlockReport, } +impl Serializer for SubmitBlockResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(SubmitBlockReport, &self.report, writer)?; + Ok(()) + } +} + +impl Deserializer for SubmitBlockResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let report = load!(SubmitBlockReport, reader)?; + + Ok(Self { report }) + } +} + /// GetBlockTemplateRequest requests a current block template. /// Callers are expected to solve the block template and submit it using the submitBlock call /// /// See: [`SubmitBlockRequest`] -#[derive(Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockTemplateRequest { /// Which spectre address should the coinbase block reward transaction pay into @@ -88,10 +130,30 @@ impl GetBlockTemplateRequest { } } -#[derive(Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBlockTemplateRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcAddress, &self.pay_address, writer)?; + store!(RpcExtraData, &self.extra_data, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlockTemplateRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let pay_address = load!(RpcAddress, reader)?; + let extra_data = load!(RpcExtraData, reader)?; + + Ok(Self { pay_address, extra_data }) + } +} + +#[derive(Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockTemplateResponse { - pub block: RpcBlock, + pub block: RpcRawBlock, /// Whether spectred thinks that it's synced. /// Callers are discouraged (but not forbidden) from solving blocks when spectred is not synced. @@ -100,8 +162,28 @@ pub struct GetBlockTemplateResponse { pub is_synced: bool, } +impl Serializer for GetBlockTemplateResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcRawBlock, &self.block, writer)?; + store!(bool, &self.is_synced, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlockTemplateResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let block = deserialize!(RpcRawBlock, reader)?; + let is_synced = load!(bool, reader)?; + + Ok(Self { block, is_synced }) + } +} + /// GetBlockRequest requests information about a specific block -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockRequest { /// The hash of the requested block @@ -116,18 +198,70 @@ impl GetBlockRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBlockRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.hash, writer)?; + store!(bool, &self.include_transactions, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlockRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let hash = load!(RpcHash, reader)?; + let include_transactions = load!(bool, reader)?; + + Ok(Self { hash, include_transactions }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockResponse { pub block: RpcBlock, } +impl Serializer for GetBlockResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcBlock, &self.block, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlockResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let block = deserialize!(RpcBlock, reader)?; + + Ok(Self { block }) + } +} + /// GetInfoRequest returns info about the node. -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetInfoRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetInfoRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetInfoRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetInfoResponse { pub p2p_id: String, @@ -139,11 +273,55 @@ pub struct GetInfoResponse { pub has_message_id: bool, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetInfoResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(String, &self.p2p_id, writer)?; + store!(u64, &self.mempool_size, writer)?; + store!(String, &self.server_version, writer)?; + store!(bool, &self.is_utxo_indexed, writer)?; + store!(bool, &self.is_synced, writer)?; + store!(bool, &self.has_notify_command, writer)?; + store!(bool, &self.has_message_id, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetInfoResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let p2p_id = load!(String, reader)?; + let mempool_size = load!(u64, reader)?; + let server_version = load!(String, reader)?; + let is_utxo_indexed = load!(bool, reader)?; + let is_synced = load!(bool, reader)?; + let has_notify_command = load!(bool, reader)?; + let has_message_id = load!(bool, reader)?; + + Ok(Self { p2p_id, mempool_size, server_version, is_utxo_indexed, is_synced, has_notify_command, has_message_id }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetCurrentNetworkRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetCurrentNetworkRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetCurrentNetworkRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetCurrentNetworkResponse { pub network: RpcNetworkType, @@ -155,11 +333,41 @@ impl GetCurrentNetworkResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetCurrentNetworkResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcNetworkType, &self.network, writer)?; + Ok(()) + } +} + +impl Deserializer for GetCurrentNetworkResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let network = load!(RpcNetworkType, reader)?; + Ok(Self { network }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetPeerAddressesRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetPeerAddressesRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetPeerAddressesRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetPeerAddressesResponse { pub known_addresses: Vec, @@ -172,11 +380,43 @@ impl GetPeerAddressesResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetPeerAddressesResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.known_addresses, writer)?; + store!(Vec, &self.banned_addresses, writer)?; + Ok(()) + } +} + +impl Deserializer for GetPeerAddressesResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let known_addresses = load!(Vec, reader)?; + let banned_addresses = load!(Vec, reader)?; + Ok(Self { known_addresses, banned_addresses }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSinkRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSinkRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetSinkRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSinkResponse { pub sink: RpcHash, @@ -188,7 +428,23 @@ impl GetSinkResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSinkResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.sink, writer)?; + Ok(()) + } +} + +impl Deserializer for GetSinkResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let sink = load!(RpcHash, reader)?; + Ok(Self { sink }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntryRequest { pub transaction_id: RpcTransactionId, @@ -203,7 +459,29 @@ impl GetMempoolEntryRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntryRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcTransactionId, &self.transaction_id, writer)?; + store!(bool, &self.include_orphan_pool, writer)?; + store!(bool, &self.filter_transaction_pool, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMempoolEntryRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let transaction_id = load!(RpcTransactionId, reader)?; + let include_orphan_pool = load!(bool, reader)?; + let filter_transaction_pool = load!(bool, reader)?; + + Ok(Self { transaction_id, include_orphan_pool, filter_transaction_pool }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntryResponse { pub mempool_entry: RpcMempoolEntry, @@ -215,7 +493,23 @@ impl GetMempoolEntryResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntryResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcMempoolEntry, &self.mempool_entry, writer)?; + Ok(()) + } +} + +impl Deserializer for GetMempoolEntryResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let mempool_entry = deserialize!(RpcMempoolEntry, reader)?; + Ok(Self { mempool_entry }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntriesRequest { pub include_orphan_pool: bool, @@ -229,7 +523,27 @@ impl GetMempoolEntriesRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntriesRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.include_orphan_pool, writer)?; + store!(bool, &self.filter_transaction_pool, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMempoolEntriesRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let include_orphan_pool = load!(bool, reader)?; + let filter_transaction_pool = load!(bool, reader)?; + + Ok(Self { include_orphan_pool, filter_transaction_pool }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntriesResponse { pub mempool_entries: Vec, @@ -241,11 +555,41 @@ impl GetMempoolEntriesResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntriesResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(Vec, &self.mempool_entries, writer)?; + Ok(()) + } +} + +impl Deserializer for GetMempoolEntriesResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let mempool_entries = deserialize!(Vec, reader)?; + Ok(Self { mempool_entries }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetConnectedPeerInfoRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetConnectedPeerInfoRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetConnectedPeerInfoRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetConnectedPeerInfoResponse { pub peer_info: Vec, @@ -257,7 +601,23 @@ impl GetConnectedPeerInfoResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetConnectedPeerInfoResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.peer_info, writer)?; + Ok(()) + } +} + +impl Deserializer for GetConnectedPeerInfoResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let peer_info = load!(Vec, reader)?; + Ok(Self { peer_info }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AddPeerRequest { pub peer_address: RpcContextualPeerAddress, @@ -270,11 +630,45 @@ impl AddPeerRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for AddPeerRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcContextualPeerAddress, &self.peer_address, writer)?; + store!(bool, &self.is_permanent, writer)?; + + Ok(()) + } +} + +impl Deserializer for AddPeerRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let peer_address = load!(RpcContextualPeerAddress, reader)?; + let is_permanent = load!(bool, reader)?; + + Ok(Self { peer_address, is_permanent }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct AddPeerResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for AddPeerResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for AddPeerResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitTransactionRequest { pub transaction: RpcTransaction, @@ -287,7 +681,27 @@ impl SubmitTransactionRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for SubmitTransactionRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcTransaction, &self.transaction, writer)?; + store!(bool, &self.allow_orphan, writer)?; + + Ok(()) + } +} + +impl Deserializer for SubmitTransactionRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let transaction = deserialize!(RpcTransaction, reader)?; + let allow_orphan = load!(bool, reader)?; + + Ok(Self { transaction, allow_orphan }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitTransactionResponse { pub transaction_id: RpcTransactionId, @@ -299,7 +713,25 @@ impl SubmitTransactionResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for SubmitTransactionResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcTransactionId, &self.transaction_id, writer)?; + + Ok(()) + } +} + +impl Deserializer for SubmitTransactionResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let transaction_id = load!(RpcTransactionId, reader)?; + + Ok(Self { transaction_id }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitTransactionReplacementRequest { pub transaction: RpcTransaction, @@ -311,7 +743,25 @@ impl SubmitTransactionReplacementRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for SubmitTransactionReplacementRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcTransaction, &self.transaction, writer)?; + + Ok(()) + } +} + +impl Deserializer for SubmitTransactionReplacementRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let transaction = deserialize!(RpcTransaction, reader)?; + + Ok(Self { transaction }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubmitTransactionReplacementResponse { pub transaction_id: RpcTransactionId, @@ -324,6 +774,26 @@ impl SubmitTransactionReplacementResponse { } } +impl Serializer for SubmitTransactionReplacementResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcTransactionId, &self.transaction_id, writer)?; + serialize!(RpcTransaction, &self.replaced_transaction, writer)?; + + Ok(()) + } +} + +impl Deserializer for SubmitTransactionReplacementResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let transaction_id = load!(RpcTransactionId, reader)?; + let replaced_transaction = deserialize!(RpcTransaction, reader)?; + + Ok(Self { transaction_id, replaced_transaction }) + } +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] pub struct GetSubnetworkRequest { @@ -336,7 +806,25 @@ impl GetSubnetworkRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSubnetworkRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcSubnetworkId, &self.subnetwork_id, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetSubnetworkRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let subnetwork_id = load!(RpcSubnetworkId, reader)?; + + Ok(Self { subnetwork_id }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSubnetworkResponse { pub gas_limit: u64, @@ -348,7 +836,25 @@ impl GetSubnetworkResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSubnetworkResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.gas_limit, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetSubnetworkResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let gas_limit = load!(u64, reader)?; + + Ok(Self { gas_limit }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetVirtualChainFromBlockRequest { pub start_hash: RpcHash, @@ -361,7 +867,27 @@ impl GetVirtualChainFromBlockRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetVirtualChainFromBlockRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.start_hash, writer)?; + store!(bool, &self.include_accepted_transaction_ids, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetVirtualChainFromBlockRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let start_hash = load!(RpcHash, reader)?; + let include_accepted_transaction_ids = load!(bool, reader)?; + + Ok(Self { start_hash, include_accepted_transaction_ids }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetVirtualChainFromBlockResponse { pub removed_chain_block_hashes: Vec, @@ -379,7 +905,29 @@ impl GetVirtualChainFromBlockResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetVirtualChainFromBlockResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.removed_chain_block_hashes, writer)?; + store!(Vec, &self.added_chain_block_hashes, writer)?; + store!(Vec, &self.accepted_transaction_ids, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetVirtualChainFromBlockResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let removed_chain_block_hashes = load!(Vec, reader)?; + let added_chain_block_hashes = load!(Vec, reader)?; + let accepted_transaction_ids = load!(Vec, reader)?; + + Ok(Self { removed_chain_block_hashes, added_chain_block_hashes, accepted_transaction_ids }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlocksRequest { pub low_hash: Option, @@ -393,7 +941,29 @@ impl GetBlocksRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBlocksRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Option, &self.low_hash, writer)?; + store!(bool, &self.include_blocks, writer)?; + store!(bool, &self.include_transactions, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlocksRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let low_hash = load!(Option, reader)?; + let include_blocks = load!(bool, reader)?; + let include_transactions = load!(bool, reader)?; + + Ok(Self { low_hash, include_blocks, include_transactions }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlocksResponse { pub block_hashes: Vec, @@ -406,17 +976,65 @@ impl GetBlocksResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetBlockCountRequest {} +impl Serializer for GetBlocksResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.block_hashes, writer)?; + serialize!(Vec, &self.blocks, writer)?; -pub type GetBlockCountResponse = BlockCount; + Ok(()) + } +} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Deserializer for GetBlocksResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let block_hashes = load!(Vec, reader)?; + let blocks = deserialize!(Vec, reader)?; + + Ok(Self { block_hashes, blocks }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetBlockCountRequest {} + +impl Serializer for GetBlockCountRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetBlockCountRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +pub type GetBlockCountResponse = BlockCount; + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockDagInfoRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBlockDagInfoRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetBlockDagInfoRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBlockDagInfoResponse { pub network: RpcNetworkId, @@ -459,7 +1077,54 @@ impl GetBlockDagInfoResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBlockDagInfoResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcNetworkId, &self.network, writer)?; + store!(u64, &self.block_count, writer)?; + store!(u64, &self.header_count, writer)?; + store!(Vec, &self.tip_hashes, writer)?; + store!(f64, &self.difficulty, writer)?; + store!(u64, &self.past_median_time, writer)?; + store!(Vec, &self.virtual_parent_hashes, writer)?; + store!(RpcHash, &self.pruning_point_hash, writer)?; + store!(u64, &self.virtual_daa_score, writer)?; + store!(RpcHash, &self.sink, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBlockDagInfoResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let network = load!(RpcNetworkId, reader)?; + let block_count = load!(u64, reader)?; + let header_count = load!(u64, reader)?; + let tip_hashes = load!(Vec, reader)?; + let difficulty = load!(f64, reader)?; + let past_median_time = load!(u64, reader)?; + let virtual_parent_hashes = load!(Vec, reader)?; + let pruning_point_hash = load!(RpcHash, reader)?; + let virtual_daa_score = load!(u64, reader)?; + let sink = load!(RpcHash, reader)?; + + Ok(Self { + network, + block_count, + header_count, + tip_hashes, + difficulty, + past_median_time, + virtual_parent_hashes, + pruning_point_hash, + virtual_daa_score, + sink, + }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ResolveFinalityConflictRequest { pub finality_block_hash: RpcHash, @@ -471,19 +1136,79 @@ impl ResolveFinalityConflictRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ResolveFinalityConflictRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.finality_block_hash, writer)?; + + Ok(()) + } +} + +impl Deserializer for ResolveFinalityConflictRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let finality_block_hash = load!(RpcHash, reader)?; + + Ok(Self { finality_block_hash }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ResolveFinalityConflictResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ResolveFinalityConflictResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for ResolveFinalityConflictResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ShutdownRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ShutdownRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for ShutdownRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ShutdownResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ShutdownResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for ShutdownResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetHeadersRequest { pub start_hash: RpcHash, @@ -497,7 +1222,29 @@ impl GetHeadersRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetHeadersRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.start_hash, writer)?; + store!(u64, &self.limit, writer)?; + store!(bool, &self.is_ascending, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetHeadersRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let start_hash = load!(RpcHash, reader)?; + let limit = load!(u64, reader)?; + let is_ascending = load!(bool, reader)?; + + Ok(Self { start_hash, limit, is_ascending }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetHeadersResponse { pub headers: Vec, @@ -509,7 +1256,25 @@ impl GetHeadersResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetHeadersResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.headers, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetHeadersResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let headers = load!(Vec, reader)?; + + Ok(Self { headers }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBalanceByAddressRequest { pub address: RpcAddress, @@ -521,7 +1286,25 @@ impl GetBalanceByAddressRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBalanceByAddressRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcAddress, &self.address, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBalanceByAddressRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let address = load!(RpcAddress, reader)?; + + Ok(Self { address }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBalanceByAddressResponse { pub balance: u64, @@ -533,7 +1316,25 @@ impl GetBalanceByAddressResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBalanceByAddressResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.balance, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBalanceByAddressResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let balance = load!(u64, reader)?; + + Ok(Self { balance }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBalancesByAddressesRequest { pub addresses: Vec, @@ -545,7 +1346,25 @@ impl GetBalancesByAddressesRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBalancesByAddressesRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.addresses, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBalancesByAddressesRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let addresses = load!(Vec, reader)?; + + Ok(Self { addresses }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetBalancesByAddressesResponse { pub entries: Vec, @@ -557,11 +1376,43 @@ impl GetBalancesByAddressesResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetBalancesByAddressesResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(Vec, &self.entries, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetBalancesByAddressesResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let entries = deserialize!(Vec, reader)?; + + Ok(Self { entries }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSinkBlueScoreRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSinkBlueScoreRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetSinkBlueScoreRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSinkBlueScoreResponse { pub blue_score: u64, @@ -573,7 +1424,25 @@ impl GetSinkBlueScoreResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSinkBlueScoreResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.blue_score, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetSinkBlueScoreResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let blue_score = load!(u64, reader)?; + + Ok(Self { blue_score }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetUtxosByAddressesRequest { pub addresses: Vec, @@ -585,7 +1454,25 @@ impl GetUtxosByAddressesRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetUtxosByAddressesRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.addresses, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetUtxosByAddressesRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let addresses = load!(Vec, reader)?; + + Ok(Self { addresses }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetUtxosByAddressesResponse { pub entries: Vec, @@ -597,7 +1484,25 @@ impl GetUtxosByAddressesResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetUtxosByAddressesResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(Vec, &self.entries, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetUtxosByAddressesResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let entries = deserialize!(Vec, reader)?; + + Ok(Self { entries }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BanRequest { pub ip: RpcIpAddress, @@ -609,11 +1514,43 @@ impl BanRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for BanRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcIpAddress, &self.ip, writer)?; + + Ok(()) + } +} + +impl Deserializer for BanRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let ip = load!(RpcIpAddress, reader)?; + + Ok(Self { ip }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BanResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for BanResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for BanResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UnbanRequest { pub ip: RpcIpAddress, @@ -625,11 +1562,43 @@ impl UnbanRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for UnbanRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcIpAddress, &self.ip, writer)?; + + Ok(()) + } +} + +impl Deserializer for UnbanRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let ip = load!(RpcIpAddress, reader)?; + + Ok(Self { ip }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UnbanResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for UnbanResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for UnbanResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EstimateNetworkHashesPerSecondRequest { pub window_size: u32, @@ -642,7 +1611,27 @@ impl EstimateNetworkHashesPerSecondRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for EstimateNetworkHashesPerSecondRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u32, &self.window_size, writer)?; + store!(Option, &self.start_hash, writer)?; + + Ok(()) + } +} + +impl Deserializer for EstimateNetworkHashesPerSecondRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let window_size = load!(u32, reader)?; + let start_hash = load!(Option, reader)?; + + Ok(Self { window_size, start_hash }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct EstimateNetworkHashesPerSecondResponse { pub network_hashes_per_second: u64, @@ -654,7 +1643,25 @@ impl EstimateNetworkHashesPerSecondResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for EstimateNetworkHashesPerSecondResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.network_hashes_per_second, writer)?; + + Ok(()) + } +} + +impl Deserializer for EstimateNetworkHashesPerSecondResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let network_hashes_per_second = load!(u64, reader)?; + + Ok(Self { network_hashes_per_second }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntriesByAddressesRequest { pub addresses: Vec, @@ -669,7 +1676,29 @@ impl GetMempoolEntriesByAddressesRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntriesByAddressesRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.addresses, writer)?; + store!(bool, &self.include_orphan_pool, writer)?; + store!(bool, &self.filter_transaction_pool, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMempoolEntriesByAddressesRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let addresses = load!(Vec, reader)?; + let include_orphan_pool = load!(bool, reader)?; + let filter_transaction_pool = load!(bool, reader)?; + + Ok(Self { addresses, include_orphan_pool, filter_transaction_pool }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMempoolEntriesByAddressesResponse { pub entries: Vec, @@ -681,11 +1710,43 @@ impl GetMempoolEntriesByAddressesResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetMempoolEntriesByAddressesResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(Vec, &self.entries, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMempoolEntriesByAddressesResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let entries = deserialize!(Vec, reader)?; + + Ok(Self { entries }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetCoinSupplyRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetCoinSupplyRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetCoinSupplyRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetCoinSupplyResponse { pub max_sompi: u64, @@ -698,26 +1759,229 @@ impl GetCoinSupplyResponse { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetCoinSupplyResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.max_sompi, writer)?; + store!(u64, &self.circulating_sompi, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetCoinSupplyResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let max_sompi = load!(u64, reader)?; + let circulating_sompi = load!(u64, reader)?; + + Ok(Self { max_sompi, circulating_sompi }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PingRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for PingRequest { + fn serialize(&self, _writer: &mut W) -> std::io::Result<()> { + Ok(()) + } +} + +impl Deserializer for PingRequest { + fn deserialize(_reader: &mut R) -> std::io::Result { + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PingResponse {} -// TODO - custom wRPC commands (need review and implementation in gRPC) +impl Serializer for PingResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for PingResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + Ok(Self {}) + } +} #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] +pub struct ConnectionsProfileData { + pub cpu_usage: f32, + pub memory_usage: u64, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetConnectionsRequest { + pub include_profile_data: bool, +} + +impl Serializer for GetConnectionsRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(bool, &self.include_profile_data, writer)?; + Ok(()) + } +} + +impl Deserializer for GetConnectionsRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let include_profile_data = load!(bool, reader)?; + Ok(Self { include_profile_data }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetConnectionsResponse { + pub clients: u32, + pub peers: u16, + pub profile_data: Option, +} + +impl Serializer for GetConnectionsResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u32, &self.clients, writer)?; + store!(u16, &self.peers, writer)?; + store!(Option, &self.profile_data, writer)?; + Ok(()) + } +} + +impl Deserializer for GetConnectionsResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let clients = load!(u32, reader)?; + let peers = load!(u16, reader)?; + let extra = load!(Option, reader)?; + Ok(Self { clients, peers, profile_data: extra }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetSystemInfoRequest {} + +impl Serializer for GetSystemInfoRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetSystemInfoRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + + Ok(Self {}) + } +} + +#[derive(Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetSystemInfoResponse { + pub version: String, + pub system_id: Option>, + pub git_hash: Option>, + pub cpu_physical_cores: u16, + pub total_memory: u64, + pub fd_limit: u32, +} + +impl std::fmt::Debug for GetSystemInfoResponse { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GetSystemInfoResponse") + .field("version", &self.version) + .field("system_id", &self.system_id.as_ref().map(|id| id.to_hex())) + .field("git_hash", &self.git_hash.as_ref().map(|hash| hash.to_hex())) + .field("cpu_physical_cores", &self.cpu_physical_cores) + .field("total_memory", &self.total_memory) + .field("fd_limit", &self.fd_limit) + .finish() + } +} + +impl Serializer for GetSystemInfoResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(String, &self.version, writer)?; + store!(Option>, &self.system_id, writer)?; + store!(Option>, &self.git_hash, writer)?; + store!(u16, &self.cpu_physical_cores, writer)?; + store!(u64, &self.total_memory, writer)?; + store!(u32, &self.fd_limit, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetSystemInfoResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let version = load!(String, reader)?; + let system_id = load!(Option>, reader)?; + let git_hash = load!(Option>, reader)?; + let cpu_physical_cores = load!(u16, reader)?; + let total_memory = load!(u64, reader)?; + let fd_limit = load!(u32, reader)?; + + Ok(Self { version, system_id, git_hash, cpu_physical_cores, total_memory, fd_limit }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct GetMetricsRequest { pub process_metrics: bool, pub connection_metrics: bool, pub bandwidth_metrics: bool, pub consensus_metrics: bool, + pub storage_metrics: bool, + pub custom_metrics: bool, +} + +impl Serializer for GetMetricsRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.process_metrics, writer)?; + store!(bool, &self.connection_metrics, writer)?; + store!(bool, &self.bandwidth_metrics, writer)?; + store!(bool, &self.consensus_metrics, writer)?; + store!(bool, &self.storage_metrics, writer)?; + store!(bool, &self.custom_metrics, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMetricsRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let process_metrics = load!(bool, reader)?; + let connection_metrics = load!(bool, reader)?; + let bandwidth_metrics = load!(bool, reader)?; + let consensus_metrics = load!(bool, reader)?; + let storage_metrics = load!(bool, reader)?; + let custom_metrics = load!(bool, reader)?; + + Ok(Self { process_metrics, connection_metrics, bandwidth_metrics, consensus_metrics, storage_metrics, custom_metrics }) + } } -#[derive(Default, Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ProcessMetrics { pub resident_set_size: u64, @@ -731,7 +1995,51 @@ pub struct ProcessMetrics { pub disk_io_write_per_sec: f32, } -#[derive(Default, Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ProcessMetrics { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.resident_set_size, writer)?; + store!(u64, &self.virtual_memory_size, writer)?; + store!(u32, &self.core_num, writer)?; + store!(f32, &self.cpu_usage, writer)?; + store!(u32, &self.fd_num, writer)?; + store!(u64, &self.disk_io_read_bytes, writer)?; + store!(u64, &self.disk_io_write_bytes, writer)?; + store!(f32, &self.disk_io_read_per_sec, writer)?; + store!(f32, &self.disk_io_write_per_sec, writer)?; + + Ok(()) + } +} + +impl Deserializer for ProcessMetrics { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let resident_set_size = load!(u64, reader)?; + let virtual_memory_size = load!(u64, reader)?; + let core_num = load!(u32, reader)?; + let cpu_usage = load!(f32, reader)?; + let fd_num = load!(u32, reader)?; + let disk_io_read_bytes = load!(u64, reader)?; + let disk_io_write_bytes = load!(u64, reader)?; + let disk_io_read_per_sec = load!(f32, reader)?; + let disk_io_write_per_sec = load!(f32, reader)?; + + Ok(Self { + resident_set_size, + virtual_memory_size, + core_num, + cpu_usage, + fd_num, + disk_io_read_bytes, + disk_io_write_bytes, + disk_io_read_per_sec, + disk_io_write_per_sec, + }) + } +} + +#[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConnectionMetrics { pub borsh_live_connections: u32, @@ -744,7 +2052,45 @@ pub struct ConnectionMetrics { pub active_peers: u32, } -#[derive(Default, Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ConnectionMetrics { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u32, &self.borsh_live_connections, writer)?; + store!(u64, &self.borsh_connection_attempts, writer)?; + store!(u64, &self.borsh_handshake_failures, writer)?; + store!(u32, &self.json_live_connections, writer)?; + store!(u64, &self.json_connection_attempts, writer)?; + store!(u64, &self.json_handshake_failures, writer)?; + store!(u32, &self.active_peers, writer)?; + + Ok(()) + } +} + +impl Deserializer for ConnectionMetrics { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let borsh_live_connections = load!(u32, reader)?; + let borsh_connection_attempts = load!(u64, reader)?; + let borsh_handshake_failures = load!(u64, reader)?; + let json_live_connections = load!(u32, reader)?; + let json_connection_attempts = load!(u64, reader)?; + let json_handshake_failures = load!(u64, reader)?; + let active_peers = load!(u32, reader)?; + + Ok(Self { + borsh_live_connections, + borsh_connection_attempts, + borsh_handshake_failures, + json_live_connections, + json_connection_attempts, + json_handshake_failures, + active_peers, + }) + } +} + +#[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BandwidthMetrics { pub borsh_bytes_tx: u64, @@ -757,7 +2103,48 @@ pub struct BandwidthMetrics { pub grpc_bytes_rx: u64, } -#[derive(Default, Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for BandwidthMetrics { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.borsh_bytes_tx, writer)?; + store!(u64, &self.borsh_bytes_rx, writer)?; + store!(u64, &self.json_bytes_tx, writer)?; + store!(u64, &self.json_bytes_rx, writer)?; + store!(u64, &self.p2p_bytes_tx, writer)?; + store!(u64, &self.p2p_bytes_rx, writer)?; + store!(u64, &self.grpc_bytes_tx, writer)?; + store!(u64, &self.grpc_bytes_rx, writer)?; + + Ok(()) + } +} + +impl Deserializer for BandwidthMetrics { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let borsh_bytes_tx = load!(u64, reader)?; + let borsh_bytes_rx = load!(u64, reader)?; + let json_bytes_tx = load!(u64, reader)?; + let json_bytes_rx = load!(u64, reader)?; + let p2p_bytes_tx = load!(u64, reader)?; + let p2p_bytes_rx = load!(u64, reader)?; + let grpc_bytes_tx = load!(u64, reader)?; + let grpc_bytes_rx = load!(u64, reader)?; + + Ok(Self { + borsh_bytes_tx, + borsh_bytes_rx, + json_bytes_tx, + json_bytes_rx, + p2p_bytes_tx, + p2p_bytes_rx, + grpc_bytes_tx, + grpc_bytes_rx, + }) + } +} + +#[derive(Default, Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct ConsensusMetrics { pub node_blocks_submitted_count: u64, @@ -779,7 +2166,115 @@ pub struct ConsensusMetrics { pub network_virtual_daa_score: u64, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for ConsensusMetrics { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.node_blocks_submitted_count, writer)?; + store!(u64, &self.node_headers_processed_count, writer)?; + store!(u64, &self.node_dependencies_processed_count, writer)?; + store!(u64, &self.node_bodies_processed_count, writer)?; + store!(u64, &self.node_transactions_processed_count, writer)?; + store!(u64, &self.node_chain_blocks_processed_count, writer)?; + store!(u64, &self.node_mass_processed_count, writer)?; + store!(u64, &self.node_database_blocks_count, writer)?; + store!(u64, &self.node_database_headers_count, writer)?; + store!(u64, &self.network_mempool_size, writer)?; + store!(u32, &self.network_tip_hashes_count, writer)?; + store!(f64, &self.network_difficulty, writer)?; + store!(u64, &self.network_past_median_time, writer)?; + store!(u32, &self.network_virtual_parent_hashes_count, writer)?; + store!(u64, &self.network_virtual_daa_score, writer)?; + + Ok(()) + } +} + +impl Deserializer for ConsensusMetrics { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let node_blocks_submitted_count = load!(u64, reader)?; + let node_headers_processed_count = load!(u64, reader)?; + let node_dependencies_processed_count = load!(u64, reader)?; + let node_bodies_processed_count = load!(u64, reader)?; + let node_transactions_processed_count = load!(u64, reader)?; + let node_chain_blocks_processed_count = load!(u64, reader)?; + let node_mass_processed_count = load!(u64, reader)?; + let node_database_blocks_count = load!(u64, reader)?; + let node_database_headers_count = load!(u64, reader)?; + let network_mempool_size = load!(u64, reader)?; + let network_tip_hashes_count = load!(u32, reader)?; + let network_difficulty = load!(f64, reader)?; + let network_past_median_time = load!(u64, reader)?; + let network_virtual_parent_hashes_count = load!(u32, reader)?; + let network_virtual_daa_score = load!(u64, reader)?; + + Ok(Self { + node_blocks_submitted_count, + node_headers_processed_count, + node_dependencies_processed_count, + node_bodies_processed_count, + node_transactions_processed_count, + node_chain_blocks_processed_count, + node_mass_processed_count, + node_database_blocks_count, + node_database_headers_count, + network_mempool_size, + network_tip_hashes_count, + network_difficulty, + network_past_median_time, + network_virtual_parent_hashes_count, + network_virtual_daa_score, + }) + } +} + +#[derive(Default, Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct StorageMetrics { + pub storage_size_bytes: u64, +} + +impl Serializer for StorageMetrics { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.storage_size_bytes, writer)?; + + Ok(()) + } +} + +impl Deserializer for StorageMetrics { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let storage_size_bytes = load!(u64, reader)?; + + Ok(Self { storage_size_bytes }) + } +} + +// TODO: Custom metrics dictionary +#[derive(Clone, Debug, Serialize, Deserialize)] +pub enum CustomMetricValue { + Placeholder, +} + +impl Serializer for CustomMetricValue { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + + Ok(()) + } +} + +impl Deserializer for CustomMetricValue { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + + Ok(CustomMetricValue::Placeholder) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetMetricsResponse { pub server_time: u64, @@ -787,6 +2282,9 @@ pub struct GetMetricsResponse { pub connection_metrics: Option, pub bandwidth_metrics: Option, pub consensus_metrics: Option, + pub storage_metrics: Option, + // TODO: this is currently a placeholder + pub custom_metrics: Option>, } impl GetMetricsResponse { @@ -796,19 +2294,95 @@ impl GetMetricsResponse { connection_metrics: Option, bandwidth_metrics: Option, consensus_metrics: Option, + storage_metrics: Option, + custom_metrics: Option>, ) -> Self { - Self { process_metrics, connection_metrics, bandwidth_metrics, consensus_metrics, server_time } + Self { + process_metrics, + connection_metrics, + bandwidth_metrics, + consensus_metrics, + storage_metrics, + server_time, + custom_metrics, + } + } +} + +impl Serializer for GetMetricsResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.server_time, writer)?; + serialize!(Option, &self.process_metrics, writer)?; + serialize!(Option, &self.connection_metrics, writer)?; + serialize!(Option, &self.bandwidth_metrics, writer)?; + serialize!(Option, &self.consensus_metrics, writer)?; + serialize!(Option, &self.storage_metrics, writer)?; + serialize!(Option>, &self.custom_metrics, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetMetricsResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let server_time = load!(u64, reader)?; + let process_metrics = deserialize!(Option, reader)?; + let connection_metrics = deserialize!(Option, reader)?; + let bandwidth_metrics = deserialize!(Option, reader)?; + let consensus_metrics = deserialize!(Option, reader)?; + let storage_metrics = deserialize!(Option, reader)?; + let custom_metrics = deserialize!(Option>, reader)?; + + Ok(Self { + server_time, + process_metrics, + connection_metrics, + bandwidth_metrics, + consensus_metrics, + storage_metrics, + custom_metrics, + }) } } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] +#[borsh(use_discriminant = true)] +pub enum RpcCaps { + Full = 0, + Blocks, + UtxoIndex, + Mempool, + Metrics, + Visualizer, + Mining, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] pub struct GetServerInfoRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetServerInfoRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetServerInfoRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetServerInfoResponse { - pub rpc_api_version: [u16; 4], + pub rpc_api_version: u16, + pub rpc_api_revision: u16, pub server_version: String, pub network_id: RpcNetworkId, pub has_utxo_index: bool, @@ -816,17 +2390,81 @@ pub struct GetServerInfoResponse { pub virtual_daa_score: u64, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] -#[serde(rename_all = "camelCase")] -pub struct GetSyncStatusRequest {} +impl Serializer for GetServerInfoResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + + store!(u16, &self.rpc_api_version, writer)?; + store!(u16, &self.rpc_api_revision, writer)?; + + store!(String, &self.server_version, writer)?; + store!(RpcNetworkId, &self.network_id, writer)?; + store!(bool, &self.has_utxo_index, writer)?; + store!(bool, &self.is_synced, writer)?; + store!(u64, &self.virtual_daa_score, writer)?; + + Ok(()) + } +} + +impl Deserializer for GetServerInfoResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + + let rpc_api_version = load!(u16, reader)?; + let rpc_api_revision = load!(u16, reader)?; + + let server_version = load!(String, reader)?; + let network_id = load!(RpcNetworkId, reader)?; + let has_utxo_index = load!(bool, reader)?; + let is_synced = load!(bool, reader)?; + let virtual_daa_score = load!(u64, reader)?; + + Ok(Self { rpc_api_version, rpc_api_revision, server_version, network_id, has_utxo_index, is_synced, virtual_daa_score }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct GetSyncStatusRequest {} + +impl Serializer for GetSyncStatusRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetSyncStatusRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetSyncStatusResponse { pub is_synced: bool, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetSyncStatusResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.is_synced, writer)?; + Ok(()) + } +} + +impl Deserializer for GetSyncStatusResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let is_synced = load!(bool, reader)?; + Ok(Self { is_synced }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetDaaScoreTimestampEstimateRequest { pub daa_scores: Vec, @@ -838,7 +2476,23 @@ impl GetDaaScoreTimestampEstimateRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetDaaScoreTimestampEstimateRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.daa_scores, writer)?; + Ok(()) + } +} + +impl Deserializer for GetDaaScoreTimestampEstimateRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let daa_scores = load!(Vec, reader)?; + Ok(Self { daa_scores }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetDaaScoreTimestampEstimateResponse { pub timestamps: Vec, @@ -850,26 +2504,88 @@ impl GetDaaScoreTimestampEstimateResponse { } } +impl Serializer for GetDaaScoreTimestampEstimateResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.timestamps, writer)?; + Ok(()) + } +} + +impl Deserializer for GetDaaScoreTimestampEstimateResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let timestamps = load!(Vec, reader)?; + Ok(Self { timestamps }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // Fee rate estimations -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateRequest {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetFeeEstimateRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for GetFeeEstimateRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateResponse { pub estimate: RpcFeeEstimate, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetFeeEstimateResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcFeeEstimate, &self.estimate, writer)?; + Ok(()) + } +} + +impl Deserializer for GetFeeEstimateResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let estimate = deserialize!(RpcFeeEstimate, reader)?; + Ok(Self { estimate }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateExperimentalRequest { pub verbose: bool, } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for GetFeeEstimateExperimentalRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.verbose, writer)?; + Ok(()) + } +} + +impl Deserializer for GetFeeEstimateExperimentalRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let verbose = load!(bool, reader)?; + Ok(Self { verbose }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GetFeeEstimateExperimentalResponse { /// The usual feerate estimate response @@ -879,6 +2595,24 @@ pub struct GetFeeEstimateExperimentalResponse { pub verbose: Option, } +impl Serializer for GetFeeEstimateExperimentalResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcFeeEstimate, &self.estimate, writer)?; + serialize!(Option, &self.verbose, writer)?; + Ok(()) + } +} + +impl Deserializer for GetFeeEstimateExperimentalResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let estimate = deserialize!(RpcFeeEstimate, reader)?; + let verbose = deserialize!(Option, reader)?; + Ok(Self { estimate, verbose }) + } +} + // ---------------------------------------------------------------------------- // Subscriptions & notifications // ---------------------------------------------------------------------------- @@ -889,7 +2623,7 @@ pub struct GetFeeEstimateExperimentalResponse { /// NotifyBlockAddedRequest registers this connection for blockAdded notifications. /// /// See: BlockAddedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyBlockAddedRequest { pub command: Command, @@ -900,20 +2634,66 @@ impl NotifyBlockAddedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyBlockAddedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyBlockAddedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyBlockAddedResponse {} +impl Serializer for NotifyBlockAddedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyBlockAddedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + /// BlockAddedNotification is sent whenever a blocks has been added (NOT accepted) /// into the DAG. /// /// See: NotifyBlockAddedRequest -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct BlockAddedNotification { pub block: Arc, } +impl Serializer for BlockAddedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(RpcBlock, &self.block, writer)?; + Ok(()) + } +} + +impl Deserializer for BlockAddedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let block = deserialize!(RpcBlock, reader)?; + Ok(Self { block: block.into() }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // VirtualChainChangedNotification @@ -921,7 +2701,7 @@ pub struct BlockAddedNotification { // virtualDaaScoreChanged notifications. // // See: VirtualChainChangedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyVirtualChainChangedRequest { pub include_accepted_transaction_ids: bool, @@ -934,15 +2714,47 @@ impl NotifyVirtualChainChangedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyVirtualChainChangedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(bool, &self.include_accepted_transaction_ids, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyVirtualChainChangedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let include_accepted_transaction_ids = load!(bool, reader)?; + let command = load!(Command, reader)?; + Ok(Self { include_accepted_transaction_ids, command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyVirtualChainChangedResponse {} +impl Serializer for NotifyVirtualChainChangedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyVirtualChainChangedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + // VirtualChainChangedNotification is sent whenever the DAG's selected parent // chain had changed. // // See: NotifyVirtualChainChangedRequest -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VirtualChainChangedNotification { pub removed_chain_block_hashes: Arc>, @@ -950,10 +2762,34 @@ pub struct VirtualChainChangedNotification { pub accepted_transaction_ids: Arc>, } +impl Serializer for VirtualChainChangedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.removed_chain_block_hashes, writer)?; + store!(Vec, &self.added_chain_block_hashes, writer)?; + store!(Vec, &self.accepted_transaction_ids, writer)?; + Ok(()) + } +} + +impl Deserializer for VirtualChainChangedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let removed_chain_block_hashes = load!(Vec, reader)?; + let added_chain_block_hashes = load!(Vec, reader)?; + let accepted_transaction_ids = load!(Vec, reader)?; + Ok(Self { + removed_chain_block_hashes: removed_chain_block_hashes.into(), + added_chain_block_hashes: added_chain_block_hashes.into(), + accepted_transaction_ids: accepted_transaction_ids.into(), + }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // FinalityConflictNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyFinalityConflictRequest { pub command: Command, @@ -965,20 +2801,66 @@ impl NotifyFinalityConflictRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyFinalityConflictRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyFinalityConflictRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyFinalityConflictResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyFinalityConflictResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyFinalityConflictResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FinalityConflictNotification { pub violating_block_hash: RpcHash, } +impl Serializer for FinalityConflictNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.violating_block_hash, writer)?; + Ok(()) + } +} + +impl Deserializer for FinalityConflictNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let violating_block_hash = load!(RpcHash, reader)?; + Ok(Self { violating_block_hash }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // FinalityConflictResolvedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyFinalityConflictResolvedRequest { pub command: Command, @@ -990,16 +2872,62 @@ impl NotifyFinalityConflictResolvedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyFinalityConflictResolvedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyFinalityConflictResolvedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyFinalityConflictResolvedResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyFinalityConflictResolvedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyFinalityConflictResolvedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct FinalityConflictResolvedNotification { pub finality_block_hash: RpcHash, } +impl Serializer for FinalityConflictResolvedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(RpcHash, &self.finality_block_hash, writer)?; + Ok(()) + } +} + +impl Deserializer for FinalityConflictResolvedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let finality_block_hash = load!(RpcHash, reader)?; + Ok(Self { finality_block_hash }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~ // UtxosChangedNotification @@ -1012,7 +2940,7 @@ pub struct FinalityConflictResolvedNotification { // This call is only available when this spectred was started with `--utxoindex` // // See: UtxosChangedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyUtxosChangedRequest { pub addresses: Vec, @@ -1025,14 +2953,46 @@ impl NotifyUtxosChangedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyUtxosChangedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Vec, &self.addresses, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyUtxosChangedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let addresses = load!(Vec, reader)?; + let command = load!(Command, reader)?; + Ok(Self { addresses, command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyUtxosChangedResponse {} +impl Serializer for NotifyUtxosChangedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyUtxosChangedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + // UtxosChangedNotificationMessage is sent whenever the UTXO index had been updated. // // See: NotifyUtxosChangedRequest -#[derive(Clone, Debug, Default, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UtxosChangedNotification { pub added: Arc>, @@ -1069,6 +3029,24 @@ impl UtxosChangedNotification { } } +impl Serializer for UtxosChangedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + serialize!(Vec, &self.added, writer)?; + serialize!(Vec, &self.removed, writer)?; + Ok(()) + } +} + +impl Deserializer for UtxosChangedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let added = deserialize!(Vec, reader)?; + let removed = deserialize!(Vec, reader)?; + Ok(Self { added: added.into(), removed: removed.into() }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // SinkBlueScoreChangedNotification @@ -1076,7 +3054,7 @@ impl UtxosChangedNotification { // sinkBlueScoreChanged notifications. // // See: SinkBlueScoreChangedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifySinkBlueScoreChangedRequest { pub command: Command, @@ -1088,20 +3066,66 @@ impl NotifySinkBlueScoreChangedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifySinkBlueScoreChangedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifySinkBlueScoreChangedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifySinkBlueScoreChangedResponse {} +impl Serializer for NotifySinkBlueScoreChangedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifySinkBlueScoreChangedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + // SinkBlueScoreChangedNotification is sent whenever the blue score // of the virtual's selected parent changes. // /// See: NotifySinkBlueScoreChangedRequest -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SinkBlueScoreChangedNotification { pub sink_blue_score: u64, } +impl Serializer for SinkBlueScoreChangedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.sink_blue_score, writer)?; + Ok(()) + } +} + +impl Deserializer for SinkBlueScoreChangedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let sink_blue_score = load!(u64, reader)?; + Ok(Self { sink_blue_score }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // VirtualDaaScoreChangedNotification @@ -1109,7 +3133,7 @@ pub struct SinkBlueScoreChangedNotification { // virtualDaaScoreChanged notifications. // // See: VirtualDaaScoreChangedNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyVirtualDaaScoreChangedRequest { pub command: Command, @@ -1121,24 +3145,70 @@ impl NotifyVirtualDaaScoreChangedRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyVirtualDaaScoreChangedRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyVirtualDaaScoreChangedRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyVirtualDaaScoreChangedResponse {} +impl Serializer for NotifyVirtualDaaScoreChangedResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyVirtualDaaScoreChangedResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + // VirtualDaaScoreChangedNotification is sent whenever the DAA score // of the virtual changes. // // See NotifyVirtualDaaScoreChangedRequest -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct VirtualDaaScoreChangedNotification { pub virtual_daa_score: u64, } +impl Serializer for VirtualDaaScoreChangedNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.virtual_daa_score, writer)?; + Ok(()) + } +} + +impl Deserializer for VirtualDaaScoreChangedNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let virtual_daa_score = load!(u64, reader)?; + Ok(Self { virtual_daa_score }) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // PruningPointUtxoSetOverrideNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyPruningPointUtxoSetOverrideRequest { pub command: Command, @@ -1150,21 +3220,65 @@ impl NotifyPruningPointUtxoSetOverrideRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyPruningPointUtxoSetOverrideRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyPruningPointUtxoSetOverrideRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyPruningPointUtxoSetOverrideResponse {} -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyPruningPointUtxoSetOverrideResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyPruningPointUtxoSetOverrideResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct PruningPointUtxoSetOverrideNotification {} +impl Serializer for PruningPointUtxoSetOverrideNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for PruningPointUtxoSetOverrideNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ // NewBlockTemplateNotification /// NotifyNewBlockTemplateRequest registers this connection for blockAdded notifications. /// /// See: NewBlockTemplateNotification -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyNewBlockTemplateRequest { pub command: Command, @@ -1175,22 +3289,66 @@ impl NotifyNewBlockTemplateRequest { } } -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +impl Serializer for NotifyNewBlockTemplateRequest { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(Command, &self.command, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyNewBlockTemplateRequest { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let command = load!(Command, reader)?; + Ok(Self { command }) + } +} + +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NotifyNewBlockTemplateResponse {} +impl Serializer for NotifyNewBlockTemplateResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NotifyNewBlockTemplateResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + /// NewBlockTemplateNotification is sent whenever a blocks has been added (NOT accepted) /// into the DAG. /// /// See: NotifyNewBlockTemplateRequest -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct NewBlockTemplateNotification {} +impl Serializer for NewBlockTemplateNotification { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for NewBlockTemplateNotification { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + Ok(Self {}) + } +} + /// /// wRPC response for RpcApiOps::Subscribe request /// -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct SubscribeResponse { id: u64, @@ -1202,9 +3360,38 @@ impl SubscribeResponse { } } +impl Serializer for SubscribeResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u64, &self.id, writer)?; + Ok(()) + } +} + +impl Deserializer for SubscribeResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader)?; + let id = load!(u64, reader)?; + Ok(Self { id }) + } +} + /// /// wRPC response for RpcApiOps::Unsubscribe request /// -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct UnsubscribeResponse {} + +impl Serializer for UnsubscribeResponse { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer) + } +} + +impl Deserializer for UnsubscribeResponse { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u16, reader); + Ok(Self {}) + } +} diff --git a/rpc/core/src/model/mod.rs b/rpc/core/src/model/mod.rs index 8950bd1..beef032 100644 --- a/rpc/core/src/model/mod.rs +++ b/rpc/core/src/model/mod.rs @@ -11,6 +11,7 @@ pub mod network; pub mod peer; pub mod script_class; pub mod subnets; +mod tests; pub mod tx; pub use address::*; diff --git a/rpc/core/src/model/tests.rs b/rpc/core/src/model/tests.rs new file mode 100644 index 0000000..91fce8e --- /dev/null +++ b/rpc/core/src/model/tests.rs @@ -0,0 +1,1325 @@ +#[cfg(test)] +mod mockery { + + use crate::{model::*, RpcScriptClass}; + use rand::Rng; + use spectre_addresses::{Prefix, Version}; + use spectre_consensus_core::api::BlockCount; + use spectre_consensus_core::network::NetworkType; + use spectre_consensus_core::subnets::SubnetworkId; + use spectre_consensus_core::tx::ScriptPublicKey; + use spectre_hashes::Hash; + use spectre_math::Uint192; + use spectre_notify::subscription::Command; + use spectre_rpc_macros::test_wrpc_serializer as test; + use spectre_utils::networking::{ContextualNetAddress, IpAddress, NetAddress}; + use std::net::{IpAddr, Ipv4Addr}; + use std::sync::Arc; + use uuid::Uuid; + use workflow_serializer::prelude::*; + + // this trait is used to generate random + // values for testing on various data types + trait Mock { + fn mock() -> Self; + } + + impl Mock for Option + where + T: Mock, + { + fn mock() -> Self { + Some(T::mock()) + } + } + + impl Mock for Vec + where + T: Mock, + { + fn mock() -> Self { + vec![T::mock()] + } + } + + impl Mock for Arc + where + T: Mock, + { + fn mock() -> Self { + Arc::new(T::mock()) + } + } + + fn mock() -> T + where + T: Mock, + { + Mock::mock() + } + + // this function tests serialization and deserialization of a type + // by serializing it (A), deserializing it, serializing it again (B) + // and comparing A and B buffers. + fn test(kind: &str) + where + T: Serializer + Deserializer + Mock, + { + let data = T::mock(); + + const PREFIX: u32 = 0x12345678; + const SUFFIX: u32 = 0x90abcdef; + + let mut buffer1 = Vec::new(); + let writer = &mut buffer1; + store!(u32, &PREFIX, writer).unwrap(); + serialize!(T, &data, writer).unwrap(); + store!(u32, &SUFFIX, writer).unwrap(); + + let reader = &mut buffer1.as_slice(); + let prefix: u32 = load!(u32, reader).unwrap(); + // this will never occur, but it's a good practice to check in case + // the serialization/deserialization logic changes in the future + assert_eq!(prefix, PREFIX, "misalignment when consuming serialized buffer in `{kind}`"); + let tmp = deserialize!(T, reader).unwrap(); + let suffix: u32 = load!(u32, reader).unwrap(); + assert_eq!(suffix, SUFFIX, "misalignment when consuming serialized buffer in `{kind}`"); + + let mut buffer2 = Vec::new(); + let writer = &mut buffer2; + store!(u32, &PREFIX, writer).unwrap(); + serialize!(T, &tmp, writer).unwrap(); + store!(u32, &SUFFIX, writer).unwrap(); + + assert!(buffer1 == buffer2, "serialization/deserialization failure while testing `{kind}`"); + } + + #[macro_export] + macro_rules! impl_mock { + ($($type:ty),*) => { + $(impl Mock for $type { + fn mock() -> Self { + rand::thread_rng().gen() + } + })* + }; + } + + impl_mock!(bool, u8, u16, u32, f32, u64, i64, f64); + + impl Mock for Uint192 { + fn mock() -> Self { + Uint192([mock(), mock(), mock()]) + } + } + + impl Mock for SubnetworkId { + fn mock() -> Self { + let mut bytes: [u8; 20] = [0; 20]; + rand::thread_rng().fill(&mut bytes); + SubnetworkId::from_bytes(bytes) + } + } + + impl Mock for Hash { + fn mock() -> Self { + let mut bytes: [u8; 32] = [0; 32]; + rand::thread_rng().fill(&mut bytes); + Hash::from_bytes(bytes) + } + } + + impl Mock for RpcAddress { + fn mock() -> Self { + RpcAddress::new(Prefix::Mainnet, Version::PubKey, Hash::mock().as_bytes().as_slice()) + } + } + + impl Mock for RpcHeader { + fn mock() -> Self { + RpcHeader { + version: mock(), + timestamp: mock(), + bits: mock(), + nonce: mock(), + hash_merkle_root: mock(), + accepted_id_merkle_root: mock(), + utxo_commitment: mock(), + hash: mock(), + parents_by_level: vec![mock()], + daa_score: mock(), + blue_score: mock(), + blue_work: mock(), + pruning_point: mock(), + } + } + } + + impl Mock for RpcRawHeader { + fn mock() -> Self { + RpcRawHeader { + version: mock(), + timestamp: mock(), + bits: mock(), + nonce: mock(), + hash_merkle_root: mock(), + accepted_id_merkle_root: mock(), + utxo_commitment: mock(), + parents_by_level: vec![mock()], + daa_score: mock(), + blue_score: mock(), + blue_work: mock(), + pruning_point: mock(), + } + } + } + + impl Mock for RpcBlockVerboseData { + fn mock() -> Self { + RpcBlockVerboseData { + hash: mock(), + difficulty: mock(), + selected_parent_hash: mock(), + transaction_ids: mock(), + is_header_only: mock(), + blue_score: mock(), + children_hashes: mock(), + merge_set_blues_hashes: mock(), + merge_set_reds_hashes: mock(), + is_chain_block: mock(), + } + } + } + + impl Mock for RpcBlock { + fn mock() -> Self { + RpcBlock { header: mock(), transactions: mock(), verbose_data: mock() } + } + } + + impl Mock for RpcRawBlock { + fn mock() -> Self { + RpcRawBlock { header: mock(), transactions: mock() } + } + } + + impl Mock for RpcTransactionInputVerboseData { + fn mock() -> Self { + RpcTransactionInputVerboseData {} + } + } + + impl Mock for RpcTransactionInput { + fn mock() -> Self { + RpcTransactionInput { + previous_outpoint: mock(), + signature_script: Hash::mock().as_bytes().to_vec(), + sequence: mock(), + sig_op_count: mock(), + verbose_data: mock(), + } + } + } + + impl Mock for RpcTransactionOutputVerboseData { + fn mock() -> Self { + RpcTransactionOutputVerboseData { script_public_key_type: RpcScriptClass::PubKey, script_public_key_address: mock() } + } + } + + impl Mock for RpcTransactionOutput { + fn mock() -> Self { + RpcTransactionOutput { value: mock(), script_public_key: mock(), verbose_data: mock() } + } + } + + impl Mock for RpcTransactionVerboseData { + fn mock() -> Self { + RpcTransactionVerboseData { transaction_id: mock(), hash: mock(), mass: mock(), block_hash: mock(), block_time: mock() } + } + } + + impl Mock for RpcTransaction { + fn mock() -> Self { + RpcTransaction { + version: mock(), + inputs: mock(), + outputs: mock(), + lock_time: mock(), + subnetwork_id: mock(), + gas: mock(), + payload: Hash::mock().as_bytes().to_vec(), + mass: mock(), + verbose_data: mock(), + } + } + } + + impl Mock for RpcNodeId { + fn mock() -> Self { + RpcNodeId::new(Uuid::new_v4()) + } + } + + impl Mock for IpAddr { + fn mock() -> Self { + IpAddr::V4(Ipv4Addr::new(mock(), mock(), mock(), mock())) + } + } + + impl Mock for IpAddress { + fn mock() -> Self { + IpAddress::new(mock()) + } + } + + impl Mock for NetAddress { + fn mock() -> Self { + NetAddress::new(IpAddress::new(mock()), mock()) + } + } + + impl Mock for ContextualNetAddress { + fn mock() -> Self { + ContextualNetAddress::new(mock(), mock()) + } + } + + impl Mock for RpcPeerInfo { + fn mock() -> Self { + RpcPeerInfo { + id: mock(), + address: mock(), + last_ping_duration: mock(), + is_outbound: mock(), + time_offset: mock(), + user_agent: "0.4.2".to_string(), + advertised_protocol_version: mock(), + time_connected: mock(), + is_ibd_peer: mock(), + } + } + } + + impl Mock for RpcMempoolEntry { + fn mock() -> Self { + RpcMempoolEntry { fee: mock(), transaction: mock(), is_orphan: mock() } + } + } + + impl Mock for RpcMempoolEntryByAddress { + fn mock() -> Self { + RpcMempoolEntryByAddress { address: mock(), sending: mock(), receiving: mock() } + } + } + + impl Mock for ScriptPublicKey { + fn mock() -> Self { + let mut bytes: [u8; 36] = [0; 36]; + rand::thread_rng().fill(&mut bytes[..]); + ScriptPublicKey::from_vec(0, bytes.to_vec()) + } + } + + impl Mock for RpcUtxoEntry { + fn mock() -> Self { + RpcUtxoEntry { amount: mock(), script_public_key: mock(), block_daa_score: mock(), is_coinbase: true } + } + } + + impl Mock for RpcTransactionOutpoint { + fn mock() -> Self { + RpcTransactionOutpoint { transaction_id: mock(), index: mock() } + } + } + + impl Mock for RpcUtxosByAddressesEntry { + fn mock() -> Self { + RpcUtxosByAddressesEntry { address: mock(), outpoint: mock(), utxo_entry: mock() } + } + } + + impl Mock for ProcessMetrics { + fn mock() -> Self { + ProcessMetrics { + resident_set_size: mock(), + virtual_memory_size: mock(), + core_num: mock(), + cpu_usage: mock(), + fd_num: mock(), + disk_io_read_bytes: mock(), + disk_io_write_bytes: mock(), + disk_io_read_per_sec: mock(), + disk_io_write_per_sec: mock(), + } + } + } + + impl Mock for ConnectionMetrics { + fn mock() -> Self { + ConnectionMetrics { + borsh_live_connections: mock(), + borsh_connection_attempts: mock(), + borsh_handshake_failures: mock(), + json_live_connections: mock(), + json_connection_attempts: mock(), + json_handshake_failures: mock(), + active_peers: mock(), + } + } + } + + impl Mock for BandwidthMetrics { + fn mock() -> Self { + BandwidthMetrics { + borsh_bytes_tx: mock(), + borsh_bytes_rx: mock(), + json_bytes_tx: mock(), + json_bytes_rx: mock(), + p2p_bytes_tx: mock(), + p2p_bytes_rx: mock(), + grpc_bytes_tx: mock(), + grpc_bytes_rx: mock(), + } + } + } + + impl Mock for ConsensusMetrics { + fn mock() -> Self { + ConsensusMetrics { + node_blocks_submitted_count: mock(), + node_headers_processed_count: mock(), + node_dependencies_processed_count: mock(), + node_bodies_processed_count: mock(), + node_transactions_processed_count: mock(), + node_chain_blocks_processed_count: mock(), + node_mass_processed_count: mock(), + node_database_blocks_count: mock(), + node_database_headers_count: mock(), + network_mempool_size: mock(), + network_tip_hashes_count: mock(), + network_difficulty: mock(), + network_past_median_time: mock(), + network_virtual_parent_hashes_count: mock(), + network_virtual_daa_score: mock(), + } + } + } + + impl Mock for StorageMetrics { + fn mock() -> Self { + StorageMetrics { storage_size_bytes: mock() } + } + } + + // -------------------------------------------- + // implementations for all the rpc request + // and response data structures. + + impl Mock for SubmitBlockRequest { + fn mock() -> Self { + SubmitBlockRequest { block: mock(), allow_non_daa_blocks: true } + } + } + + test!(SubmitBlockRequest); + + impl Mock for SubmitBlockResponse { + fn mock() -> Self { + SubmitBlockResponse { report: SubmitBlockReport::Success } + } + } + + test!(SubmitBlockResponse); + + impl Mock for GetBlockTemplateRequest { + fn mock() -> Self { + GetBlockTemplateRequest { pay_address: mock(), extra_data: vec![4, 2] } + } + } + + test!(GetBlockTemplateRequest); + + impl Mock for GetBlockTemplateResponse { + fn mock() -> Self { + GetBlockTemplateResponse { block: mock(), is_synced: true } + } + } + + test!(GetBlockTemplateResponse); + + impl Mock for GetBlockRequest { + fn mock() -> Self { + GetBlockRequest { hash: mock(), include_transactions: true } + } + } + + test!(GetBlockRequest); + + impl Mock for GetBlockResponse { + fn mock() -> Self { + GetBlockResponse { block: mock() } + } + } + + test!(GetBlockResponse); + + impl Mock for GetInfoRequest { + fn mock() -> Self { + GetInfoRequest {} + } + } + + test!(GetInfoRequest); + + impl Mock for GetInfoResponse { + fn mock() -> Self { + GetInfoResponse { + p2p_id: Hash::mock().to_string(), + mempool_size: mock(), + server_version: "0.4.2".to_string(), + is_utxo_indexed: true, + is_synced: false, + has_notify_command: true, + has_message_id: false, + } + } + } + + test!(GetInfoResponse); + + impl Mock for GetCurrentNetworkRequest { + fn mock() -> Self { + GetCurrentNetworkRequest {} + } + } + + test!(GetCurrentNetworkRequest); + + impl Mock for GetCurrentNetworkResponse { + fn mock() -> Self { + GetCurrentNetworkResponse { network: NetworkType::Mainnet } + } + } + + test!(GetCurrentNetworkResponse); + + impl Mock for GetPeerAddressesRequest { + fn mock() -> Self { + GetPeerAddressesRequest {} + } + } + + test!(GetPeerAddressesRequest); + + impl Mock for GetPeerAddressesResponse { + fn mock() -> Self { + GetPeerAddressesResponse { known_addresses: mock(), banned_addresses: mock() } + } + } + + test!(GetPeerAddressesResponse); + + impl Mock for GetSinkRequest { + fn mock() -> Self { + GetSinkRequest {} + } + } + + test!(GetSinkRequest); + + impl Mock for GetSinkResponse { + fn mock() -> Self { + GetSinkResponse { sink: mock() } + } + } + + test!(GetSinkResponse); + + impl Mock for GetMempoolEntryRequest { + fn mock() -> Self { + GetMempoolEntryRequest { transaction_id: mock(), include_orphan_pool: true, filter_transaction_pool: false } + } + } + + test!(GetMempoolEntryRequest); + + impl Mock for GetMempoolEntryResponse { + fn mock() -> Self { + GetMempoolEntryResponse { mempool_entry: RpcMempoolEntry { fee: mock(), transaction: mock(), is_orphan: false } } + } + } + + test!(GetMempoolEntryResponse); + + impl Mock for GetMempoolEntriesRequest { + fn mock() -> Self { + GetMempoolEntriesRequest { include_orphan_pool: true, filter_transaction_pool: false } + } + } + + test!(GetMempoolEntriesRequest); + + impl Mock for GetMempoolEntriesResponse { + fn mock() -> Self { + GetMempoolEntriesResponse { mempool_entries: mock() } + } + } + + test!(GetMempoolEntriesResponse); + + impl Mock for GetConnectedPeerInfoRequest { + fn mock() -> Self { + GetConnectedPeerInfoRequest {} + } + } + + test!(GetConnectedPeerInfoRequest); + + impl Mock for GetConnectedPeerInfoResponse { + fn mock() -> Self { + GetConnectedPeerInfoResponse { peer_info: mock() } + } + } + + test!(GetConnectedPeerInfoResponse); + + impl Mock for AddPeerRequest { + fn mock() -> Self { + AddPeerRequest { peer_address: mock(), is_permanent: mock() } + } + } + + test!(AddPeerRequest); + + impl Mock for AddPeerResponse { + fn mock() -> Self { + AddPeerResponse {} + } + } + + test!(AddPeerResponse); + + impl Mock for SubmitTransactionRequest { + fn mock() -> Self { + SubmitTransactionRequest { transaction: mock(), allow_orphan: mock() } + } + } + + test!(SubmitTransactionRequest); + + impl Mock for SubmitTransactionResponse { + fn mock() -> Self { + SubmitTransactionResponse { transaction_id: mock() } + } + } + + test!(SubmitTransactionResponse); + + impl Mock for GetSubnetworkRequest { + fn mock() -> Self { + GetSubnetworkRequest { subnetwork_id: mock() } + } + } + + test!(GetSubnetworkRequest); + + impl Mock for GetSubnetworkResponse { + fn mock() -> Self { + GetSubnetworkResponse { gas_limit: mock() } + } + } + + test!(GetSubnetworkResponse); + + impl Mock for GetVirtualChainFromBlockRequest { + fn mock() -> Self { + GetVirtualChainFromBlockRequest { start_hash: mock(), include_accepted_transaction_ids: mock() } + } + } + + test!(GetVirtualChainFromBlockRequest); + + impl Mock for RpcAcceptedTransactionIds { + fn mock() -> Self { + RpcAcceptedTransactionIds { accepting_block_hash: mock(), accepted_transaction_ids: mock() } + } + } + + impl Mock for GetVirtualChainFromBlockResponse { + fn mock() -> Self { + GetVirtualChainFromBlockResponse { + removed_chain_block_hashes: mock(), + added_chain_block_hashes: mock(), + accepted_transaction_ids: mock(), + } + } + } + + test!(GetVirtualChainFromBlockResponse); + + impl Mock for GetBlocksRequest { + fn mock() -> Self { + GetBlocksRequest { low_hash: mock(), include_blocks: mock(), include_transactions: mock() } + } + } + + test!(GetBlocksRequest); + + impl Mock for GetBlocksResponse { + fn mock() -> Self { + GetBlocksResponse { block_hashes: mock(), blocks: mock() } + } + } + + test!(GetBlocksResponse); + + impl Mock for GetBlockCountRequest { + fn mock() -> Self { + GetBlockCountRequest {} + } + } + + test!(GetBlockCountRequest); + + impl Mock for BlockCount { + fn mock() -> Self { + BlockCount { header_count: mock(), block_count: mock() } + } + } + + test!(BlockCount); + + impl Mock for GetBlockDagInfoRequest { + fn mock() -> Self { + GetBlockDagInfoRequest {} + } + } + + test!(GetBlockDagInfoRequest); + + impl Mock for GetBlockDagInfoResponse { + fn mock() -> Self { + GetBlockDagInfoResponse { + network: NetworkType::Mainnet.try_into().unwrap(), + block_count: mock(), + header_count: mock(), + tip_hashes: mock(), + difficulty: mock(), + past_median_time: mock(), + virtual_parent_hashes: mock(), + pruning_point_hash: mock(), + virtual_daa_score: mock(), + sink: mock(), + } + } + } + + test!(GetBlockDagInfoResponse); + + impl Mock for ResolveFinalityConflictRequest { + fn mock() -> Self { + ResolveFinalityConflictRequest { finality_block_hash: mock() } + } + } + + test!(ResolveFinalityConflictRequest); + + impl Mock for ResolveFinalityConflictResponse { + fn mock() -> Self { + ResolveFinalityConflictResponse {} + } + } + + test!(ResolveFinalityConflictResponse); + + impl Mock for ShutdownRequest { + fn mock() -> Self { + ShutdownRequest {} + } + } + + test!(ShutdownRequest); + + impl Mock for ShutdownResponse { + fn mock() -> Self { + ShutdownResponse {} + } + } + + test!(ShutdownResponse); + + impl Mock for GetHeadersRequest { + fn mock() -> Self { + GetHeadersRequest { start_hash: mock(), limit: mock(), is_ascending: mock() } + } + } + + test!(GetHeadersRequest); + + impl Mock for GetHeadersResponse { + fn mock() -> Self { + GetHeadersResponse { headers: mock() } + } + } + + test!(GetHeadersResponse); + + impl Mock for GetBalanceByAddressRequest { + fn mock() -> Self { + GetBalanceByAddressRequest { address: mock() } + } + } + + test!(GetBalanceByAddressRequest); + + impl Mock for GetBalanceByAddressResponse { + fn mock() -> Self { + GetBalanceByAddressResponse { balance: mock() } + } + } + + test!(GetBalanceByAddressResponse); + + impl Mock for GetBalancesByAddressesRequest { + fn mock() -> Self { + GetBalancesByAddressesRequest { addresses: mock() } + } + } + + test!(GetBalancesByAddressesRequest); + + impl Mock for RpcBalancesByAddressesEntry { + fn mock() -> Self { + RpcBalancesByAddressesEntry { address: mock(), balance: mock() } + } + } + + impl Mock for GetBalancesByAddressesResponse { + fn mock() -> Self { + GetBalancesByAddressesResponse { entries: mock() } + } + } + + test!(GetBalancesByAddressesResponse); + + impl Mock for GetSinkBlueScoreRequest { + fn mock() -> Self { + GetSinkBlueScoreRequest {} + } + } + + test!(GetSinkBlueScoreRequest); + + impl Mock for GetSinkBlueScoreResponse { + fn mock() -> Self { + GetSinkBlueScoreResponse { blue_score: mock() } + } + } + + test!(GetSinkBlueScoreResponse); + + impl Mock for GetUtxosByAddressesRequest { + fn mock() -> Self { + GetUtxosByAddressesRequest { addresses: mock() } + } + } + + test!(GetUtxosByAddressesRequest); + + impl Mock for GetUtxosByAddressesResponse { + fn mock() -> Self { + GetUtxosByAddressesResponse { entries: mock() } + } + } + + test!(GetUtxosByAddressesResponse); + + impl Mock for BanRequest { + fn mock() -> Self { + BanRequest { ip: mock() } + } + } + + test!(BanRequest); + + impl Mock for BanResponse { + fn mock() -> Self { + BanResponse {} + } + } + + test!(BanResponse); + + impl Mock for UnbanRequest { + fn mock() -> Self { + UnbanRequest { ip: mock() } + } + } + + test!(UnbanRequest); + + impl Mock for UnbanResponse { + fn mock() -> Self { + UnbanResponse {} + } + } + + test!(UnbanResponse); + + impl Mock for EstimateNetworkHashesPerSecondRequest { + fn mock() -> Self { + EstimateNetworkHashesPerSecondRequest { window_size: mock(), start_hash: mock() } + } + } + + test!(EstimateNetworkHashesPerSecondRequest); + + impl Mock for EstimateNetworkHashesPerSecondResponse { + fn mock() -> Self { + EstimateNetworkHashesPerSecondResponse { network_hashes_per_second: mock() } + } + } + + test!(EstimateNetworkHashesPerSecondResponse); + + impl Mock for GetMempoolEntriesByAddressesRequest { + fn mock() -> Self { + GetMempoolEntriesByAddressesRequest { addresses: mock(), include_orphan_pool: true, filter_transaction_pool: false } + } + } + + test!(GetMempoolEntriesByAddressesRequest); + + impl Mock for GetMempoolEntriesByAddressesResponse { + fn mock() -> Self { + GetMempoolEntriesByAddressesResponse { entries: mock() } + } + } + + test!(GetMempoolEntriesByAddressesResponse); + + impl Mock for GetCoinSupplyRequest { + fn mock() -> Self { + GetCoinSupplyRequest {} + } + } + + test!(GetCoinSupplyRequest); + + impl Mock for GetCoinSupplyResponse { + fn mock() -> Self { + GetCoinSupplyResponse { max_sompi: mock(), circulating_sompi: mock() } + } + } + + test!(GetCoinSupplyResponse); + + impl Mock for PingRequest { + fn mock() -> Self { + PingRequest {} + } + } + + test!(PingRequest); + + impl Mock for PingResponse { + fn mock() -> Self { + PingResponse {} + } + } + + test!(PingResponse); + + impl Mock for GetConnectionsRequest { + fn mock() -> Self { + GetConnectionsRequest { include_profile_data: false } + } + } + + test!(GetConnectionsRequest); + + impl Mock for GetConnectionsResponse { + fn mock() -> Self { + GetConnectionsResponse { clients: mock(), peers: mock(), profile_data: None } + } + } + + test!(GetConnectionsResponse); + + impl Mock for GetSystemInfoRequest { + fn mock() -> Self { + GetSystemInfoRequest {} + } + } + + test!(GetSystemInfoRequest); + + impl Mock for GetSystemInfoResponse { + fn mock() -> Self { + GetSystemInfoResponse { + version: "1.2.3".to_string(), + system_id: mock(), + git_hash: mock(), + cpu_physical_cores: mock(), + total_memory: mock(), + fd_limit: mock(), + } + } + } + + test!(GetSystemInfoResponse); + + impl Mock for GetMetricsRequest { + fn mock() -> Self { + GetMetricsRequest { + process_metrics: true, + connection_metrics: true, + bandwidth_metrics: true, + consensus_metrics: true, + storage_metrics: true, + custom_metrics: false, + } + } + } + + test!(GetMetricsRequest); + + impl Mock for GetMetricsResponse { + fn mock() -> Self { + GetMetricsResponse { + server_time: mock(), + process_metrics: mock(), + connection_metrics: mock(), + bandwidth_metrics: mock(), + consensus_metrics: mock(), + storage_metrics: mock(), + custom_metrics: None, + } + } + } + + test!(GetMetricsResponse); + + impl Mock for GetServerInfoRequest { + fn mock() -> Self { + GetServerInfoRequest {} + } + } + + test!(GetServerInfoRequest); + + impl Mock for GetServerInfoResponse { + fn mock() -> Self { + GetServerInfoResponse { + rpc_api_version: mock(), + rpc_api_revision: mock(), + server_version: "0.4.2".to_string(), + network_id: NetworkType::Mainnet.try_into().unwrap(), + has_utxo_index: true, + is_synced: false, + virtual_daa_score: mock(), + } + } + } + + test!(GetServerInfoResponse); + + impl Mock for GetSyncStatusRequest { + fn mock() -> Self { + GetSyncStatusRequest {} + } + } + + test!(GetSyncStatusRequest); + + impl Mock for GetSyncStatusResponse { + fn mock() -> Self { + GetSyncStatusResponse { is_synced: true } + } + } + + test!(GetSyncStatusResponse); + + impl Mock for GetDaaScoreTimestampEstimateRequest { + fn mock() -> Self { + GetDaaScoreTimestampEstimateRequest { daa_scores: mock() } + } + } + + test!(GetDaaScoreTimestampEstimateRequest); + + impl Mock for GetDaaScoreTimestampEstimateResponse { + fn mock() -> Self { + GetDaaScoreTimestampEstimateResponse { timestamps: mock() } + } + } + + test!(GetDaaScoreTimestampEstimateResponse); + + impl Mock for NotifyBlockAddedRequest { + fn mock() -> Self { + NotifyBlockAddedRequest { command: Command::Start } + } + } + + test!(NotifyBlockAddedRequest); + + impl Mock for NotifyBlockAddedResponse { + fn mock() -> Self { + NotifyBlockAddedResponse {} + } + } + + test!(NotifyBlockAddedResponse); + + impl Mock for BlockAddedNotification { + fn mock() -> Self { + BlockAddedNotification { block: mock() } + } + } + + test!(BlockAddedNotification); + + impl Mock for NotifyVirtualChainChangedRequest { + fn mock() -> Self { + NotifyVirtualChainChangedRequest { command: Command::Start, include_accepted_transaction_ids: true } + } + } + + test!(NotifyVirtualChainChangedRequest); + + impl Mock for NotifyVirtualChainChangedResponse { + fn mock() -> Self { + NotifyVirtualChainChangedResponse {} + } + } + + test!(NotifyVirtualChainChangedResponse); + + impl Mock for VirtualChainChangedNotification { + fn mock() -> Self { + VirtualChainChangedNotification { + removed_chain_block_hashes: mock(), + added_chain_block_hashes: mock(), + accepted_transaction_ids: mock(), + } + } + } + + test!(VirtualChainChangedNotification); + + impl Mock for NotifyFinalityConflictRequest { + fn mock() -> Self { + NotifyFinalityConflictRequest { command: Command::Start } + } + } + + test!(NotifyFinalityConflictRequest); + + impl Mock for NotifyFinalityConflictResponse { + fn mock() -> Self { + NotifyFinalityConflictResponse {} + } + } + + test!(NotifyFinalityConflictResponse); + + impl Mock for FinalityConflictNotification { + fn mock() -> Self { + FinalityConflictNotification { violating_block_hash: mock() } + } + } + + test!(FinalityConflictNotification); + + impl Mock for NotifyFinalityConflictResolvedRequest { + fn mock() -> Self { + NotifyFinalityConflictResolvedRequest { command: Command::Start } + } + } + + test!(NotifyFinalityConflictResolvedRequest); + + impl Mock for NotifyFinalityConflictResolvedResponse { + fn mock() -> Self { + NotifyFinalityConflictResolvedResponse {} + } + } + + test!(NotifyFinalityConflictResolvedResponse); + + impl Mock for FinalityConflictResolvedNotification { + fn mock() -> Self { + FinalityConflictResolvedNotification { finality_block_hash: mock() } + } + } + + test!(FinalityConflictResolvedNotification); + + impl Mock for NotifyUtxosChangedRequest { + fn mock() -> Self { + NotifyUtxosChangedRequest { addresses: mock(), command: Command::Start } + } + } + + test!(NotifyUtxosChangedRequest); + + impl Mock for NotifyUtxosChangedResponse { + fn mock() -> Self { + NotifyUtxosChangedResponse {} + } + } + + test!(NotifyUtxosChangedResponse); + + impl Mock for UtxosChangedNotification { + fn mock() -> Self { + UtxosChangedNotification { added: mock(), removed: mock() } + } + } + + test!(UtxosChangedNotification); + + impl Mock for NotifySinkBlueScoreChangedRequest { + fn mock() -> Self { + NotifySinkBlueScoreChangedRequest { command: Command::Start } + } + } + + test!(NotifySinkBlueScoreChangedRequest); + + impl Mock for NotifySinkBlueScoreChangedResponse { + fn mock() -> Self { + NotifySinkBlueScoreChangedResponse {} + } + } + + test!(NotifySinkBlueScoreChangedResponse); + + impl Mock for SinkBlueScoreChangedNotification { + fn mock() -> Self { + SinkBlueScoreChangedNotification { sink_blue_score: mock() } + } + } + + test!(SinkBlueScoreChangedNotification); + + impl Mock for NotifyVirtualDaaScoreChangedRequest { + fn mock() -> Self { + NotifyVirtualDaaScoreChangedRequest { command: Command::Start } + } + } + + test!(NotifyVirtualDaaScoreChangedRequest); + + impl Mock for NotifyVirtualDaaScoreChangedResponse { + fn mock() -> Self { + NotifyVirtualDaaScoreChangedResponse {} + } + } + + test!(NotifyVirtualDaaScoreChangedResponse); + + impl Mock for VirtualDaaScoreChangedNotification { + fn mock() -> Self { + VirtualDaaScoreChangedNotification { virtual_daa_score: mock() } + } + } + + test!(VirtualDaaScoreChangedNotification); + + impl Mock for NotifyPruningPointUtxoSetOverrideRequest { + fn mock() -> Self { + NotifyPruningPointUtxoSetOverrideRequest { command: Command::Start } + } + } + + test!(NotifyPruningPointUtxoSetOverrideRequest); + + impl Mock for NotifyPruningPointUtxoSetOverrideResponse { + fn mock() -> Self { + NotifyPruningPointUtxoSetOverrideResponse {} + } + } + + test!(NotifyPruningPointUtxoSetOverrideResponse); + + impl Mock for PruningPointUtxoSetOverrideNotification { + fn mock() -> Self { + PruningPointUtxoSetOverrideNotification {} + } + } + + test!(PruningPointUtxoSetOverrideNotification); + + impl Mock for NotifyNewBlockTemplateRequest { + fn mock() -> Self { + NotifyNewBlockTemplateRequest { command: Command::Start } + } + } + + test!(NotifyNewBlockTemplateRequest); + + impl Mock for NotifyNewBlockTemplateResponse { + fn mock() -> Self { + NotifyNewBlockTemplateResponse {} + } + } + + test!(NotifyNewBlockTemplateResponse); + + impl Mock for NewBlockTemplateNotification { + fn mock() -> Self { + NewBlockTemplateNotification {} + } + } + + test!(NewBlockTemplateNotification); + + impl Mock for SubscribeResponse { + fn mock() -> Self { + SubscribeResponse::new(mock()) + } + } + + test!(SubscribeResponse); + + impl Mock for UnsubscribeResponse { + fn mock() -> Self { + UnsubscribeResponse {} + } + } + + test!(UnsubscribeResponse); + + struct Misalign; + + impl Mock for Misalign { + fn mock() -> Self { + Misalign + } + } + + impl Serializer for Misalign { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u32, &1, writer)?; + store!(u32, &2, writer)?; + store!(u32, &3, writer)?; + Ok(()) + } + } + + impl Deserializer for Misalign { + fn deserialize(reader: &mut R) -> std::io::Result { + let version: u32 = load!(u32, reader)?; + assert_eq!(version, 1); + Ok(Self) + } + } + + #[test] + fn test_misalignment() { + test::("Misalign"); + } +} diff --git a/rpc/core/src/model/tx.rs b/rpc/core/src/model/tx.rs index 8e0ea0e..4473b40 100644 --- a/rpc/core/src/model/tx.rs +++ b/rpc/core/src/model/tx.rs @@ -2,8 +2,11 @@ use borsh::{BorshDeserialize, BorshSerialize}; use serde::{Deserialize, Serialize}; use spectre_addresses::Address; use spectre_consensus_core::tx::{ - ScriptPublicKey, ScriptVec, TransactionId, TransactionInput, TransactionOutpoint, TransactionOutput, UtxoEntry, + ScriptPublicKey, ScriptVec, TransactionId, TransactionIndexType, TransactionInput, TransactionOutpoint, TransactionOutput, + UtxoEntry, }; +use spectre_utils::serde_bytes_fixed_ref; +use workflow_serializer::prelude::*; use crate::prelude::{RpcHash, RpcScriptClass, RpcSubnetworkId}; @@ -12,13 +15,123 @@ pub type RpcTransactionId = TransactionId; pub type RpcScriptVec = ScriptVec; pub type RpcScriptPublicKey = ScriptPublicKey; -pub type RpcUtxoEntry = UtxoEntry; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcUtxoEntry { + pub amount: u64, + pub script_public_key: ScriptPublicKey, + pub block_daa_score: u64, + pub is_coinbase: bool, +} + +impl RpcUtxoEntry { + pub fn new(amount: u64, script_public_key: ScriptPublicKey, block_daa_score: u64, is_coinbase: bool) -> Self { + Self { amount, script_public_key, block_daa_score, is_coinbase } + } +} + +impl From for RpcUtxoEntry { + fn from(entry: UtxoEntry) -> Self { + Self { + amount: entry.amount, + script_public_key: entry.script_public_key, + block_daa_score: entry.block_daa_score, + is_coinbase: entry.is_coinbase, + } + } +} + +impl From for UtxoEntry { + fn from(entry: RpcUtxoEntry) -> Self { + Self { + amount: entry.amount, + script_public_key: entry.script_public_key, + block_daa_score: entry.block_daa_score, + is_coinbase: entry.is_coinbase, + } + } +} + +impl Serializer for RpcUtxoEntry { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(u64, &self.amount, writer)?; + store!(ScriptPublicKey, &self.script_public_key, writer)?; + store!(u64, &self.block_daa_score, writer)?; + store!(bool, &self.is_coinbase, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcUtxoEntry { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let amount = load!(u64, reader)?; + let script_public_key = load!(ScriptPublicKey, reader)?; + let block_daa_score = load!(u64, reader)?; + let is_coinbase = load!(bool, reader)?; + + Ok(Self { amount, script_public_key, block_daa_score, is_coinbase }) + } +} /// Represents a Spectre transaction outpoint -pub type RpcTransactionOutpoint = TransactionOutpoint; +#[derive(Eq, Hash, PartialEq, Debug, Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct RpcTransactionOutpoint { + #[serde(with = "serde_bytes_fixed_ref")] + pub transaction_id: TransactionId, + pub index: TransactionIndexType, +} + +impl From for RpcTransactionOutpoint { + fn from(outpoint: TransactionOutpoint) -> Self { + Self { transaction_id: outpoint.transaction_id, index: outpoint.index } + } +} + +impl From for TransactionOutpoint { + fn from(outpoint: RpcTransactionOutpoint) -> Self { + Self { transaction_id: outpoint.transaction_id, index: outpoint.index } + } +} + +impl From for RpcTransactionOutpoint { + fn from(outpoint: spectre_consensus_client::TransactionOutpoint) -> Self { + TransactionOutpoint::from(outpoint).into() + } +} + +impl From for spectre_consensus_client::TransactionOutpoint { + fn from(outpoint: RpcTransactionOutpoint) -> Self { + TransactionOutpoint::from(outpoint).into() + } +} + +impl Serializer for RpcTransactionOutpoint { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(TransactionId, &self.transaction_id, writer)?; + store!(TransactionIndexType, &self.index, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransactionOutpoint { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let transaction_id = load!(TransactionId, reader)?; + let index = load!(TransactionIndexType, reader)?; + + Ok(Self { transaction_id, index }) + } +} /// Represents a Spectre transaction input -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransactionInput { pub previous_outpoint: RpcTransactionOutpoint, @@ -32,7 +145,7 @@ pub struct RpcTransactionInput { impl From for RpcTransactionInput { fn from(input: TransactionInput) -> Self { Self { - previous_outpoint: input.previous_outpoint, + previous_outpoint: input.previous_outpoint.into(), signature_script: input.signature_script, sequence: input.sequence, sig_op_count: input.sig_op_count, @@ -47,13 +160,53 @@ impl RpcTransactionInput { } } +impl Serializer for RpcTransactionInput { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + serialize!(RpcTransactionOutpoint, &self.previous_outpoint, writer)?; + store!(Vec, &self.signature_script, writer)?; + store!(u64, &self.sequence, writer)?; + store!(u8, &self.sig_op_count, writer)?; + serialize!(Option, &self.verbose_data, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransactionInput { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let previous_outpoint = deserialize!(RpcTransactionOutpoint, reader)?; + let signature_script = load!(Vec, reader)?; + let sequence = load!(u64, reader)?; + let sig_op_count = load!(u8, reader)?; + let verbose_data = deserialize!(Option, reader)?; + + Ok(Self { previous_outpoint, signature_script, sequence, sig_op_count, verbose_data }) + } +} + /// Represent Spectre transaction input verbose data -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransactionInputVerboseData {} +impl Serializer for RpcTransactionInputVerboseData { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + Ok(()) + } +} + +impl Deserializer for RpcTransactionInputVerboseData { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + Ok(Self {}) + } +} + /// Represents a Spectred transaction output -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransactionOutput { pub value: u64, @@ -73,16 +226,58 @@ impl From for RpcTransactionOutput { } } +impl Serializer for RpcTransactionOutput { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(u64, &self.value, writer)?; + store!(RpcScriptPublicKey, &self.script_public_key, writer)?; + serialize!(Option, &self.verbose_data, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransactionOutput { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let value = load!(u64, reader)?; + let script_public_key = load!(RpcScriptPublicKey, reader)?; + let verbose_data = deserialize!(Option, reader)?; + + Ok(Self { value, script_public_key, verbose_data }) + } +} + /// Represent Spectre transaction output verbose data -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransactionOutputVerboseData { pub script_public_key_type: RpcScriptClass, pub script_public_key_address: Address, } +impl Serializer for RpcTransactionOutputVerboseData { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(RpcScriptClass, &self.script_public_key_type, writer)?; + store!(Address, &self.script_public_key_address, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransactionOutputVerboseData { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let script_public_key_type = load!(RpcScriptClass, reader)?; + let script_public_key_address = load!(Address, reader)?; + + Ok(Self { script_public_key_type, script_public_key_address }) + } +} + /// Represents a Spectre transaction -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransaction { pub version: u16, @@ -97,8 +292,42 @@ pub struct RpcTransaction { pub verbose_data: Option, } +impl Serializer for RpcTransaction { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u16, &1, writer)?; + store!(u16, &self.version, writer)?; + serialize!(Vec, &self.inputs, writer)?; + serialize!(Vec, &self.outputs, writer)?; + store!(u64, &self.lock_time, writer)?; + store!(RpcSubnetworkId, &self.subnetwork_id, writer)?; + store!(u64, &self.gas, writer)?; + store!(Vec, &self.payload, writer)?; + store!(u64, &self.mass, writer)?; + serialize!(Option, &self.verbose_data, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransaction { + fn deserialize(reader: &mut R) -> std::io::Result { + let _struct_version = load!(u16, reader)?; + let version = load!(u16, reader)?; + let inputs = deserialize!(Vec, reader)?; + let outputs = deserialize!(Vec, reader)?; + let lock_time = load!(u64, reader)?; + let subnetwork_id = load!(RpcSubnetworkId, reader)?; + let gas = load!(u64, reader)?; + let payload = load!(Vec, reader)?; + let mass = load!(u64, reader)?; + let verbose_data = deserialize!(Option, reader)?; + + Ok(Self { version, inputs, outputs, lock_time, subnetwork_id, gas, payload, mass, verbose_data }) + } +} + /// Represent Spectre transaction verbose data -#[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct RpcTransactionVerboseData { pub transaction_id: RpcTransactionId, @@ -108,6 +337,32 @@ pub struct RpcTransactionVerboseData { pub block_time: u64, } +impl Serializer for RpcTransactionVerboseData { + fn serialize(&self, writer: &mut W) -> std::io::Result<()> { + store!(u8, &1, writer)?; + store!(RpcTransactionId, &self.transaction_id, writer)?; + store!(RpcHash, &self.hash, writer)?; + store!(u64, &self.mass, writer)?; + store!(RpcHash, &self.block_hash, writer)?; + store!(u64, &self.block_time, writer)?; + + Ok(()) + } +} + +impl Deserializer for RpcTransactionVerboseData { + fn deserialize(reader: &mut R) -> std::io::Result { + let _version = load!(u8, reader)?; + let transaction_id = load!(RpcTransactionId, reader)?; + let hash = load!(RpcHash, reader)?; + let mass = load!(u64, reader)?; + let block_hash = load!(RpcHash, reader)?; + let block_time = load!(u64, reader)?; + + Ok(Self { transaction_id, hash, mass, block_hash, block_time }) + } +} + /// Represents accepted transaction ids #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "camelCase")] diff --git a/rpc/core/src/wasm/convert.rs b/rpc/core/src/wasm/convert.rs index e621d68..03e4532 100644 --- a/rpc/core/src/wasm/convert.rs +++ b/rpc/core/src/wasm/convert.rs @@ -1,12 +1,11 @@ use crate::model::*; use spectre_consensus_client::*; -use spectre_consensus_core::tx as cctx; use std::sync::Arc; impl From for UtxoEntry { fn from(entry: RpcUtxosByAddressesEntry) -> UtxoEntry { let RpcUtxosByAddressesEntry { address, outpoint, utxo_entry } = entry; - let cctx::UtxoEntry { amount, script_public_key, block_daa_score, is_coinbase } = utxo_entry; + let RpcUtxoEntry { amount, script_public_key, block_daa_score, is_coinbase } = utxo_entry; UtxoEntry { address, outpoint: outpoint.into(), amount, script_public_key, block_daa_score, is_coinbase } } } @@ -31,7 +30,7 @@ cfg_if::cfg_if! { let inner = tx_input.inner(); RpcTransactionInput { previous_outpoint: inner.previous_outpoint.clone().into(), - signature_script: inner.signature_script.clone(), + signature_script: inner.signature_script.clone().unwrap_or_default(), sequence: inner.sequence, sig_op_count: inner.sig_op_count, verbose_data: None, diff --git a/rpc/core/src/wasm/message.rs b/rpc/core/src/wasm/message.rs index 7dfc327..aa03402 100644 --- a/rpc/core/src/wasm/message.rs +++ b/rpc/core/src/wasm/message.rs @@ -1,5 +1,4 @@ #![allow(non_snake_case)] - use crate::error::RpcError as Error; use crate::error::RpcResult as Result; use crate::model::*; @@ -8,6 +7,7 @@ use spectre_addresses::Address; use spectre_addresses::AddressOrStringArrayT; use spectre_consensus_client::Transaction; use spectre_consensus_client::UtxoEntryReference; +use spectre_consensus_core::tx as cctx; use spectre_rpc_macros::declare_typescript_wasm_interface as declare; use wasm_bindgen::prelude::*; use workflow_wasm::convert::*; @@ -317,6 +317,38 @@ try_from! ( args: GetMetricsResponse, IGetMetricsResponse, { // --- +declare! { + IGetConnectionsRequest, + r#" + /** + * @category Node RPC + */ + export interface IGetConnectionsRequest { } + "#, +} + +try_from! ( args: IGetConnectionsRequest, GetConnectionsRequest, { + Ok(from_value(args.into())?) +}); + +declare! { + IGetConnectionsResponse, + r#" + /** + * @category Node RPC + */ + export interface IGetConnectionsResponse { + [key: string]: any + } + "#, +} + +try_from! ( args: GetConnectionsResponse, IGetConnectionsResponse, { + Ok(to_value(&args)?.into()) +}); + +// --- + declare! { IGetSinkRequest, r#" @@ -788,7 +820,7 @@ declare! { } try_from! ( args: IGetBlockTemplateRequest, GetBlockTemplateRequest, { - let pay_address = args.get_cast::
("payAddress")?.into_owned(); + let pay_address = args.cast_into::
("payAddress")?; let extra_data = if let Some(extra_data) = args.try_get_value("extraData")? { if let Some(text) = extra_data.as_string() { text.into_bytes() @@ -1129,7 +1161,7 @@ declare! { * @category Node RPC */ export interface IGetUtxosByAddressesResponse { - entries : IUtxoEntry[]; + entries : UtxoEntryReference[]; } "#, } @@ -1292,6 +1324,66 @@ try_from! ( args: SubmitBlockResponse, ISubmitBlockResponse, { // --- +declare! { + ISubmitTransactionReplacementRequest, + // "ISubmitTransactionRequest | Transaction", + r#" + /** + * Submit transaction replacement to the node. + * + * @category Node RPC + */ + export interface ISubmitTransactionReplacementRequest { + transaction : Transaction, + } + "#, +} + +try_from! ( args: ISubmitTransactionReplacementRequest, SubmitTransactionReplacementRequest, { + let transaction = if let Some(transaction) = args.try_get_value("transaction")? { + transaction + } else { + args.into() + }; + + let request = if let Ok(transaction) = Transaction::try_owned_from(&transaction) { + SubmitTransactionReplacementRequest { + transaction : transaction.into(), + } + } else { + from_value(transaction)? + }; + Ok(request) +}); + +declare! { + ISubmitTransactionReplacementResponse, + r#" + /** + * + * + * @category Node RPC + */ + export interface ISubmitTransactionReplacementResponse { + transactionId : HexString; + replacedTransaction: Transaction; + } + "#, +} + +try_from! ( args: SubmitTransactionReplacementResponse, ISubmitTransactionReplacementResponse, { + let transaction_id = args.transaction_id; + let replaced_transaction = cctx::Transaction::try_from(args.replaced_transaction)?; + let replaced_transaction = Transaction::from(replaced_transaction); + + let response = ISubmitTransactionReplacementResponse::default(); + response.set("transactionId", &transaction_id.into())?; + response.set("replacedTransaction", &replaced_transaction.into())?; + Ok(response) +}); + +// --- + declare! { ISubmitTransactionRequest, // "ISubmitTransactionRequest | Transaction", @@ -1322,7 +1414,11 @@ try_from! ( args: ISubmitTransactionRequest, SubmitTransactionRequest, { allow_orphan, } } else { - from_value(transaction)? + let tx = Transaction::try_cast_from(&transaction)?; + SubmitTransactionRequest { + transaction : tx.as_ref().into(), + allow_orphan, + } }; Ok(request) }); @@ -1383,3 +1479,210 @@ declare! { try_from! ( args: UnbanResponse, IUnbanResponse, { Ok(to_value(&args)?.into()) }); + +// --- + +declare! { + IFeerateBucket, + r#" + /** + * + * + * @category Node RPC + */ + export interface IFeerateBucket { + /** + * The fee/mass ratio estimated to be required for inclusion time <= estimated_seconds + */ + feerate : number; + /** + * The estimated inclusion time for a transaction with fee/mass = feerate + */ + estimatedSeconds : number; + } + "#, +} + +declare! { + IFeeEstimate, + r#" + /** + * + * + * @category Node RPC + */ + export interface IFeeEstimate { + /** + * *Top-priority* feerate bucket. Provides an estimation of the feerate required for sub-second DAG inclusion. + * + * Note: for all buckets, feerate values represent fee/mass of a transaction in `sompi/gram` units. + * Given a feerate value recommendation, calculate the required fee by + * taking the transaction mass and multiplying it by feerate: `fee = feerate * mass(tx)` + */ + + priorityBucket : IFeerateBucket; + /** + * A vector of *normal* priority feerate values. The first value of this vector is guaranteed to exist and + * provide an estimation for sub-*minute* DAG inclusion. All other values will have shorter estimation + * times than all `low_bucket` values. Therefor by chaining `[priority] | normal | low` and interpolating + * between them, one can compose a complete feerate function on the client side. The API makes an effort + * to sample enough "interesting" points on the feerate-to-time curve, so that the interpolation is meaningful. + */ + + normalBuckets : IFeerateBucket[]; + /** + * An array of *low* priority feerate values. The first value of this vector is guaranteed to + * exist and provide an estimation for sub-*hour* DAG inclusion. + */ + lowBuckets : IFeerateBucket[]; + } + "#, +} + +try_from!( estimate: RpcFeeEstimate, IFeeEstimate, { + + let priority_bucket = IFeerateBucket::default(); + priority_bucket.set("feerate", &estimate.priority_bucket.feerate.into())?; + priority_bucket.set("estimatedSeconds", &estimate.priority_bucket.estimated_seconds.into())?; + + let normal_buckets = estimate.normal_buckets.into_iter().map(|normal_bucket| { + let bucket = IFeerateBucket::default(); + bucket.set("feerate", &normal_bucket.feerate.into())?; + bucket.set("estimatedSeconds", &normal_bucket.estimated_seconds.into())?; + Ok(bucket) + }).collect::>>()?; + + let low_buckets = estimate.low_buckets.into_iter().map(|low_bucket| { + let bucket = IFeerateBucket::default(); + bucket.set("feerate", &low_bucket.feerate.into())?; + bucket.set("estimatedSeconds", &low_bucket.estimated_seconds.into())?; + Ok(bucket) + }).collect::>>()?; + + let estimate = IFeeEstimate::default(); + estimate.set("priorityBucket", &priority_bucket)?; + estimate.set("normalBuckets", &js_sys::Array::from_iter(normal_buckets))?; + estimate.set("lowBuckets", &js_sys::Array::from_iter(low_buckets))?; + + Ok(estimate) +}); + +// --- + +declare! { + IGetFeeEstimateRequest, + r#" + /** + * Get fee estimate from the node. + * + * @category Node RPC + */ + export interface IGetFeeEstimateRequest { } + "#, +} + +try_from! ( args: IGetFeeEstimateRequest, GetFeeEstimateRequest, { + Ok(from_value(args.into())?) +}); + +declare! { + IGetFeeEstimateResponse, + r#" + /** + * + * + * @category Node RPC + */ + export interface IGetFeeEstimateResponse { + estimate : IFeeEstimate; + } + "#, +} + +try_from!( args: GetFeeEstimateResponse, IGetFeeEstimateResponse, { + let estimate = IFeeEstimate::try_from(args.estimate)?; + let response = IGetFeeEstimateResponse::default(); + response.set("estimate", &estimate)?; + Ok(response) +}); + +// --- + +declare! { + IFeeEstimateVerboseExperimentalData, + r#" + /** + * + * + * @category Node RPC + */ + export interface IFeeEstimateVerboseExperimentalData { + mempoolReadyTransactionsCount : bigint; + mempoolReadyTransactionsTotalMass : bigint; + networkMassPerSecond : bigint; + nextBlockTemplateFeerateMin : number; + nextBlockTemplateFeerateMedian : number; + nextBlockTemplateFeerateMax : number; + } + "#, +} + +try_from!( data: RpcFeeEstimateVerboseExperimentalData, IFeeEstimateVerboseExperimentalData, { + + let target = IFeeEstimateVerboseExperimentalData::default(); + target.set("mempoolReadyTransactionsCount", &js_sys::BigInt::from(data.mempool_ready_transactions_count).into())?; + target.set("mempoolReadyTransactionsTotalMass", &js_sys::BigInt::from(data.mempool_ready_transactions_total_mass).into())?; + target.set("networkMassPerSecond", &js_sys::BigInt::from(data.network_mass_per_second).into())?; + target.set("nextBlockTemplateFeerateMin", &data.next_block_template_feerate_min.into())?; + target.set("nextBlockTemplateFeerateMedian", &data.next_block_template_feerate_median.into())?; + target.set("nextBlockTemplateFeerateMax", &data.next_block_template_feerate_max.into())?; + + Ok(target) +}); + +declare! { + IGetFeeEstimateExperimentalRequest, + // "ISubmitTransactionRequest | Transaction", + r#" + /** + * Get fee estimate from the node. + * + * @category Node RPC + */ + export interface IGetFeeEstimateExperimentalRequest { } + "#, +} + +try_from! ( args: IGetFeeEstimateExperimentalRequest, GetFeeEstimateExperimentalRequest, { + Ok(from_value(args.into())?) +}); + +declare! { + IGetFeeEstimateExperimentalResponse, + r#" + /** + * + * + * @category Node RPC + */ + export interface IGetFeeEstimateExperimentalResponse { + estimate : IFeeEstimate; + verbose? : IFeeEstimateVerboseExperimentalData + } + "#, +} + +try_from!( args: GetFeeEstimateExperimentalResponse, IGetFeeEstimateExperimentalResponse, { + let estimate = IFeeEstimate::try_from(args.estimate)?; + let response = IGetFeeEstimateExperimentalResponse::default(); + response.set("estimate", &estimate)?; + + if let Some(verbose) = args.verbose { + let verbose = IFeeEstimateVerboseExperimentalData::try_from(verbose)?; + response.set("verbose", &verbose)?; + } + + Ok(response) +}); + +// --- diff --git a/rpc/grpc/client/src/lib.rs b/rpc/grpc/client/src/lib.rs index d2d38a8..cb2d985 100644 --- a/rpc/grpc/client/src/lib.rs +++ b/rpc/grpc/client/src/lib.rs @@ -241,6 +241,8 @@ impl RpcApi for GrpcClient { route!(get_sync_status_call, GetSyncStatus); route!(get_server_info_call, GetServerInfo); route!(get_metrics_call, GetMetrics); + route!(get_connections_call, GetConnections); + route!(get_system_info_call, GetSystemInfo); route!(submit_block_call, SubmitBlock); route!(get_block_template_call, GetBlockTemplate); route!(get_block_call, GetBlock); diff --git a/rpc/grpc/client/src/route.rs b/rpc/grpc/client/src/route.rs index 0449f21..f7edb52 100644 --- a/rpc/grpc/client/src/route.rs +++ b/rpc/grpc/client/src/route.rs @@ -9,12 +9,14 @@ macro_rules! route { clippy::type_repetition_in_bounds, clippy::used_underscore_binding )] - fn $fn<'life0, 'async_trait>( + fn $fn<'life0, 'life1, 'async_trait>( &'life0 self, + _connection : ::core::option::Option<&'life1 Arc>, request: [<$name Request>], ) -> ::core::pin::Pin]>> + ::core::marker::Send + 'async_trait>> where 'life0: 'async_trait, + 'life1: 'async_trait, Self: 'async_trait, { Box::pin(async move { diff --git a/rpc/grpc/core/proto/messages.proto b/rpc/grpc/core/proto/messages.proto index 230c625..e070b05 100644 --- a/rpc/grpc/core/proto/messages.proto +++ b/rpc/grpc/core/proto/messages.proto @@ -59,7 +59,9 @@ message SpectredRequest { GetServerInfoRequestMessage getServerInfoRequest = 1092; GetSyncStatusRequestMessage getSyncStatusRequest = 1094; GetDaaScoreTimestampEstimateRequestMessage getDaaScoreTimestampEstimateRequest = 1096; - SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 2000; + SubmitTransactionReplacementRequestMessage submitTransactionReplacementRequest = 1100; + GetConnectionsRequestMessage getConnectionsRequest = 1102; + GetSystemInfoRequestMessage getSystemInfoRequest = 1104; GetFeeEstimateRequestMessage getFeeEstimateRequest = 1106; GetFeeEstimateExperimentalRequestMessage getFeeEstimateExperimentalRequest = 1108; } @@ -121,7 +123,9 @@ message SpectredResponse { GetServerInfoResponseMessage getServerInfoResponse = 1093; GetSyncStatusResponseMessage getSyncStatusResponse = 1095; GetDaaScoreTimestampEstimateResponseMessage getDaaScoreTimestampEstimateResponse = 1097; - SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 2001; + SubmitTransactionReplacementResponseMessage submitTransactionReplacementResponse = 1101; + GetConnectionsResponseMessage getConnectionsResponse= 1103; + GetSystemInfoResponseMessage getSystemInfoResponse= 1105; GetFeeEstimateResponseMessage getFeeEstimateResponse = 1107; GetFeeEstimateExperimentalResponseMessage getFeeEstimateExperimentalResponse = 1109; } diff --git a/rpc/grpc/core/proto/rpc.proto b/rpc/grpc/core/proto/rpc.proto index 0570dde..ce1e7c9 100644 --- a/rpc/grpc/core/proto/rpc.proto +++ b/rpc/grpc/core/proto/rpc.proto @@ -821,11 +821,46 @@ message ConsensusMetrics{ uint64 virtualDaaScore = 18; } +message StorageMetrics{ + uint64 storageSizeBytes = 1; +} + +message GetConnectionsRequestMessage{ + bool includeProfileData = 1; +} + +message ConnectionsProfileData { + double cpuUsage = 1; + uint64 memoryUsage = 2; +} + +message GetConnectionsResponseMessage{ + uint32 clients = 1; + uint32 peers = 2; + ConnectionsProfileData profileData = 3; + RPCError error = 1000; +} + +message GetSystemInfoRequestMessage{ +} + +message GetSystemInfoResponseMessage{ + string version = 1; + string systemId = 2; + string gitHash = 3; + uint32 coreNum = 4; + uint64 totalMemory = 5; + uint32 fdLimit = 6; + RPCError error = 1000; +} + message GetMetricsRequestMessage{ bool processMetrics = 1; bool connectionMetrics = 2; bool bandwidthMetrics = 3; bool consensusMetrics = 4; + bool storageMetrics = 5; + bool customMetrics = 6; } message GetMetricsResponseMessage{ @@ -834,6 +869,7 @@ message GetMetricsResponseMessage{ ConnectionMetrics connectionMetrics = 12; BandwidthMetrics bandwidthMetrics = 13; ConsensusMetrics consensusMetrics = 14; + StorageMetrics storageMetrics = 15; RPCError error = 1000; } @@ -841,12 +877,13 @@ message GetServerInfoRequestMessage{ } message GetServerInfoResponseMessage{ - repeated uint32 rpcApiVersion = 1; // Expecting exactly 4 elements - string serverVersion = 2; - string networkId = 3; - bool hasUtxoIndex = 4; - bool isSynced = 5; - uint64 virtualDaaScore = 6; + uint32 rpcApiVersion = 1; + uint32 rpcApiRevision = 2; + string serverVersion = 3; + string networkId = 4; + bool hasUtxoIndex = 5; + bool isSynced = 6; + uint64 virtualDaaScore = 7; RPCError error = 1000; } diff --git a/rpc/grpc/core/src/convert/block.rs b/rpc/grpc/core/src/convert/block.rs index 612da80..7e017d5 100644 --- a/rpc/grpc/core/src/convert/block.rs +++ b/rpc/grpc/core/src/convert/block.rs @@ -15,6 +15,14 @@ from!(item: &spectre_rpc_core::RpcBlock, protowire::RpcBlock, { } }); +from!(item: &spectre_rpc_core::RpcRawBlock, protowire::RpcBlock, { + Self { + header: Some(protowire::RpcBlockHeader::from(&item.header)), + transactions: item.transactions.iter().map(protowire::RpcTransaction::from).collect(), + verbose_data: None, + } +}); + from!(item: &spectre_rpc_core::RpcBlockVerboseData, protowire::RpcBlockVerboseData, { Self { hash: item.hash.to_string(), @@ -46,6 +54,17 @@ try_from!(item: &protowire::RpcBlock, spectre_rpc_core::RpcBlock, { } }); +try_from!(item: &protowire::RpcBlock, spectre_rpc_core::RpcRawBlock, { + Self { + header: item + .header + .as_ref() + .ok_or_else(|| RpcError::MissingRpcFieldError("RpcBlock".to_string(), "header".to_string()))? + .try_into()?, + transactions: item.transactions.iter().map(spectre_rpc_core::RpcTransaction::try_from).collect::, _>>()?, + } +}); + try_from!(item: &protowire::RpcBlockVerboseData, spectre_rpc_core::RpcBlockVerboseData, { Self { hash: RpcHash::from_str(&item.hash)?, diff --git a/rpc/grpc/core/src/convert/header.rs b/rpc/grpc/core/src/convert/header.rs index f4229f4..d70aeda 100644 --- a/rpc/grpc/core/src/convert/header.rs +++ b/rpc/grpc/core/src/convert/header.rs @@ -1,5 +1,6 @@ use crate::protowire; use crate::{from, try_from}; +use spectre_consensus_core::header::Header; use spectre_rpc_core::{FromRpcHex, RpcError, RpcHash, RpcResult, ToRpcHex}; use std::str::FromStr; @@ -24,6 +25,23 @@ from!(item: &spectre_rpc_core::RpcHeader, protowire::RpcBlockHeader, { } }); +from!(item: &spectre_rpc_core::RpcRawHeader, protowire::RpcBlockHeader, { + Self { + version: item.version.into(), + parents: item.parents_by_level.iter().map(protowire::RpcBlockLevelParents::from).collect(), + hash_merkle_root: item.hash_merkle_root.to_string(), + accepted_id_merkle_root: item.accepted_id_merkle_root.to_string(), + utxo_commitment: item.utxo_commitment.to_string(), + timestamp: item.timestamp.try_into().expect("timestamp is always convertible to i64"), + bits: item.bits, + nonce: item.nonce, + daa_score: item.daa_score, + blue_work: item.blue_work.to_rpc_hex(), + blue_score: item.blue_score, + pruning_point: item.pruning_point.to_string(), + } +}); + from!(item: &Vec, protowire::RpcBlockLevelParents, { Self { parent_hashes: item.iter().map(|x| x.to_string()).collect() } }); // ---------------------------------------------------------------------------- @@ -32,7 +50,7 @@ from!(item: &Vec, protowire::RpcBlockLevelParents, { Self { parent_hash try_from!(item: &protowire::RpcBlockHeader, spectre_rpc_core::RpcHeader, { // We re-hash the block to remain as most trustless as possible - Self::new_finalized( + let header = Header::new_finalized( item.version.try_into()?, item.parents.iter().map(Vec::::try_from).collect::>>>()?, RpcHash::from_str(&item.hash_merkle_root)?, @@ -45,7 +63,26 @@ try_from!(item: &protowire::RpcBlockHeader, spectre_rpc_core::RpcHeader, { spectre_rpc_core::RpcBlueWorkType::from_rpc_hex(&item.blue_work)?, item.blue_score, RpcHash::from_str(&item.pruning_point)?, - ) + ); + + header.into() +}); + +try_from!(item: &protowire::RpcBlockHeader, spectre_rpc_core::RpcRawHeader, { + Self { + version: item.version.try_into()?, + parents_by_level: item.parents.iter().map(Vec::::try_from).collect::>>>()?, + hash_merkle_root: RpcHash::from_str(&item.hash_merkle_root)?, + accepted_id_merkle_root: RpcHash::from_str(&item.accepted_id_merkle_root)?, + utxo_commitment: RpcHash::from_str(&item.utxo_commitment)?, + timestamp: item.timestamp.try_into()?, + bits: item.bits, + nonce: item.nonce, + daa_score: item.daa_score, + blue_work: spectre_rpc_core::RpcBlueWorkType::from_rpc_hex(&item.blue_work)?, + blue_score: item.blue_score, + pruning_point: RpcHash::from_str(&item.pruning_point)?, + } }); try_from!(item: &protowire::RpcBlockLevelParents, Vec, { @@ -55,7 +92,8 @@ try_from!(item: &protowire::RpcBlockLevelParents, Vec, { #[cfg(test)] mod tests { use crate::protowire; - use spectre_rpc_core::{RpcHash, RpcHeader}; + use spectre_consensus_core::{block::Block, header::Header}; + use spectre_rpc_core::{RpcBlock, RpcHash, RpcHeader}; fn new_unique() -> RpcHash { use std::sync::atomic::{AtomicU64, Ordering}; @@ -106,7 +144,7 @@ mod tests { #[test] fn test_rpc_header() { - let r = RpcHeader::new_finalized( + let r = Header::new_finalized( 0, vec![vec![new_unique(), new_unique(), new_unique()], vec![new_unique()], vec![new_unique(), new_unique()]], new_unique(), @@ -120,6 +158,7 @@ mod tests { 1928374, new_unique(), ); + let r = RpcHeader::from(r); let p: protowire::RpcBlockHeader = (&r).into(); let r2: RpcHeader = (&p).try_into().unwrap(); let p2: protowire::RpcBlockHeader = (&r2).into(); @@ -134,4 +173,42 @@ mod tests { assert_eq!(r.hash, r2.hash); assert_eq!(p, p2); } + + #[test] + fn test_rpc_block() { + let h = Header::new_finalized( + 0, + vec![vec![new_unique(), new_unique(), new_unique()], vec![new_unique()], vec![new_unique(), new_unique()]], + new_unique(), + new_unique(), + new_unique(), + 123, + 12345, + 98765, + 120055, + 459912.into(), + 1928374, + new_unique(), + ); + let b = Block::from_header(h); + let r: RpcBlock = (&b).into(); + let p: protowire::RpcBlock = (&r).into(); + let r2: RpcBlock = (&p).try_into().unwrap(); + let b2: Block = r2.clone().try_into().unwrap(); + let r3: RpcBlock = (&b2).into(); + let p2: protowire::RpcBlock = (&r3).into(); + + assert_eq!(r.header.parents_by_level, r2.header.parents_by_level); + assert_eq!(p.header.as_ref().unwrap().parents, p2.header.as_ref().unwrap().parents); + test_parents_by_level_rxr(&r.header.parents_by_level, &r2.header.parents_by_level); + test_parents_by_level_rxr(&r.header.parents_by_level, &r3.header.parents_by_level); + test_parents_by_level_rxr(&b.header.parents_by_level, &r2.header.parents_by_level); + test_parents_by_level_rxr(&b.header.parents_by_level, &b2.header.parents_by_level); + test_parents_by_level_rxp(&r.header.parents_by_level, &p.header.as_ref().unwrap().parents); + test_parents_by_level_rxp(&r.header.parents_by_level, &p2.header.as_ref().unwrap().parents); + test_parents_by_level_rxp(&r2.header.parents_by_level, &p2.header.as_ref().unwrap().parents); + + assert_eq!(b.hash(), b2.hash()); + assert_eq!(p, p2); + } } diff --git a/rpc/grpc/core/src/convert/message.rs b/rpc/grpc/core/src/convert/message.rs index 990a421..e3de147 100644 --- a/rpc/grpc/core/src/convert/message.rs +++ b/rpc/grpc/core/src/convert/message.rs @@ -26,6 +26,7 @@ use spectre_rpc_core::{ RpcContextualPeerAddress, RpcError, RpcExtraData, RpcHash, RpcIpAddress, RpcNetworkType, RpcPeerAddress, RpcResult, SubmitBlockRejectReason, SubmitBlockReport, }; +use spectre_utils::hex::*; use std::str::FromStr; macro_rules! from { @@ -429,6 +430,8 @@ from!(item: &spectre_rpc_core::GetMetricsRequest, protowire::GetMetricsRequestMe connection_metrics: item.connection_metrics, bandwidth_metrics: item.bandwidth_metrics, consensus_metrics: item.consensus_metrics, + storage_metrics: item.storage_metrics, + custom_metrics: item.custom_metrics, } }); from!(item: RpcResult<&spectre_rpc_core::GetMetricsResponse>, protowire::GetMetricsResponseMessage, { @@ -438,13 +441,45 @@ from!(item: RpcResult<&spectre_rpc_core::GetMetricsResponse>, protowire::GetMetr connection_metrics: item.connection_metrics.as_ref().map(|x| x.into()), bandwidth_metrics: item.bandwidth_metrics.as_ref().map(|x| x.into()), consensus_metrics: item.consensus_metrics.as_ref().map(|x| x.into()), + storage_metrics: item.storage_metrics.as_ref().map(|x| x.into()), + // TODO + // custom_metrics : None, error: None, } }); + +from!(item: &spectre_rpc_core::GetConnectionsRequest, protowire::GetConnectionsRequestMessage, { + Self { + include_profile_data : item.include_profile_data, + } +}); +from!(item: RpcResult<&spectre_rpc_core::GetConnectionsResponse>, protowire::GetConnectionsResponseMessage, { + Self { + clients: item.clients, + peers: item.peers as u32, + profile_data: item.profile_data.as_ref().map(|x| x.into()), + error: None, + } +}); + +from!(&spectre_rpc_core::GetSystemInfoRequest, protowire::GetSystemInfoRequestMessage); +from!(item: RpcResult<&spectre_rpc_core::GetSystemInfoResponse>, protowire::GetSystemInfoResponseMessage, { + Self { + version : item.version.clone(), + system_id : item.system_id.as_ref().map(|system_id|system_id.to_hex()).unwrap_or_default(), + git_hash : item.git_hash.as_ref().map(|git_hash|git_hash.to_hex()).unwrap_or_default(), + total_memory : item.total_memory, + core_num : item.cpu_physical_cores as u32, + fd_limit : item.fd_limit, + error: None, + } +}); + from!(&spectre_rpc_core::GetServerInfoRequest, protowire::GetServerInfoRequestMessage); from!(item: RpcResult<&spectre_rpc_core::GetServerInfoResponse>, protowire::GetServerInfoResponseMessage, { Self { - rpc_api_version: item.rpc_api_version.iter().map(|x| *x as u32).collect(), + rpc_api_version: item.rpc_api_version as u32, + rpc_api_revision: item.rpc_api_revision as u32, server_version: item.server_version.clone(), network_id: item.network_id.to_string(), has_utxo_index: item.has_utxo_index, @@ -865,7 +900,14 @@ try_from!(&protowire::PingRequestMessage, spectre_rpc_core::PingRequest); try_from!(&protowire::PingResponseMessage, RpcResult); try_from!(item: &protowire::GetMetricsRequestMessage, spectre_rpc_core::GetMetricsRequest, { - Self { process_metrics: item.process_metrics, connection_metrics: item.connection_metrics, bandwidth_metrics:item.bandwidth_metrics, consensus_metrics: item.consensus_metrics } + Self { + process_metrics: item.process_metrics, + connection_metrics: item.connection_metrics, + bandwidth_metrics:item.bandwidth_metrics, + consensus_metrics: item.consensus_metrics, + storage_metrics: item.storage_metrics, + custom_metrics : item.custom_metrics, + } }); try_from!(item: &protowire::GetMetricsResponseMessage, RpcResult, { Self { @@ -874,13 +916,40 @@ try_from!(item: &protowire::GetMetricsResponseMessage, RpcResult, { + Self { + clients: item.clients, + peers: item.peers as u16, + profile_data: item.profile_data.as_ref().map(|x| x.try_into()).transpose()?, + } +}); + +try_from!(&protowire::GetSystemInfoRequestMessage, spectre_rpc_core::GetSystemInfoRequest); +try_from!(item: &protowire::GetSystemInfoResponseMessage, RpcResult, { + Self { + version: item.version.clone(), + system_id: (!item.system_id.is_empty()).then(|| FromHex::from_hex(&item.system_id)).transpose()?, + git_hash: (!item.git_hash.is_empty()).then(|| FromHex::from_hex(&item.git_hash)).transpose()?, + total_memory: item.total_memory, + cpu_physical_cores: item.core_num as u16, + fd_limit: item.fd_limit, } }); try_from!(&protowire::GetServerInfoRequestMessage, spectre_rpc_core::GetServerInfoRequest); try_from!(item: &protowire::GetServerInfoResponseMessage, RpcResult, { Self { - rpc_api_version: item.rpc_api_version.iter().map(|x| *x as u16).collect::>().as_slice().try_into().map_err(|_| RpcError::RpcApiVersionFormatError)?, + rpc_api_version: item.rpc_api_version as u16, + rpc_api_revision: item.rpc_api_revision as u16, server_version: item.server_version.clone(), network_id: NetworkId::from_str(&item.network_id)?, has_utxo_index: item.has_utxo_index, diff --git a/rpc/grpc/core/src/convert/metrics.rs b/rpc/grpc/core/src/convert/metrics.rs index 2e4a47e..bceec01 100644 --- a/rpc/grpc/core/src/convert/metrics.rs +++ b/rpc/grpc/core/src/convert/metrics.rs @@ -6,6 +6,14 @@ use spectre_rpc_core::RpcError; // rpc_core to protowire // ---------------------------------------------------------------------------- +from!(item: &spectre_rpc_core::ConnectionsProfileData, protowire::ConnectionsProfileData, { + Self { + cpu_usage: item.cpu_usage as f64, + memory_usage: item.memory_usage, + + } +}); + from!(item: &spectre_rpc_core::ProcessMetrics, protowire::ProcessMetrics, { Self { resident_set_size: item.resident_set_size, @@ -66,10 +74,20 @@ from!(item: &spectre_rpc_core::ConsensusMetrics, protowire::ConsensusMetrics, { } }); +from!(item: &spectre_rpc_core::StorageMetrics, protowire::StorageMetrics, { + Self { + storage_size_bytes: item.storage_size_bytes, + } +}); + // ---------------------------------------------------------------------------- // protowire to rpc_core // ---------------------------------------------------------------------------- +try_from!(item: &protowire::ConnectionsProfileData, spectre_rpc_core::ConnectionsProfileData, { + Self { cpu_usage : item.cpu_usage as f32, memory_usage : item.memory_usage } +}); + try_from!(item: &protowire::ProcessMetrics, spectre_rpc_core::ProcessMetrics, { Self { resident_set_size: item.resident_set_size, @@ -129,3 +147,9 @@ try_from!(item: &protowire::ConsensusMetrics, spectre_rpc_core::ConsensusMetrics network_virtual_daa_score: item.virtual_daa_score, } }); + +try_from!(item: &protowire::StorageMetrics, spectre_rpc_core::StorageMetrics, { + Self { + storage_size_bytes: item.storage_size_bytes, + } +}); diff --git a/rpc/grpc/core/src/convert/spectred.rs b/rpc/grpc/core/src/convert/spectred.rs index 85e0c5c..dcea9ae 100644 --- a/rpc/grpc/core/src/convert/spectred.rs +++ b/rpc/grpc/core/src/convert/spectred.rs @@ -55,6 +55,8 @@ pub mod spectred_request_convert { impl_into_spectred_request!(GetCoinSupply); impl_into_spectred_request!(Ping); impl_into_spectred_request!(GetMetrics); + impl_into_spectred_request!(GetConnections); + impl_into_spectred_request!(GetSystemInfo); impl_into_spectred_request!(GetServerInfo); impl_into_spectred_request!(GetSyncStatus); impl_into_spectred_request!(GetDaaScoreTimestampEstimate); @@ -189,6 +191,8 @@ pub mod spectred_response_convert { impl_into_spectred_response!(GetCoinSupply); impl_into_spectred_response!(Ping); impl_into_spectred_response!(GetMetrics); + impl_into_spectred_response!(GetConnections); + impl_into_spectred_response!(GetSystemInfo); impl_into_spectred_response!(GetServerInfo); impl_into_spectred_response!(GetSyncStatus); impl_into_spectred_response!(GetDaaScoreTimestampEstimate); diff --git a/rpc/grpc/core/src/ops.rs b/rpc/grpc/core/src/ops.rs index d4b5bfb..c17a4b1 100644 --- a/rpc/grpc/core/src/ops.rs +++ b/rpc/grpc/core/src/ops.rs @@ -79,6 +79,8 @@ pub enum SpectredPayloadOps { GetCoinSupply, Ping, GetMetrics, + GetConnections, + GetSystemInfo, GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, diff --git a/rpc/grpc/server/src/request_handler/factory.rs b/rpc/grpc/server/src/request_handler/factory.rs index 96d4101..d2f9343 100644 --- a/rpc/grpc/server/src/request_handler/factory.rs +++ b/rpc/grpc/server/src/request_handler/factory.rs @@ -73,6 +73,8 @@ impl Factory { GetCoinSupply, Ping, GetMetrics, + GetConnections, + GetSystemInfo, GetServerInfo, GetSyncStatus, GetDaaScoreTimestampEstimate, diff --git a/rpc/grpc/server/src/tests/rpc_core_mock.rs b/rpc/grpc/server/src/tests/rpc_core_mock.rs index 91efb2d..fcf81f2 100644 --- a/rpc/grpc/server/src/tests/rpc_core_mock.rs +++ b/rpc/grpc/server/src/tests/rpc_core_mock.rs @@ -6,7 +6,7 @@ use spectre_notify::notifier::{Notifier, Notify}; use spectre_notify::scope::Scope; use spectre_notify::subscription::context::SubscriptionContext; use spectre_notify::subscription::{MutationPolicies, UtxosChangedMutationPolicy}; -use spectre_rpc_core::{api::rpc::RpcApi, *}; +use spectre_rpc_core::{api::connection::DynRpcConnection, api::rpc::RpcApi, *}; use spectre_rpc_core::{notify::connection::ChannelConnection, RpcResult}; use std::sync::Arc; @@ -66,7 +66,7 @@ impl RpcCoreMock { #[async_trait] impl RpcApi for RpcCoreMock { // This fn needs to succeed while the client connects - async fn get_info_call(&self, _request: GetInfoRequest) -> RpcResult { + async fn get_info_call(&self, _connection: Option<&DynRpcConnection>, _request: GetInfoRequest) -> RpcResult { Ok(GetInfoResponse { p2p_id: "p2p-mock".to_string(), mempool_size: 1234, @@ -78,140 +78,237 @@ impl RpcApi for RpcCoreMock { }) } - async fn ping_call(&self, _request: PingRequest) -> RpcResult { + async fn ping_call(&self, _connection: Option<&DynRpcConnection>, _request: PingRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_metrics_call(&self, _request: GetMetricsRequest) -> RpcResult { + async fn get_metrics_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMetricsRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_server_info_call(&self, _request: GetServerInfoRequest) -> RpcResult { + async fn get_connections_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetConnectionsRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sync_status_call(&self, _request: GetSyncStatusRequest) -> RpcResult { + async fn get_system_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSystemInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_current_network_call(&self, _request: GetCurrentNetworkRequest) -> RpcResult { + async fn get_server_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetServerInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn submit_block_call(&self, _request: SubmitBlockRequest) -> RpcResult { + async fn get_sync_status_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSyncStatusRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_template_call(&self, _request: GetBlockTemplateRequest) -> RpcResult { + async fn get_current_network_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetCurrentNetworkRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_peer_addresses_call(&self, _request: GetPeerAddressesRequest) -> RpcResult { + async fn submit_block_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: SubmitBlockRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sink_call(&self, _request: GetSinkRequest) -> RpcResult { + async fn get_block_template_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockTemplateRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_mempool_entry_call(&self, _request: GetMempoolEntryRequest) -> RpcResult { + async fn get_peer_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetPeerAddressesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_mempool_entries_call(&self, _request: GetMempoolEntriesRequest) -> RpcResult { + async fn get_sink_call(&self, _connection: Option<&DynRpcConnection>, _request: GetSinkRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_connected_peer_info_call(&self, _request: GetConnectedPeerInfoRequest) -> RpcResult { + async fn get_mempool_entry_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMempoolEntryRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn add_peer_call(&self, _request: AddPeerRequest) -> RpcResult { + async fn get_mempool_entries_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMempoolEntriesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn submit_transaction_call(&self, _request: SubmitTransactionRequest) -> RpcResult { + async fn get_connected_peer_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetConnectedPeerInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn submit_transaction_replacement_call( &self, + _connection: Option<&DynRpcConnection>, _request: SubmitTransactionReplacementRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { + async fn add_peer_call(&self, _connection: Option<&DynRpcConnection>, _request: AddPeerRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_subnetwork_call(&self, _request: GetSubnetworkRequest) -> RpcResult { + async fn submit_transaction_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: SubmitTransactionRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_block_call(&self, _connection: Option<&DynRpcConnection>, _request: GetBlockRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_subnetwork_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSubnetworkRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_virtual_chain_from_block_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetVirtualChainFromBlockRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_blocks_call(&self, _request: GetBlocksRequest) -> RpcResult { + async fn get_blocks_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlocksRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_count_call(&self, _request: GetBlockCountRequest) -> RpcResult { + async fn get_block_count_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockCountRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_dag_info_call(&self, _request: GetBlockDagInfoRequest) -> RpcResult { + async fn get_block_dag_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockDagInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn resolve_finality_conflict_call( &self, + _connection: Option<&DynRpcConnection>, _request: ResolveFinalityConflictRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn shutdown_call(&self, _request: ShutdownRequest) -> RpcResult { + async fn shutdown_call(&self, _connection: Option<&DynRpcConnection>, _request: ShutdownRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_headers_call(&self, _request: GetHeadersRequest) -> RpcResult { + async fn get_headers_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetHeadersRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_balance_by_address_call(&self, _request: GetBalanceByAddressRequest) -> RpcResult { + async fn get_balance_by_address_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBalanceByAddressRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_balances_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetBalancesByAddressesRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_utxos_by_addresses_call(&self, _request: GetUtxosByAddressesRequest) -> RpcResult { + async fn get_utxos_by_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetUtxosByAddressesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sink_blue_score_call(&self, _request: GetSinkBlueScoreRequest) -> RpcResult { + async fn get_sink_blue_score_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSinkBlueScoreRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn ban_call(&self, _request: BanRequest) -> RpcResult { + async fn ban_call(&self, _connection: Option<&DynRpcConnection>, _request: BanRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn unban_call(&self, _request: UnbanRequest) -> RpcResult { + async fn unban_call(&self, _connection: Option<&DynRpcConnection>, _request: UnbanRequest) -> RpcResult { Err(RpcError::NotImplemented) } async fn estimate_network_hashes_per_second_call( &self, + _connection: Option<&DynRpcConnection>, _request: EstimateNetworkHashesPerSecondRequest, ) -> RpcResult { Err(RpcError::NotImplemented) @@ -219,28 +316,39 @@ impl RpcApi for RpcCoreMock { async fn get_mempool_entries_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetMempoolEntriesByAddressesRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_coin_supply_call(&self, _request: GetCoinSupplyRequest) -> RpcResult { + async fn get_coin_supply_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetCoinSupplyRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_daa_score_timestamp_estimate_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + async fn get_fee_estimate_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetFeeEstimateRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_fee_estimate_experimental_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetFeeEstimateExperimentalRequest, ) -> RpcResult { Err(RpcError::NotImplemented) diff --git a/rpc/macros/src/grpc/server.rs b/rpc/macros/src/grpc/server.rs index b444c16..ba82c71 100644 --- a/rpc/macros/src/grpc/server.rs +++ b/rpc/macros/src/grpc/server.rs @@ -72,7 +72,8 @@ impl ToTokens for RpcTable { Box::pin(async move { let mut response: #spectred_response_type = match request.payload { Some(Payload::#request_type(ref request)) => match request.try_into() { - Ok(request) => server_ctx.core_service.#fn_call(request).await.into(), + // TODO: RPC-CONNECTION + Ok(request) => server_ctx.core_service.#fn_call(None,request).await.into(), Err(err) => #response_message_type::from(err).into(), }, _ => { @@ -128,7 +129,7 @@ impl ToTokens for RpcTable { { let mut interface = Interface::new(#server_ctx); - for op in #payload_ops::list() { + for op in #payload_ops::iter() { match op { #(#targets)* } diff --git a/rpc/macros/src/lib.rs b/rpc/macros/src/lib.rs index 1c205c2..9ca49bf 100644 --- a/rpc/macros/src/lib.rs +++ b/rpc/macros/src/lib.rs @@ -39,3 +39,9 @@ pub fn build_wrpc_wasm_bindgen_subscriptions(input: TokenStream) -> TokenStream pub fn build_grpc_server_interface(input: TokenStream) -> TokenStream { grpc::server::build_grpc_server_interface(input) } + +#[proc_macro] +#[proc_macro_error] +pub fn test_wrpc_serializer(input: TokenStream) -> TokenStream { + wrpc::test::build_test(input) +} diff --git a/rpc/macros/src/wrpc/client.rs b/rpc/macros/src/wrpc/client.rs index a24d80e..2b325a5 100644 --- a/rpc/macros/src/wrpc/client.rs +++ b/rpc/macros/src/wrpc/client.rs @@ -52,26 +52,29 @@ impl ToTokens for RpcTable { // the async implementation of the RPC caller is inlined targets.push(quote! { - fn #fn_call<'life0, 'async_trait>( + fn #fn_call<'life0, 'life1, 'async_trait>( &'life0 self, + _connection : ::core::option::Option<&'life1 Arc>, request: #request_type, ) -> ::core::pin::Pin> + ::core::marker::Send + 'async_trait>> where 'life0: 'async_trait, + 'life1: 'async_trait, Self: 'async_trait, { + use workflow_serializer::prelude::*; Box::pin(async move { if let ::core::option::Option::Some(__ret) = ::core::option::Option::None::> { return __ret; } let __self = self; //let request = request; - let __ret: RpcResult<#response_type> = { - let resp: ClientResult<#response_type> = __self.inner.rpc_client.call(#rpc_api_ops::#handler, request).await; + let __ret: RpcResult> = { + let resp: ClientResult> = __self.inner.rpc_client.call(#rpc_api_ops::#handler, Serializable(request)).await; Ok(resp.map_err(|e| spectre_rpc_core::error::RpcError::RpcSubsystem(e.to_string()))?) }; #[allow(unreachable_code)] - __ret + __ret.map(Serializable::into_inner) }) } diff --git a/rpc/macros/src/wrpc/mod.rs b/rpc/macros/src/wrpc/mod.rs index 1a15b06..8c8238c 100644 --- a/rpc/macros/src/wrpc/mod.rs +++ b/rpc/macros/src/wrpc/mod.rs @@ -1,3 +1,4 @@ pub mod client; pub mod server; +pub mod test; pub mod wasm; diff --git a/rpc/macros/src/wrpc/server.rs b/rpc/macros/src/wrpc/server.rs index 09a3e0c..092b1ed 100644 --- a/rpc/macros/src/wrpc/server.rs +++ b/rpc/macros/src/wrpc/server.rs @@ -50,13 +50,14 @@ impl ToTokens for RpcTable { targets.push(quote! { #rpc_api_ops::#handler => { - interface.method(#rpc_api_ops::#handler, method!(|server_ctx: #server_ctx_type, connection_ctx: #connection_ctx_type, request: #request_type| async move { + interface.method(#rpc_api_ops::#handler, method!(|server_ctx: #server_ctx_type, connection_ctx: #connection_ctx_type, request: Serializable<#request_type>| async move { let verbose = server_ctx.verbose(); if verbose { workflow_log::log_info!("request: {:?}",request); } - let response: #response_type = server_ctx.rpc_service(&connection_ctx).#fn_call(request).await + // TODO: RPC-CONNECT + let response: #response_type = server_ctx.rpc_service(&connection_ctx).#fn_call(None, request.into_inner()).await .map_err(|e|ServerError::Text(e.to_string()))?; if verbose { workflow_log::log_info!("response: {:?}",response); } - Ok(response) + Ok(Serializable(response)) })); } }); @@ -71,7 +72,8 @@ impl ToTokens for RpcTable { #rpc_api_ops >::new(#server_ctx); - for op in #rpc_api_ops::list() { + for op in #rpc_api_ops::iter() { + use workflow_serializer::prelude::*; match op { #(#targets)* _ => { } diff --git a/rpc/macros/src/wrpc/test.rs b/rpc/macros/src/wrpc/test.rs new file mode 100644 index 0000000..92591b2 --- /dev/null +++ b/rpc/macros/src/wrpc/test.rs @@ -0,0 +1,60 @@ +use convert_case::{Case, Casing}; +use proc_macro2::TokenStream; +use proc_macro2::{Ident, Span}; +use quote::{quote, ToTokens}; +use std::convert::Into; +use syn::{ + parse::{Parse, ParseStream}, + parse_macro_input, + punctuated::Punctuated, + Error, Expr, Result, Token, +}; + +#[derive(Debug)] +struct TestTable { + rpc_op: Expr, +} + +impl Parse for TestTable { + fn parse(input: ParseStream) -> Result { + let parsed = Punctuated::::parse_terminated(input).unwrap(); + if parsed.len() != 1 { + return Err(Error::new_spanned(parsed, "usage: test!(GetInfo)".to_string())); + } + + let mut iter = parsed.iter(); + let rpc_op = iter.next().unwrap().clone(); + + Ok(TestTable { rpc_op }) + } +} + +impl ToTokens for TestTable { + fn to_tokens(&self, tokens: &mut TokenStream) { + let rpc_op = &self.rpc_op; + + let (name, _docs) = match rpc_op { + syn::Expr::Path(expr_path) => (expr_path.path.to_token_stream().to_string(), expr_path.attrs.clone()), + _ => (rpc_op.to_token_stream().to_string(), vec![]), + }; + let typename = Ident::new(&name.to_string(), Span::call_site()); + let fn_test = Ident::new(&format!("test_wrpc_serializer_{}", name.to_case(Case::Snake)), Span::call_site()); + + quote! { + + #[test] + fn #fn_test() { + test::<#typename>(#name); + } + + } + .to_tokens(tokens); + } +} + +pub fn build_test(input: proc_macro::TokenStream) -> proc_macro::TokenStream { + let rpc_table = parse_macro_input!(input as TestTable); + let ts = rpc_table.to_token_stream(); + // println!("MACRO: {}", ts.to_string()); + ts.into() +} diff --git a/rpc/macros/src/wrpc/wasm.rs b/rpc/macros/src/wrpc/wasm.rs index 0118220..30af3e7 100644 --- a/rpc/macros/src/wrpc/wasm.rs +++ b/rpc/macros/src/wrpc/wasm.rs @@ -59,7 +59,7 @@ impl ToTokens for RpcHandlers { pub async fn #fn_no_suffix(&self, request : Option<#ts_request_type>) -> Result<#ts_response_type> { let request: #request_type = request.unwrap_or_default().try_into()?; // log_info!("request: {:#?}",request); - let result: RpcResult<#response_type> = self.inner.client.#fn_call(request).await; + let result: RpcResult<#response_type> = self.inner.client.#fn_call(None, request).await; // log_info!("result: {:#?}",result); let response: #response_type = result.map_err(|err|wasm_bindgen::JsError::new(&err.to_string()))?; //log_info!("response: {:#?}",response); @@ -83,7 +83,7 @@ impl ToTokens for RpcHandlers { #[wasm_bindgen(js_name = #fn_camel)] pub async fn #fn_no_suffix(&self, request: #ts_request_type) -> Result<#ts_response_type> { let request: #request_type = request.try_into()?; - let result: RpcResult<#response_type> = self.inner.client.#fn_call(request).await; + let result: RpcResult<#response_type> = self.inner.client.#fn_call(None, request).await; let response: #response_type = result.map_err(|err|wasm_bindgen::JsError::new(&err.to_string()))?; Ok(response.try_into()?) } diff --git a/rpc/service/Cargo.toml b/rpc/service/Cargo.toml index 670b54c..ffe443c 100644 --- a/rpc/service/Cargo.toml +++ b/rpc/service/Cargo.toml @@ -33,4 +33,4 @@ async-trait.workspace = true log.workspace = true tokio.workspace = true triggered.workspace = true -workflow-rpc.workspace = true +workflow-rpc.workspace = true \ No newline at end of file diff --git a/rpc/service/src/converter/consensus.rs b/rpc/service/src/converter/consensus.rs index 678d6e6..0c3dad5 100644 --- a/rpc/service/src/converter/consensus.rs +++ b/rpc/service/src/converter/consensus.rs @@ -81,7 +81,7 @@ impl ConsensusConverter { vec![] }; - Ok(RpcBlock { header: (*block.header).clone(), transactions, verbose_data }) + Ok(RpcBlock { header: block.header.as_ref().into(), transactions, verbose_data }) } pub fn get_mempool_entry(&self, consensus: &ConsensusProxy, transaction: &MutableTransaction) -> RpcMempoolEntry { diff --git a/rpc/service/src/service.rs b/rpc/service/src/service.rs index e31bdac..59d2b7a 100644 --- a/rpc/service/src/service.rs +++ b/rpc/service/src/service.rs @@ -55,7 +55,8 @@ use spectre_p2p_lib::common::ProtocolError; use spectre_perf_monitor::{counters::CountersSnapshot, Monitor as PerfMonitor}; use spectre_rpc_core::{ api::{ - ops::RPC_API_VERSION, + connection::DynRpcConnection, + ops::{RPC_API_REVISION, RPC_API_VERSION}, rpc::{RpcApi, MAX_SAFE_WINDOW_SIZE}, }, model::*, @@ -64,6 +65,7 @@ use spectre_rpc_core::{ }; use spectre_txscript::{extract_script_pub_key_address, pay_to_address_script}; use spectre_utils::expiring_cache::ExpiringCache; +use spectre_utils::sysinfo::SystemInfo; use spectre_utils::{channel::Channel, triggers::SingleTrigger}; use spectre_utils_tower::counters::TowerConnectionCounters; use spectre_utxoindex::api::UtxoIndexProxy; @@ -113,6 +115,7 @@ pub struct RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + system_info: SystemInfo, fee_estimate_cache: ExpiringCache, fee_estimate_verbose_cache: ExpiringCache>, } @@ -139,6 +142,7 @@ impl RpcCoreService { perf_monitor: Arc>>, p2p_tower_counters: Arc, grpc_tower_counters: Arc, + system_info: SystemInfo, ) -> Self { // This notifier UTXOs subscription granularity to index-processor or consensus notifier let policies = match index_notifier { @@ -214,6 +218,7 @@ impl RpcCoreService { perf_monitor, p2p_tower_counters, grpc_tower_counters, + system_info, fee_estimate_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), fee_estimate_verbose_cache: ExpiringCache::new(Duration::from_millis(500), Duration::from_millis(1000)), } @@ -283,7 +288,11 @@ impl RpcCoreService { #[async_trait] impl RpcApi for RpcCoreService { - async fn submit_block_call(&self, request: SubmitBlockRequest) -> RpcResult { + async fn submit_block_call( + &self, + _connection: Option<&DynRpcConnection>, + request: SubmitBlockRequest, + ) -> RpcResult { let session = self.consensus_manager.consensus().unguarded_session(); // TODO: consider adding an error field to SubmitBlockReport to document both the report and error fields @@ -294,7 +303,7 @@ impl RpcApi for RpcCoreService { return Ok(SubmitBlockResponse { report: SubmitBlockReport::Reject(SubmitBlockRejectReason::IsInIBD) }); } - let try_block: RpcResult = (&request.block).try_into(); + let try_block: RpcResult = request.block.try_into(); if let Err(err) = &try_block { trace!("incoming SubmitBlockRequest with block conversion error: {}", err); // error = format!("Could not parse block: {0}", err) @@ -341,7 +350,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and } } - async fn get_block_template_call(&self, request: GetBlockTemplateRequest) -> RpcResult { + async fn get_block_template_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetBlockTemplateRequest, + ) -> RpcResult { trace!("incoming GetBlockTemplate request"); if *self.config.net == NetworkType::Mainnet && !self.config.enable_mainnet_mining { @@ -368,12 +381,12 @@ NOTE: This error usually indicates an RPC conversion error between the node and let is_nearly_synced = self.config.is_nearly_synced(block_template.selected_parent_timestamp, block_template.selected_parent_daa_score); Ok(GetBlockTemplateResponse { - block: (&block_template.block).into(), + block: block_template.block.into(), is_synced: self.has_sufficient_peer_connectivity() && is_nearly_synced, }) } - async fn get_block_call(&self, request: GetBlockRequest) -> RpcResult { + async fn get_block_call(&self, _connection: Option<&DynRpcConnection>, request: GetBlockRequest) -> RpcResult { // TODO: test let session = self.consensus_manager.consensus().session().await; let block = session.async_get_block_even_if_header_only(request.hash).await?; @@ -385,7 +398,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and }) } - async fn get_blocks_call(&self, request: GetBlocksRequest) -> RpcResult { + async fn get_blocks_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetBlocksRequest, + ) -> RpcResult { // Validate that user didn't set include_transactions without setting include_blocks if !request.include_blocks && request.include_transactions { return Err(RpcError::InvalidGetBlocksRequest); @@ -434,7 +451,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetBlocksResponse { block_hashes, blocks }) } - async fn get_info_call(&self, _request: GetInfoRequest) -> RpcResult { + async fn get_info_call(&self, _connection: Option<&DynRpcConnection>, _request: GetInfoRequest) -> RpcResult { let is_nearly_synced = self.consensus_manager.consensus().unguarded_session().async_is_nearly_synced().await; Ok(GetInfoResponse { p2p_id: self.flow_context.node_id.to_string(), @@ -447,7 +464,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and }) } - async fn get_mempool_entry_call(&self, request: GetMempoolEntryRequest) -> RpcResult { + async fn get_mempool_entry_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetMempoolEntryRequest, + ) -> RpcResult { let query = self.extract_tx_query(request.filter_transaction_pool, request.include_orphan_pool)?; let Some(transaction) = self.mining_manager.clone().get_transaction(request.transaction_id, query).await else { return Err(RpcError::TransactionNotFound(request.transaction_id)); @@ -456,7 +477,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetMempoolEntryResponse::new(self.consensus_converter.get_mempool_entry(&session, &transaction))) } - async fn get_mempool_entries_call(&self, request: GetMempoolEntriesRequest) -> RpcResult { + async fn get_mempool_entries_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetMempoolEntriesRequest, + ) -> RpcResult { let query = self.extract_tx_query(request.filter_transaction_pool, request.include_orphan_pool)?; let session = self.consensus_manager.consensus().unguarded_session(); let (transactions, orphans) = self.mining_manager.clone().get_all_transactions(query).await; @@ -470,6 +495,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn get_mempool_entries_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, request: GetMempoolEntriesByAddressesRequest, ) -> RpcResult { let query = self.extract_tx_query(request.filter_transaction_pool, request.include_orphan_pool)?; @@ -493,13 +519,17 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetMempoolEntriesByAddressesResponse::new(mempool_entries)) } - async fn submit_transaction_call(&self, request: SubmitTransactionRequest) -> RpcResult { + async fn submit_transaction_call( + &self, + _connection: Option<&DynRpcConnection>, + request: SubmitTransactionRequest, + ) -> RpcResult { let allow_orphan = self.config.unsafe_rpc && request.allow_orphan; if !self.config.unsafe_rpc && request.allow_orphan { warn!("SubmitTransaction RPC command called with AllowOrphan enabled while node in safe RPC mode -- switching to ForbidOrphan."); } - let transaction: Transaction = (&request.transaction).try_into()?; + let transaction: Transaction = request.transaction.try_into()?; let transaction_id = transaction.id(); let session = self.consensus_manager.consensus().unguarded_session(); let orphan = match allow_orphan { @@ -516,9 +546,10 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn submit_transaction_replacement_call( &self, + _connection: Option<&DynRpcConnection>, request: SubmitTransactionReplacementRequest, ) -> RpcResult { - let transaction: Transaction = (&request.transaction).try_into()?; + let transaction: Transaction = request.transaction.try_into()?; let transaction_id = transaction.id(); let session = self.consensus_manager.consensus().unguarded_session(); let replaced_transaction = @@ -530,25 +561,38 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(SubmitTransactionReplacementResponse::new(transaction_id, (&*replaced_transaction).into())) } - async fn get_current_network_call(&self, _: GetCurrentNetworkRequest) -> RpcResult { + async fn get_current_network_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetCurrentNetworkRequest, + ) -> RpcResult { Ok(GetCurrentNetworkResponse::new(*self.config.net)) } - async fn get_subnetwork_call(&self, _: GetSubnetworkRequest) -> RpcResult { + async fn get_subnetwork_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetSubnetworkRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sink_call(&self, _: GetSinkRequest) -> RpcResult { + async fn get_sink_call(&self, _connection: Option<&DynRpcConnection>, _: GetSinkRequest) -> RpcResult { Ok(GetSinkResponse::new(self.consensus_manager.consensus().unguarded_session().async_get_sink().await)) } - async fn get_sink_blue_score_call(&self, _: GetSinkBlueScoreRequest) -> RpcResult { + async fn get_sink_blue_score_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetSinkBlueScoreRequest, + ) -> RpcResult { let session = self.consensus_manager.consensus().unguarded_session(); Ok(GetSinkBlueScoreResponse::new(session.async_get_ghostdag_data(session.async_get_sink().await).await?.blue_score)) } async fn get_virtual_chain_from_block_call( &self, + _connection: Option<&DynRpcConnection>, request: GetVirtualChainFromBlockRequest, ) -> RpcResult { let session = self.consensus_manager.consensus().session().await; @@ -561,11 +605,19 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetVirtualChainFromBlockResponse::new(virtual_chain.removed, virtual_chain.added, accepted_transaction_ids)) } - async fn get_block_count_call(&self, _: GetBlockCountRequest) -> RpcResult { + async fn get_block_count_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetBlockCountRequest, + ) -> RpcResult { Ok(self.consensus_manager.consensus().unguarded_session().async_estimate_block_count().await) } - async fn get_utxos_by_addresses_call(&self, request: GetUtxosByAddressesRequest) -> RpcResult { + async fn get_utxos_by_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetUtxosByAddressesRequest, + ) -> RpcResult { if !self.config.utxoindex { return Err(RpcError::NoUtxoIndex); } @@ -575,7 +627,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetUtxosByAddressesResponse::new(self.index_converter.get_utxos_by_addresses_entries(&entry_map))) } - async fn get_balance_by_address_call(&self, request: GetBalanceByAddressRequest) -> RpcResult { + async fn get_balance_by_address_call( + &self, + _connection: Option<&DynRpcConnection>, + request: GetBalanceByAddressRequest, + ) -> RpcResult { if !self.config.utxoindex { return Err(RpcError::NoUtxoIndex); } @@ -586,6 +642,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn get_balances_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, request: GetBalancesByAddressesRequest, ) -> RpcResult { if !self.config.utxoindex { @@ -604,7 +661,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetBalancesByAddressesResponse::new(entries)) } - async fn get_coin_supply_call(&self, _: GetCoinSupplyRequest) -> RpcResult { + async fn get_coin_supply_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetCoinSupplyRequest, + ) -> RpcResult { if !self.config.utxoindex { return Err(RpcError::NoUtxoIndex); } @@ -615,6 +676,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn get_daa_score_timestamp_estimate_call( &self, + _connection: Option<&DynRpcConnection>, request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult { let session = self.consensus_manager.consensus().session().await; @@ -671,7 +733,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(GetDaaScoreTimestampEstimateResponse::new(timestamps)) } - async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + async fn get_fee_estimate_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetFeeEstimateRequest, + ) -> RpcResult { let mining_manager = self.mining_manager.clone(); let estimate = self.fee_estimate_cache.get(async move { mining_manager.get_realtime_feerate_estimations().await.into_rpc() }).await; @@ -680,6 +746,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn get_fee_estimate_experimental_call( &self, + connection: Option<&DynRpcConnection>, request: GetFeeEstimateExperimentalRequest, ) -> RpcResult { if request.verbose { @@ -696,20 +763,28 @@ NOTE: This error usually indicates an RPC conversion error between the node and .await?; Ok(response) } else { - let estimate = self.get_fee_estimate_call(GetFeeEstimateRequest {}).await?.estimate; + let estimate = self.get_fee_estimate_call(connection, GetFeeEstimateRequest {}).await?.estimate; Ok(GetFeeEstimateExperimentalResponse { estimate, verbose: None }) } } - async fn ping_call(&self, _: PingRequest) -> RpcResult { + async fn ping_call(&self, _connection: Option<&DynRpcConnection>, _: PingRequest) -> RpcResult { Ok(PingResponse {}) } - async fn get_headers_call(&self, _request: GetHeadersRequest) -> RpcResult { + async fn get_headers_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetHeadersRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_dag_info_call(&self, _: GetBlockDagInfoRequest) -> RpcResult { + async fn get_block_dag_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetBlockDagInfoRequest, + ) -> RpcResult { let session = self.consensus_manager.consensus().unguarded_session(); let (consensus_stats, tips, pruning_point, sink) = join!(session.async_get_stats(), session.async_get_tips(), session.async_pruning_point(), session.async_get_sink()); @@ -729,6 +804,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn estimate_network_hashes_per_second_call( &self, + _connection: Option<&DynRpcConnection>, request: EstimateNetworkHashesPerSecondRequest, ) -> RpcResult { if !self.config.unsafe_rpc && request.window_size > MAX_SAFE_WINDOW_SIZE { @@ -758,7 +834,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and )) } - async fn add_peer_call(&self, request: AddPeerRequest) -> RpcResult { + async fn add_peer_call(&self, _connection: Option<&DynRpcConnection>, request: AddPeerRequest) -> RpcResult { if !self.config.unsafe_rpc { warn!("AddPeer RPC command called while node in safe RPC mode -- ignoring."); return Err(RpcError::UnavailableInSafeMode); @@ -772,12 +848,16 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(AddPeerResponse {}) } - async fn get_peer_addresses_call(&self, _: GetPeerAddressesRequest) -> RpcResult { + async fn get_peer_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetPeerAddressesRequest, + ) -> RpcResult { let address_manager = self.flow_context.address_manager.lock(); Ok(GetPeerAddressesResponse::new(address_manager.get_all_addresses(), address_manager.get_all_banned_addresses())) } - async fn ban_call(&self, request: BanRequest) -> RpcResult { + async fn ban_call(&self, _connection: Option<&DynRpcConnection>, request: BanRequest) -> RpcResult { if !self.config.unsafe_rpc { warn!("Ban RPC command called while node in safe RPC mode -- ignoring."); return Err(RpcError::UnavailableInSafeMode); @@ -794,7 +874,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(BanResponse {}) } - async fn unban_call(&self, request: UnbanRequest) -> RpcResult { + async fn unban_call(&self, _connection: Option<&DynRpcConnection>, request: UnbanRequest) -> RpcResult { if !self.config.unsafe_rpc { warn!("Unban RPC command called while node in safe RPC mode -- ignoring."); return Err(RpcError::UnavailableInSafeMode); @@ -808,13 +888,17 @@ NOTE: This error usually indicates an RPC conversion error between the node and Ok(UnbanResponse {}) } - async fn get_connected_peer_info_call(&self, _: GetConnectedPeerInfoRequest) -> RpcResult { + async fn get_connected_peer_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _: GetConnectedPeerInfoRequest, + ) -> RpcResult { let peers = self.flow_context.hub().active_peers(); let peer_info = self.protocol_converter.get_peers_info(&peers); Ok(GetConnectedPeerInfoResponse::new(peer_info)) } - async fn shutdown_call(&self, _: ShutdownRequest) -> RpcResult { + async fn shutdown_call(&self, _connection: Option<&DynRpcConnection>, _: ShutdownRequest) -> RpcResult { if !self.config.unsafe_rpc { warn!("Shutdown RPC command called while node in safe RPC mode -- ignoring."); return Err(RpcError::UnavailableInSafeMode); @@ -837,6 +921,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and async fn resolve_finality_conflict_call( &self, + _connection: Option<&DynRpcConnection>, _request: ResolveFinalityConflictRequest, ) -> RpcResult { if !self.config.unsafe_rpc { @@ -846,7 +931,25 @@ NOTE: This error usually indicates an RPC conversion error between the node and Err(RpcError::NotImplemented) } - async fn get_metrics_call(&self, req: GetMetricsRequest) -> RpcResult { + async fn get_connections_call( + &self, + _connection: Option<&DynRpcConnection>, + req: GetConnectionsRequest, + ) -> RpcResult { + let clients = (self.wrpc_borsh_counters.active_connections.load(Ordering::Relaxed) + + self.wrpc_json_counters.active_connections.load(Ordering::Relaxed)) as u32; + let peers = self.flow_context.hub().active_peers_len() as u16; + + let profile_data = req.include_profile_data.then(|| { + let CountersSnapshot { resident_set_size: memory_usage, cpu_usage, .. } = self.perf_monitor.snapshot(); + + ConnectionsProfileData { cpu_usage: cpu_usage as f32, memory_usage } + }); + + Ok(GetConnectionsResponse { clients, peers, profile_data }) + } + + async fn get_metrics_call(&self, _connection: Option<&DynRpcConnection>, req: GetMetricsRequest) -> RpcResult { let CountersSnapshot { resident_set_size, virtual_memory_size, @@ -871,7 +974,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and disk_io_write_per_sec: disk_io_write_per_sec as f32, }); - let connection_metrics = req.connection_metrics.then_some(ConnectionMetrics { + let connection_metrics = req.connection_metrics.then(|| ConnectionMetrics { borsh_live_connections: self.wrpc_borsh_counters.active_connections.load(Ordering::Relaxed) as u32, borsh_connection_attempts: self.wrpc_borsh_counters.total_connections.load(Ordering::Relaxed) as u64, borsh_handshake_failures: self.wrpc_borsh_counters.handshake_failures.load(Ordering::Relaxed) as u64, @@ -882,7 +985,7 @@ NOTE: This error usually indicates an RPC conversion error between the node and active_peers: self.flow_context.hub().active_peers_len() as u32, }); - let bandwidth_metrics = req.bandwidth_metrics.then_some(BandwidthMetrics { + let bandwidth_metrics = req.bandwidth_metrics.then(|| BandwidthMetrics { borsh_bytes_tx: self.wrpc_borsh_counters.tx_bytes.load(Ordering::Relaxed) as u64, borsh_bytes_rx: self.wrpc_borsh_counters.rx_bytes.load(Ordering::Relaxed) as u64, json_bytes_tx: self.wrpc_json_counters.tx_bytes.load(Ordering::Relaxed) as u64, @@ -920,20 +1023,54 @@ NOTE: This error usually indicates an RPC conversion error between the node and None }; + let storage_metrics = req.storage_metrics.then_some(StorageMetrics { storage_size_bytes: 0 }); + + let custom_metrics: Option> = None; + let server_time = unix_now(); - let response = GetMetricsResponse { server_time, process_metrics, connection_metrics, bandwidth_metrics, consensus_metrics }; + let response = GetMetricsResponse { + server_time, + process_metrics, + connection_metrics, + bandwidth_metrics, + consensus_metrics, + storage_metrics, + custom_metrics, + }; Ok(response) } - async fn get_server_info_call(&self, _request: GetServerInfoRequest) -> RpcResult { + async fn get_system_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSystemInfoRequest, + ) -> RpcResult { + let response = GetSystemInfoResponse { + version: self.system_info.version.clone(), + system_id: self.system_info.system_id.clone(), + git_hash: self.system_info.git_short_hash.clone(), + cpu_physical_cores: self.system_info.cpu_physical_cores, + total_memory: self.system_info.total_memory, + fd_limit: self.system_info.fd_limit, + }; + + Ok(response) + } + + async fn get_server_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetServerInfoRequest, + ) -> RpcResult { let session = self.consensus_manager.consensus().unguarded_session(); let is_synced: bool = self.has_sufficient_peer_connectivity() && session.async_is_nearly_synced().await; let virtual_daa_score = session.get_virtual_daa_score(); Ok(GetServerInfoResponse { rpc_api_version: RPC_API_VERSION, + rpc_api_revision: RPC_API_REVISION, server_version: version().to_string(), network_id: self.config.net, has_utxo_index: self.config.utxoindex, @@ -942,7 +1079,11 @@ NOTE: This error usually indicates an RPC conversion error between the node and }) } - async fn get_sync_status_call(&self, _request: GetSyncStatusRequest) -> RpcResult { + async fn get_sync_status_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSyncStatusRequest, + ) -> RpcResult { let session = self.consensus_manager.consensus().unguarded_session(); let is_synced: bool = self.has_sufficient_peer_connectivity() && session.async_is_nearly_synced().await; Ok(GetSyncStatusResponse { is_synced }) diff --git a/rpc/wrpc/client/Cargo.toml b/rpc/wrpc/client/Cargo.toml index f2d2880..79a51cd 100644 --- a/rpc/wrpc/client/Cargo.toml +++ b/rpc/wrpc/client/Cargo.toml @@ -44,4 +44,5 @@ workflow-dom.workspace = true workflow-http.workspace = true workflow-log.workspace = true workflow-rpc.workspace = true +workflow-serializer.workspace = true workflow-wasm.workspace = true diff --git a/rpc/wrpc/client/Resolvers.toml b/rpc/wrpc/client/Resolvers.toml index 42a6b1d..e9548b8 100644 --- a/rpc/wrpc/client/Resolvers.toml +++ b/rpc/wrpc/client/Resolvers.toml @@ -1,7 +1,12 @@ [[resolver]] -url = "https://resolver.spectre-network.org" enable = true +address = "https://resolver.spectre-network.org" [[resolver]] -url = "http://127.0.0.1:8888" enable = false +address = "http://127.0.0.1:8888" + +[[group]] +enable = false +template = "https://*." +nodes = ["rudy", "todd", "clay", "walt"] diff --git a/rpc/wrpc/client/src/client.rs b/rpc/wrpc/client/src/client.rs index 6df428f..fe0c40c 100644 --- a/rpc/wrpc/client/src/client.rs +++ b/rpc/wrpc/client/src/client.rs @@ -18,7 +18,7 @@ use workflow_rpc::client::Ctl as WrpcCtl; pub use workflow_rpc::client::{ ConnectOptions, ConnectResult, ConnectStrategy, Resolver as RpcResolver, ResolverResult, WebSocketConfig, WebSocketError, }; - +use workflow_serializer::prelude::*; type RpcClientNotifier = Arc>; struct Inner { @@ -34,7 +34,14 @@ struct Inner { connect_guard: AsyncMutex<()>, disconnect_guard: AsyncMutex<()>, // --- + // The permanent url passed in the constructor + // (dominant, overrides Resolver if supplied). + ctor_url: Mutex>, + // The url passed in the connect() method + // (overrides default URL and the Resolver). default_url: Mutex>, + // The current url wRPC is connected to + // (possibly acquired via the Resolver). current_url: Mutex>, resolver: Mutex>, network_id: Mutex>, @@ -73,16 +80,16 @@ impl Inner { let notification_sender_ = notification_relay_channel.sender.clone(); interface.notification( notification_op, - workflow_rpc::client::Notification::new(move |notification: spectre_rpc_core::Notification| { + workflow_rpc::client::Notification::new(move |notification: Serializable| { let notification_sender = notification_sender_.clone(); Box::pin(async move { // log_info!("notification receivers: {}", notification_sender.receiver_count()); // log_trace!("notification {:?}", notification); if notification_sender.receiver_count() > 1 { // log_info!("notification: posting to channel: {notification:?}"); - notification_sender.send(notification).await?; + notification_sender.send(notification.into_inner()).await?; } else { - log_warn!("WARNING: Spectre RPC notification is not consumed by user: {:?}", notification); + log_warn!("WARNING: Spectre RPC notification is not consumed by user: {:?}", notification.into_inner()); } Ok(()) }) @@ -104,7 +111,8 @@ impl Inner { connect_guard: async_std::sync::Mutex::new(()), disconnect_guard: async_std::sync::Mutex::new(()), // --- - default_url: Mutex::new(url.map(|s| s.to_string())), + ctor_url: Mutex::new(url.map(|s| s.to_string())), + default_url: Mutex::new(None), current_url: Mutex::new(None), resolver: Mutex::new(resolver), network_id: Mutex::new(network_id), @@ -121,17 +129,22 @@ impl Inner { /// Start sending notifications of some type to the client. async fn start_notify_to_client(&self, scope: Scope) -> RpcResult<()> { - let _response: SubscribeResponse = self.rpc_client.call(RpcApiOps::Subscribe, scope).await.map_err(|err| err.to_string())?; + let _response: Serializable = + self.rpc_client.call(RpcApiOps::Subscribe, Serializable(scope)).await.map_err(|err| err.to_string())?; Ok(()) } /// Stop sending notifications of some type to the client. async fn stop_notify_to_client(&self, scope: Scope) -> RpcResult<()> { - let _response: UnsubscribeResponse = - self.rpc_client.call(RpcApiOps::Unsubscribe, scope).await.map_err(|err| err.to_string())?; + let _response: Serializable = + self.rpc_client.call(RpcApiOps::Unsubscribe, Serializable(scope)).await.map_err(|err| err.to_string())?; Ok(()) } + fn ctor_url(&self) -> Option { + self.ctor_url.lock().unwrap().clone() + } + fn default_url(&self) -> Option { self.default_url.lock().unwrap().clone() } @@ -213,7 +226,7 @@ impl SubscriptionManager for Inner { #[async_trait] impl RpcResolver for Inner { async fn resolve_url(&self) -> ResolverResult { - let url = if let Some(url) = self.default_url() { + let url = if let Some(url) = self.default_url().or(self.ctor_url()) { url } else if let Some(resolver) = self.resolver().as_ref() { let network_id = self.network_id().expect("Resolver requires network id in RPC client configuration"); @@ -222,7 +235,7 @@ impl RpcResolver for Inner { self.node_descriptor.lock().unwrap().replace(Arc::new(node)); url } else { - panic!("RpcClient resolver configuration error (expecting Some(Resolver))") + panic!("RpcClient resolver configuration error (expecting `url` or `resolver` as `Some(Resolver))`") }; self.rpc_ctl.set_descriptor(Some(url.clone())); @@ -254,7 +267,11 @@ impl Debug for SpectreRpcClient { } impl SpectreRpcClient { - /// Create a new `SpectreRpcClient` with the given Encoding and URL + /// Create a new `SpectreRpcClient` with the given Encoding, and an optional url or a Resolver. + /// Please note that if you pass the url to the constructor, it will force the SpectreRpcClient + /// to always use this url. If you want to have the ability to switch between urls, + /// you must pass [`Option::None`] as the `url` argument and then supply your own url to the `connect()` + /// function each time you connect. pub fn new( encoding: Encoding, url: Option<&str>, @@ -371,6 +388,10 @@ impl SpectreRpcClient { &self.inner.rpc_ctl } + pub fn ctl_multiplexer(&self) -> Multiplexer { + self.inner.wrpc_ctl_multiplexer.clone() + } + /// Start background RPC services. pub async fn start(&self) -> Result<()> { if !self.inner.background_services_running.load(Ordering::SeqCst) { @@ -403,12 +424,10 @@ impl SpectreRpcClient { pub async fn connect(&self, options: Option) -> ConnectResult { let _guard = self.inner.connect_guard.lock().await; - let mut options = options.unwrap_or_default(); + let options = options.unwrap_or_default(); let strategy = options.strategy; - if let Some(url) = options.url.take() { - self.set_url(Some(&url))?; - } + self.inner.set_default_url(options.url.as_deref()); // 1Gb message and frame size limits (on native and NodeJs platforms) let ws_config = WebSocketConfig { @@ -584,6 +603,7 @@ impl RpcApi for SpectreRpcClient { build_wrpc_client_interface!( RpcApiOps, [ + Ping, AddPeer, Ban, EstimateNetworkHashesPerSecond, @@ -596,6 +616,7 @@ impl RpcApi for SpectreRpcClient { GetBlockTemplate, GetCoinSupply, GetConnectedPeerInfo, + GetConnections, GetCurrentNetwork, GetDaaScoreTimestampEstimate, GetFeeEstimate, @@ -612,9 +633,9 @@ impl RpcApi for SpectreRpcClient { GetSinkBlueScore, GetSubnetwork, GetSyncStatus, + GetSystemInfo, GetUtxosByAddresses, GetVirtualChainFromBlock, - Ping, ResolveFinalityConflict, Shutdown, SubmitBlock, diff --git a/rpc/wrpc/client/src/node.rs b/rpc/wrpc/client/src/node.rs index 4afb0f1..ca7e19c 100644 --- a/rpc/wrpc/client/src/node.rs +++ b/rpc/wrpc/client/src/node.rs @@ -11,16 +11,10 @@ use crate::imports::*; pub struct NodeDescriptor { /// The unique identifier of the node. #[wasm_bindgen(getter_with_clone)] - pub id: String, + pub uid: String, /// The URL of the node WebSocket (wRPC URL). #[wasm_bindgen(getter_with_clone)] pub url: String, - /// Optional name of the node provider. - #[wasm_bindgen(getter_with_clone)] - pub provider_name: Option, - /// Optional site URL of the node provider. - #[wasm_bindgen(getter_with_clone)] - pub provider_url: Option, } impl Eq for NodeDescriptor {} diff --git a/rpc/wrpc/client/src/resolver.rs b/rpc/wrpc/client/src/resolver.rs index 6c91ad3..23ab4a0 100644 --- a/rpc/wrpc/client/src/resolver.rs +++ b/rpc/wrpc/client/src/resolver.rs @@ -1,40 +1,79 @@ +use std::sync::OnceLock; + use crate::error::Error; use crate::imports::*; use crate::node::NodeDescriptor; pub use futures::future::join_all; use rand::seq::SliceRandom; use rand::thread_rng; +use workflow_core::runtime; use workflow_http::get_json; -const DEFAULT_VERSION: usize = 1; +const CURRENT_VERSION: usize = 2; +const RESOLVER_CONFIG: &str = include_str!("../Resolvers.toml"); #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ResolverRecord { - pub url: String, + pub address: String, + pub enable: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResolverGroup { + pub template: String, + pub nodes: Vec, pub enable: Option, } #[derive(Clone, Debug, Serialize, Deserialize)] pub struct ResolverConfig { - resolver: Vec, + #[serde(rename = "group")] + groups: Vec, + #[serde(rename = "resolver")] + resolvers: Vec, } fn try_parse_resolvers(toml: &str) -> Result>> { - Ok(toml::from_str::(toml)? - .resolver + let config = toml::from_str::(toml)?; + + let mut resolvers = config + .resolvers .into_iter() - .filter_map(|resolver| resolver.enable.unwrap_or(true).then_some(Arc::new(resolver.url))) - .collect::>()) + .filter_map(|resolver| resolver.enable.unwrap_or(true).then_some(resolver.address)) + .collect::>(); + + let groups = config.groups.into_iter().filter(|group| group.enable.unwrap_or(true)).collect::>(); + + for group in groups { + let ResolverGroup { template, nodes, .. } = group; + for node in nodes { + resolvers.push(template.replace('*', &node)); + } + } + + Ok(resolvers.into_iter().map(Arc::new).collect::>()) } #[derive(Debug)] struct Inner { pub urls: Vec>, + pub tls: bool, + public: bool, } impl Inner { - pub fn new(urls: Vec>) -> Self { - Self { urls } + pub fn new(urls: Option>>, tls: bool) -> Self { + if urls.as_ref().is_some_and(|urls| urls.is_empty()) { + panic!("Resolver: Empty URL list supplied to the constructor."); + } + + let mut public = false; + let urls = urls.unwrap_or_else(|| { + public = true; + try_parse_resolvers(RESOLVER_CONFIG).expect("TOML: Unable to parse RPC Resolver list") + }); + + Self { urls, tls, public } } } @@ -48,27 +87,61 @@ pub struct Resolver { impl Default for Resolver { fn default() -> Self { - let toml = include_str!("../Resolvers.toml"); - let urls = try_parse_resolvers(toml).expect("TOML: Unable to parse RPC Resolver list"); - Self { inner: Arc::new(Inner::new(urls)) } + Self { inner: Arc::new(Inner::new(None, false)) } } } impl Resolver { - pub fn new(urls: Vec>) -> Self { - if urls.is_empty() { - panic!("Resolver: Empty URL list supplied to the constructor."); + pub fn new(urls: Option>>, tls: bool) -> Self { + Self { inner: Arc::new(Inner::new(urls, tls)) } + } + + pub fn urls(&self) -> Option>> { + if self.inner.public { + None + } else { + Some(self.inner.urls.clone()) } + } + + pub fn tls(&self) -> bool { + self.inner.tls + } - Self { inner: Arc::new(Inner::new(urls)) } + pub fn tls_as_str(&self) -> &'static str { + if self.inner.tls { + "tls" + } else { + "any" + } } - pub fn urls(&self) -> Vec> { - self.inner.urls.clone() + fn make_url(&self, url: &str, encoding: Encoding, network_id: NetworkId) -> String { + static TLS: OnceLock<&'static str> = OnceLock::new(); + + let tls = *TLS.get_or_init(|| { + if runtime::is_web() { + let tls = js_sys::Reflect::get(&js_sys::global(), &"location".into()) + .and_then(|location| js_sys::Reflect::get(&location, &"protocol".into())) + .ok() + .and_then(|protocol| protocol.as_string()) + .map(|protocol| protocol.starts_with("https")) + .unwrap_or(false); + if tls { + "tls" + } else { + self.tls_as_str() + } + } else { + self.tls_as_str() + } + }); + + format!("{url}/v{CURRENT_VERSION}/spectre/{network_id}/{tls}/wrpc/{encoding}") } async fn fetch_node_info(&self, url: &str, encoding: Encoding, network_id: NetworkId) -> Result { - let url = format!("{}/v{}/wrpc/{}/{}", url, DEFAULT_VERSION, encoding, network_id); + let url = self.make_url(url, encoding, network_id); let node = get_json::(&url).await.map_err(|error| Error::custom(format!("Unable to connect to {url}: {error}")))?; Ok(node) @@ -88,27 +161,6 @@ impl Resolver { Err(Error::Custom(format!("Failed to connect: {:?}", errors))) } - pub async fn fetch_all(&self, encoding: Encoding, network_id: NetworkId) -> Result> { - let futures = self.inner.urls.iter().map(|url| self.fetch_node_info(url, encoding, network_id)).collect::>(); - let mut errors = Vec::default(); - let result = join_all(futures) - .await - .into_iter() - .filter_map(|result| match result { - Ok(node) => Some(node), - Err(error) => { - errors.push(format!("{:?}", error)); - None - } - }) - .collect::>(); - if result.is_empty() { - Err(Error::Custom(format!("Failed to connect: {:?}", errors))) - } else { - Ok(result) - } - } - pub async fn get_node(&self, encoding: Encoding, network_id: NetworkId) -> Result { self.fetch(encoding, network_id).await } @@ -118,3 +170,37 @@ impl Resolver { Ok(nodes.url.clone()) } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_resolver_config_1() { + let toml = r#" + [[group]] + enable = true + template = "https://*.example.org" + nodes = ["alpha", "beta", "gamma", "delta", "epsilon", "zeta", "eta", "theta"] + + [[group]] + enable = true + template = "https://*.example.com" + nodes = ["iota", "kappa", "lambda", "mu", "nu", "xi", "omicron", "pi"] + + [[resolver]] + enable = true + address = "http://127.0.0.1:8888" + "#; + + let urls = try_parse_resolvers(toml).expect("TOML: Unable to parse RPC Resolver list"); + // println!("{:#?}", urls); + assert_eq!(urls.len(), 17); + } + + #[test] + fn test_resolver_config_2() { + let _urls = try_parse_resolvers(RESOLVER_CONFIG).expect("TOML: Unable to parse RPC Resolver list"); + // println!("{:#?}", urls); + } +} diff --git a/rpc/wrpc/proxy/src/main.rs b/rpc/wrpc/proxy/src/main.rs index f261bb0..6680a3c 100644 --- a/rpc/wrpc/proxy/src/main.rs +++ b/rpc/wrpc/proxy/src/main.rs @@ -90,13 +90,15 @@ async fn main() -> Result<()> { rpc_handler.clone(), router.interface.clone(), Some(counters), + false, ); log_info!("Spectre wRPC server is listening on {}", options.listen_address); log_info!("Using `{encoding}` protocol encoding"); let config = WebSocketConfig { max_message_size: Some(1024 * 1024 * 1024), ..Default::default() }; - server.listen(&options.listen_address, Some(config)).await?; + let listener = server.bind(&options.listen_address).await?; + server.listen(listener, Some(config)).await?; Ok(()) } diff --git a/rpc/wrpc/resolver/src/args.rs b/rpc/wrpc/resolver/src/args.rs deleted file mode 100644 index 7a526b9..0000000 --- a/rpc/wrpc/resolver/src/args.rs +++ /dev/null @@ -1,54 +0,0 @@ -pub use clap::Parser; -use std::str::FromStr; - -#[derive(Default, Parser, Debug)] -#[command(version, about, long_about = None)] -pub struct Args { - /// HTTP server port - #[arg(long, default_value = "127.0.0.1:8888")] - pub listen: String, - - /// Optional rate limit in the form `:`, where `requests` is the number of requests allowed per specified number of `seconds` - #[arg(long = "rate-limit", value_name = "REQUESTS:SECONDS")] - pub rate_limit: Option, - - /// Verbose mode - #[arg(short, long, default_value = "false")] - pub verbose: bool, - - /// Show node data on each election - #[arg(short, long, default_value = "false")] - pub election: bool, - - /// Enable resolver status access via `/status` - #[arg(long, default_value = "false")] - pub status: bool, -} - -#[derive(Clone, Debug)] -pub struct RateLimit { - pub requests: u64, - pub period: u64, -} - -impl FromStr for RateLimit { - type Err = String; - - fn from_str(s: &str) -> Result { - let parts = s.split_once(':'); - let (requests, period) = match parts { - None | Some(("", _)) | Some((_, "")) => { - return Err("invalid rate limit, must be `:`".to_string()); - } - Some(x) => x, - }; - let requests = requests - .parse() - .map_err(|_| format!("Unable to parse number of requests, the value must be an integer, supplied: {:?}", requests))?; - let period = period.parse().map_err(|_| { - format!("Unable to parse period, the value must be an integer specifying number of seconds, supplied: {:?}", period) - })?; - - Ok(RateLimit { requests, period }) - } -} diff --git a/rpc/wrpc/resolver/src/connection.rs b/rpc/wrpc/resolver/src/connection.rs index 370faa3..8354355 100644 --- a/rpc/wrpc/resolver/src/connection.rs +++ b/rpc/wrpc/resolver/src/connection.rs @@ -187,7 +187,7 @@ impl Connection { self.is_synced.store(is_synced, Ordering::Relaxed); if is_synced { - match self.client.get_metrics(false, true, false, false).await { + match self.client.get_metrics(false, true, false, false, false, false).await { Ok(metrics) => { if let Some(connection_metrics) = metrics.connection_metrics { // update diff --git a/rpc/wrpc/resolver/src/error.rs b/rpc/wrpc/resolver/src/error.rs deleted file mode 100644 index 918ae55..0000000 --- a/rpc/wrpc/resolver/src/error.rs +++ /dev/null @@ -1,53 +0,0 @@ -use spectre_wrpc_client::error::Error as RpcError; -use thiserror::Error; -use toml::de::Error as TomlError; - -#[derive(Error, Debug)] -pub enum Error { - #[error("{0}")] - Custom(String), - - #[error("RPC error: {0}")] - Rpc(#[from] RpcError), - - #[error("TOML error: {0}")] - Toml(#[from] TomlError), - - #[error("IO Error: {0}")] - Io(#[from] std::io::Error), - - #[error(transparent)] - Serde(#[from] serde_json::Error), - - #[error("Connection Metrics")] - ConnectionMetrics, - #[error("Metrics")] - Metrics, - #[error("Sync")] - Sync, - #[error("Status")] - Status, - - #[error("Channel send error")] - ChannelSend, - #[error("Channel try send error")] - TryChannelSend, -} - -impl Error { - pub fn custom(msg: T) -> Self { - Error::Custom(msg.to_string()) - } -} - -impl From> for Error { - fn from(_: workflow_core::channel::SendError) -> Self { - Error::ChannelSend - } -} - -impl From> for Error { - fn from(_: workflow_core::channel::TrySendError) -> Self { - Error::TryChannelSend - } -} diff --git a/rpc/wrpc/resolver/src/log.rs b/rpc/wrpc/resolver/src/log.rs deleted file mode 100644 index 5f66416..0000000 --- a/rpc/wrpc/resolver/src/log.rs +++ /dev/null @@ -1,44 +0,0 @@ -pub mod impls { - use console::style; - use std::fmt; - - pub fn log_success(source: &str, args: &fmt::Arguments<'_>) { - println!("{:>12} {}", style(source).green().bold(), args); - } - - pub fn log_warn(source: &str, args: &fmt::Arguments<'_>) { - println!("{:>12} {}", style(source).yellow().bold(), args); - } - - pub fn log_error(source: &str, args: &fmt::Arguments<'_>) { - println!("{:>12} {}", style(source).red().bold(), args); - } -} - -#[macro_export] -macro_rules! log_success { - ($target:expr, $($t:tt)*) => ( - $crate::log::impls::log_success($target, &format_args!($($t)*)) - ) -} - -pub use log_success; - -#[macro_export] -macro_rules! log_warn { - - ($target:expr, $($t:tt)*) => ( - $crate::log::impls::log_warn($target, &format_args!($($t)*)) - ) -} - -pub use log_warn; - -#[macro_export] -macro_rules! log_error { - ($target:expr, $($t:tt)*) => ( - $crate::log::impls::log_error($target, &format_args!($($t)*)) - ) -} - -pub use log_error; diff --git a/rpc/wrpc/resolver/src/main.rs b/rpc/wrpc/resolver/src/main.rs deleted file mode 100644 index 2df7dd5..0000000 --- a/rpc/wrpc/resolver/src/main.rs +++ /dev/null @@ -1,41 +0,0 @@ -mod args; -mod connection; -mod error; -pub mod imports; -mod log; -mod monitor; -mod node; -mod panic; -mod params; -mod result; -mod server; -mod transport; - -use args::*; -use result::Result; -use std::sync::Arc; - -#[tokio::main] -async fn main() { - if let Err(error) = run().await { - eprintln!("Error: {}", error); - std::process::exit(1); - } -} - -async fn run() -> Result<()> { - let args = Arc::new(Args::parse()); - - workflow_log::set_log_level(workflow_log::LevelFilter::Info); - panic::init_ungraceful_panic_handler(); - - println!(); - println!("Spectre wRPC Resolver v{} starting...", env!("CARGO_PKG_VERSION")); - - monitor::init(&args); - let (listener, app) = server::server(&args).await?; - monitor::start().await?; - axum::serve(listener, app).await?; - monitor::stop().await?; - Ok(()) -} diff --git a/rpc/wrpc/resolver/src/monitor.rs b/rpc/wrpc/resolver/src/monitor.rs deleted file mode 100644 index 748a514..0000000 --- a/rpc/wrpc/resolver/src/monitor.rs +++ /dev/null @@ -1,241 +0,0 @@ -use crate::connection::{Connection, Descriptor}; -use crate::imports::*; - -static MONITOR: OnceLock> = OnceLock::new(); - -pub fn init(args: &Arc) { - MONITOR.set(Arc::new(Monitor::new(args))).unwrap(); -} - -pub fn monitor() -> &'static Arc { - MONITOR.get().unwrap() -} - -pub async fn start() -> Result<()> { - monitor().start().await -} - -pub async fn stop() -> Result<()> { - monitor().stop().await -} - -/// Monitor receives updates from [Connection] monitoring tasks -/// and updates the descriptors for each [Params] based on the -/// connection store (number of connections * bias). -pub struct Monitor { - args: Arc, - connections: RwLock>>>, - descriptors: RwLock>, - channel: Channel, - shutdown_ctl: DuplexChannel<()>, -} - -impl Default for Monitor { - fn default() -> Self { - Self { - args: Arc::new(Args::default()), - connections: Default::default(), - descriptors: Default::default(), - channel: Channel::unbounded(), - shutdown_ctl: DuplexChannel::oneshot(), - } - } -} - -impl fmt::Debug for Monitor { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.debug_struct("Monitor") - .field("verbose", &self.verbose()) - .field("connections", &self.connections) - .field("descriptors", &self.descriptors) - .finish() - } -} - -impl Monitor { - pub fn new(args: &Arc) -> Self { - Self { args: args.clone(), ..Default::default() } - } - - pub fn verbose(&self) -> bool { - self.args.verbose - } - - pub fn connections(&self) -> AHashMap>> { - self.connections.read().unwrap().clone() - } - - /// Process an update to `Server.toml` removing or adding node connections accordingly. - pub async fn update_nodes(&self, nodes: Vec>) -> Result<()> { - let mut connections = self.connections(); - - for params in PathParams::iter() { - let nodes = nodes.iter().filter(|node| node.params() == params).collect::>(); - - let list = connections.entry(params).or_default(); - - let create: Vec<_> = nodes.iter().filter(|node| !list.iter().any(|connection| connection.node == ***node)).collect(); - - let remove: Vec<_> = - list.iter().filter(|connection| !nodes.iter().any(|node| connection.node == **node)).cloned().collect(); - - for node in create { - let created = Arc::new(Connection::try_new((*node).clone(), self.channel.sender.clone(), &self.args)?); - created.start()?; - list.push(created); - } - - for removed in remove { - removed.stop().await?; - list.retain(|c| c.node != removed.node); - } - } - - *self.connections.write().unwrap() = connections; - - // flush all params to the update channel to refresh selected descriptors - PathParams::iter().for_each(|param| self.channel.sender.try_send(param).unwrap()); - - Ok(()) - } - - pub async fn start(self: &Arc) -> Result<()> { - let toml = std::fs::read_to_string(Path::new("Servers.toml"))?; - let nodes = crate::node::try_parse_nodes(toml.as_str())?; - - let this = self.clone(); - spawn(async move { - if let Err(error) = this.task().await { - println!("NodeConnection task error: {:?}", error); - } - }); - - self.update_nodes(nodes).await?; - - Ok(()) - } - - pub async fn stop(&self) -> Result<()> { - self.shutdown_ctl.signal(()).await.expect("Monitor shutdown signal error"); - Ok(()) - } - - async fn task(self: Arc) -> Result<()> { - let receiver = self.channel.receiver.clone(); - let shutdown_ctl_receiver = self.shutdown_ctl.request.receiver.clone(); - let shutdown_ctl_sender = self.shutdown_ctl.response.sender.clone(); - - loop { - select! { - - msg = receiver.recv().fuse() => { - match msg { - Ok(params) => { - - // run node elections - - let mut connections = self.connections() - .get(¶ms) - .expect("Monitor: expecting existing connection params") - .clone() - .into_iter() - .filter(|connection|connection.online()) - .collect::>(); - if connections.is_empty() { - self.descriptors.write().unwrap().remove(¶ms); - } else { - connections.sort_by_key(|connection| connection.score()); - - if self.args.election { - log_success!("",""); - connections.iter().for_each(|connection| { - log_warn!("Node","{}", connection); - }); - } - - if let Some(descriptor) = connections.first().unwrap().descriptor() { - let mut descriptors = self.descriptors.write().unwrap(); - - // extra debug output & monitoring - if self.args.verbose || self.args.election { - if let Some(current) = descriptors.get(¶ms) { - if current.connection.node.id != descriptor.connection.node.id { - log_success!("Election","{}", descriptor.connection); - descriptors.insert(params,descriptor); - } else { - log_success!("Keep","{}", descriptor.connection); - } - } else { - log_success!("Default","{}", descriptor.connection); - descriptors.insert(params,descriptor); - } - } else { - descriptors.insert(params,descriptor); - } - } - - if self.args.election && self.args.verbose { - log_success!("",""); - } - } - } - Err(err) => { - println!("Monitor: error while receiving update message: {err}"); - } - } - - } - _ = shutdown_ctl_receiver.recv().fuse() => { - break; - }, - - } - } - - shutdown_ctl_sender.send(()).await.unwrap(); - - Ok(()) - } - - /// Get the status of all nodes as a JSON string (available via `/status` endpoint if enabled). - pub fn get_all_json(&self) -> String { - let connections = self.connections(); - let nodes = connections.values().flatten().map(Status::from).collect::>(); - serde_json::to_string(&nodes).unwrap() - } - - /// Get JSON string representing node information (id, url, provider, link) - pub fn get_json(&self, params: &PathParams) -> Option { - self.descriptors.read().unwrap().get(params).cloned().map(|descriptor| descriptor.json) - } -} - -#[derive(Serialize)] -pub struct Status<'a> { - pub id: &'a str, - pub url: &'a str, - #[serde(skip_serializing_if = "Option::is_none")] - pub provider_name: Option<&'a str>, - #[serde(skip_serializing_if = "Option::is_none")] - pub provider_url: Option<&'a str>, - pub transport: Transport, - pub encoding: WrpcEncoding, - pub network: NetworkId, - pub online: bool, - pub status: &'static str, -} - -impl<'a> From<&'a Arc> for Status<'a> { - fn from(connection: &'a Arc) -> Self { - let url = connection.node.address.as_str(); - let provider_name = connection.node.provider.as_ref().map(|provider| provider.name.as_str()); - let provider_url = connection.node.provider.as_ref().map(|provider| provider.url.as_str()); - let id = connection.node.id_string.as_str(); - let transport = connection.node.transport; - let encoding = connection.node.encoding; - let network = connection.node.network; - let status = connection.status(); - let online = connection.online(); - Self { id, url, provider_name, provider_url, transport, encoding, network, status, online } - } -} diff --git a/rpc/wrpc/resolver/src/node.rs b/rpc/wrpc/resolver/src/node.rs deleted file mode 100644 index d096896..0000000 --- a/rpc/wrpc/resolver/src/node.rs +++ /dev/null @@ -1,75 +0,0 @@ -use crate::imports::*; -use xxhash_rust::xxh3::xxh3_64; - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Provider { - pub name: String, - pub url: String, -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct Node { - #[serde(skip)] - pub id: u64, - #[serde(skip)] - pub id_string: String, - - pub name: Option, - pub location: Option, - pub address: String, - pub transport: Transport, - pub encoding: WrpcEncoding, - pub network: NetworkId, - pub enable: Option, - pub bias: Option, - pub version: Option, - pub provider: Option, -} - -impl Eq for Node {} - -impl PartialEq for Node { - fn eq(&self, other: &Self) -> bool { - self.address == other.address - } -} - -impl std::fmt::Display for Node { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let title = self.name.clone().unwrap_or(self.address.to_string()); - write!(f, "{}", title) - } -} - -impl Node { - pub fn params(&self) -> PathParams { - PathParams::new(self.encoding, self.network) - } -} - -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct NodeConfig { - #[serde(rename = "node")] - nodes: Vec, -} - -pub fn try_parse_nodes(toml: &str) -> Result>> { - let nodes: Vec> = toml::from_str::(toml)? - .nodes - .into_iter() - .filter_map(|mut node| { - let id = xxh3_64(node.address.as_bytes()); - let id_string = format!("{id:x}"); - node.id = id; - node.id_string = id_string.chars().take(8).collect(); - node.enable.unwrap_or(true).then_some(node).map(Arc::new) - }) - .collect::>(); - Ok(nodes) -} - -impl AsRef for Node { - fn as_ref(&self) -> &Node { - self - } -} diff --git a/rpc/wrpc/resolver/src/panic.rs b/rpc/wrpc/resolver/src/panic.rs deleted file mode 100644 index 7e6d78a..0000000 --- a/rpc/wrpc/resolver/src/panic.rs +++ /dev/null @@ -1,10 +0,0 @@ -use std::panic; - -pub fn init_ungraceful_panic_handler() { - let default_hook = panic::take_hook(); - panic::set_hook(Box::new(move |panic_info| { - default_hook(panic_info); - println!("Exiting..."); - std::process::exit(1); - })); -} diff --git a/rpc/wrpc/resolver/src/params.rs b/rpc/wrpc/resolver/src/params.rs deleted file mode 100644 index 7e31b69..0000000 --- a/rpc/wrpc/resolver/src/params.rs +++ /dev/null @@ -1,146 +0,0 @@ -use serde::{de, Deserializer, Serializer}; - -use crate::imports::*; -use std::{fmt, str::FromStr}; -// use convert_case::{Case, Casing}; - -#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Hash)] -pub struct PathParams { - pub encoding: WrpcEncoding, - pub network: NetworkId, -} - -impl PathParams { - pub fn new(encoding: WrpcEncoding, network: NetworkId) -> Self { - Self { encoding, network } - } - - pub fn iter() -> impl Iterator { - NetworkId::iter().flat_map(move |network_id| WrpcEncoding::iter().map(move |encoding| PathParams::new(*encoding, network_id))) - } -} - -impl fmt::Display for PathParams { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - write!(f, "{}:{}", self.encoding.to_string().to_lowercase(), self.network) - } -} - -// --- - -#[derive(Debug, Deserialize)] -pub struct QueryParams { - // Accessible via a query string like "?access=utxo-index+tx-index+block-dag+metrics+visualizer+mining" - pub access: Option, -} - -#[derive(Debug, Deserialize, Serialize, Clone, Copy, PartialEq, Eq, Hash)] -#[serde(rename_all = "kebab-case")] -pub enum AccessType { - Transact, // UTXO and TX index, submit transaction, single mempool entry - Mempool, // Full mempool data access - BlockDag, // Access to Blocks - Network, // Network data access (peers, ban, etc.) - Metrics, // Access to Metrics - Visualizer, // Access to Visualization data feeds - Mining, // Access to submit block, GBT, etc. -} - -impl AccessType { - pub fn iter() -> impl Iterator { - [ - AccessType::Transact, - AccessType::Mempool, - AccessType::BlockDag, - AccessType::Network, - AccessType::Metrics, - AccessType::Visualizer, - AccessType::Mining, - ] - .into_iter() - } -} - -impl fmt::Display for AccessType { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - let s = match self { - AccessType::Transact => "transact", - AccessType::Mempool => "mempool", - AccessType::BlockDag => "block-dag", - AccessType::Network => "network", - AccessType::Metrics => "metrics", - AccessType::Visualizer => "visualizer", - AccessType::Mining => "mining", - }; - write!(f, "{s}") - } -} - -impl FromStr for AccessType { - type Err = String; - fn from_str(s: &str) -> std::result::Result { - match s { - "transact" => Ok(AccessType::Transact), - "mempool" => Ok(AccessType::Mempool), - "block-dag" => Ok(AccessType::BlockDag), - "network" => Ok(AccessType::Network), - "metrics" => Ok(AccessType::Metrics), - "visualizer" => Ok(AccessType::Visualizer), - "mining" => Ok(AccessType::Mining), - _ => Err(format!("Invalid access type: {}", s)), - } - } -} - -#[derive(Debug, Clone)] -pub struct AccessList { - pub access: Vec, -} - -impl std::fmt::Display for AccessList { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.access.iter().map(|access| access.to_string()).collect::>().join(" ")) - } -} - -impl FromStr for AccessList { - type Err = String; - - fn from_str(s: &str) -> std::result::Result { - let access = s.split(' ').map(|s| s.parse::()).collect::, _>>()?; - Ok(AccessList { access }) - } -} - -impl Serialize for AccessList { - fn serialize(&self, serializer: S) -> std::result::Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -struct AccessListVisitor; -impl<'de> de::Visitor<'de> for AccessListVisitor { - type Value = AccessList; - fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result { - formatter.write_str("a string containing list of permissions separated by a '+'") - } - - fn visit_str(self, value: &str) -> std::result::Result - where - E: de::Error, - { - AccessList::from_str(value).map_err(|err| de::Error::custom(err.to_string())) - } -} - -impl<'de> Deserialize<'de> for AccessList { - fn deserialize(deserializer: D) -> std::result::Result - where - D: Deserializer<'de>, - { - deserializer.deserialize_str(AccessListVisitor) - } -} diff --git a/rpc/wrpc/resolver/src/result.rs b/rpc/wrpc/resolver/src/result.rs deleted file mode 100644 index 605dc25..0000000 --- a/rpc/wrpc/resolver/src/result.rs +++ /dev/null @@ -1 +0,0 @@ -pub type Result = std::result::Result; diff --git a/rpc/wrpc/resolver/src/server.rs b/rpc/wrpc/resolver/src/server.rs deleted file mode 100644 index 3717a6e..0000000 --- a/rpc/wrpc/resolver/src/server.rs +++ /dev/null @@ -1,149 +0,0 @@ -use crate::imports::*; -use crate::monitor::monitor; -use axum::{ - async_trait, - extract::{path::ErrorKind, rejection::PathRejection, FromRequestParts, Query}, - http::{header, request::Parts, HeaderValue, StatusCode}, - response::IntoResponse, - routing::get, - // Json, - Router, -}; -use tokio::net::TcpListener; - -use axum::{error_handling::HandleErrorLayer, BoxError}; -use std::time::Duration; -use tower::{buffer::BufferLayer, limit::RateLimitLayer, ServiceBuilder}; -use tower_http::cors::{Any, CorsLayer}; - -pub async fn server(args: &Args) -> Result<(TcpListener, Router)> { - // initialize tracing - tracing_subscriber::fmt::init(); - - let app = Router::new().route("/v1/wrpc/:encoding/:network", get(get_elected_node)); - - let app = if args.status { - log_warn!("Routes", "Enabling `/status` route"); - app.route("/status", get(get_status_all_nodes)) - } else { - log_success!("Routes", "Disabling `/status` route"); - app - }; - - let app = if let Some(rate_limit) = args.rate_limit.as_ref() { - log_success!("Limits", "Setting rate limit to: {} requests per {} seconds", rate_limit.requests, rate_limit.period); - app.layer( - ServiceBuilder::new() - .layer(HandleErrorLayer::new(|err: BoxError| async move { - (StatusCode::INTERNAL_SERVER_ERROR, format!("Unhandled error: {}", err)) - })) - .layer(BufferLayer::new(1024)) - .layer(RateLimitLayer::new(rate_limit.requests, Duration::from_secs(rate_limit.period))), - ) - } else { - log_warn!("Limits", "Rate limit is disabled"); - app - }; - - let app = app.layer(CorsLayer::new().allow_origin(Any)); - - log_success!("Server", "Listening on http://{}", args.listen.as_str()); - let listener = tokio::net::TcpListener::bind(args.listen.as_str()).await.unwrap(); - Ok((listener, app)) -} - -// respond with a JSON object containing the status of all nodes -async fn get_status_all_nodes() -> impl IntoResponse { - let json = monitor().get_all_json(); - (StatusCode::OK, [(header::CONTENT_TYPE, HeaderValue::from_static(mime::APPLICATION_JSON.as_ref()))], json).into_response() -} - -// respond with a JSON object containing the elected node -async fn get_elected_node(Query(_query): Query, Path(params): Path) -> impl IntoResponse { - // println!("params: {:?}", params); - // println!("query: {:?}", query); - - if let Some(json) = monitor().get_json(¶ms) { - ([(header::CONTENT_TYPE, HeaderValue::from_static(mime::APPLICATION_JSON.as_ref()))], json).into_response() - } else { - ( - StatusCode::NOT_FOUND, - [(header::CONTENT_TYPE, HeaderValue::from_static(mime::TEXT_PLAIN_UTF_8.as_ref()))], - "NOT FOUND".to_string(), - ) - .into_response() - } -} - -// We define our own `Path` extractor that customizes the error from `axum::extract::Path` -struct Path(T); - -#[async_trait] -impl FromRequestParts for Path -where - // these trait bounds are copied from `impl FromRequest for axum::extract::path::Path` - T: DeserializeOwned + Send, - S: Send + Sync, -{ - type Rejection = (StatusCode, axum::Json); - - async fn from_request_parts(parts: &mut Parts, state: &S) -> std::result::Result { - match axum::extract::Path::::from_request_parts(parts, state).await { - Ok(value) => Ok(Self(value.0)), - Err(rejection) => { - let (status, body) = match rejection { - PathRejection::FailedToDeserializePathParams(inner) => { - let mut status = StatusCode::BAD_REQUEST; - - let kind = inner.into_kind(); - let body = match &kind { - ErrorKind::WrongNumberOfParameters { .. } => PathError { message: kind.to_string(), location: None }, - - ErrorKind::ParseErrorAtKey { key, .. } => { - PathError { message: kind.to_string(), location: Some(key.clone()) } - } - - ErrorKind::ParseErrorAtIndex { index, .. } => { - PathError { message: kind.to_string(), location: Some(index.to_string()) } - } - - ErrorKind::ParseError { .. } => PathError { message: kind.to_string(), location: None }, - - ErrorKind::InvalidUtf8InPathParam { key } => { - PathError { message: kind.to_string(), location: Some(key.clone()) } - } - - ErrorKind::UnsupportedType { .. } => { - // this error is caused by the programmer using an unsupported type - // (such as nested maps) so respond with `500` instead - status = StatusCode::INTERNAL_SERVER_ERROR; - PathError { message: kind.to_string(), location: None } - } - - ErrorKind::Message(msg) => PathError { message: msg.clone(), location: None }, - - _ => PathError { message: format!("Unhandled deserialization error: {kind}"), location: None }, - }; - - (status, body) - } - PathRejection::MissingPathParams(error) => { - (StatusCode::INTERNAL_SERVER_ERROR, PathError { message: error.to_string(), location: None }) - } - _ => ( - StatusCode::INTERNAL_SERVER_ERROR, - PathError { message: format!("Unhandled path rejection: {rejection}"), location: None }, - ), - }; - - Err((status, axum::Json(body))) - } - } - } -} - -#[derive(Serialize)] -struct PathError { - message: String, - location: Option, -} diff --git a/rpc/wrpc/resolver/src/transport.rs b/rpc/wrpc/resolver/src/transport.rs deleted file mode 100644 index ccfd6de..0000000 --- a/rpc/wrpc/resolver/src/transport.rs +++ /dev/null @@ -1,8 +0,0 @@ -use serde::{Deserialize, Serialize}; - -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)] -#[serde(rename_all = "lowercase")] -pub enum Transport { - Grpc, - Wrpc, -} diff --git a/rpc/wrpc/server/Cargo.toml b/rpc/wrpc/server/Cargo.toml index 1d0cdcb..66d7004 100644 --- a/rpc/wrpc/server/Cargo.toml +++ b/rpc/wrpc/server/Cargo.toml @@ -33,8 +33,9 @@ tokio.workspace = true workflow-core.workspace = true workflow-log.workspace = true workflow-rpc.workspace = true +workflow-serializer.workspace = true -[target.'cfg(unix)'.dependencies] +[target.x86_64-unknown-linux-gnu.dependencies] # Adding explicitely the openssl dependency here is needed for a successful build with zigbuild # as used in the release deployment in GitHub CI # see: https://github.com/rust-cross/cargo-zigbuild/issues/127 diff --git a/rpc/wrpc/server/src/address.rs b/rpc/wrpc/server/src/address.rs index d52d7c2..c8b847f 100644 --- a/rpc/wrpc/server/src/address.rs +++ b/rpc/wrpc/server/src/address.rs @@ -43,7 +43,6 @@ impl WrpcNetAddress { } } } - impl FromStr for WrpcNetAddress { type Err = AddrParseError; fn from_str(s: &str) -> Result { diff --git a/rpc/wrpc/server/src/connection.rs b/rpc/wrpc/server/src/connection.rs index fea9ce9..05cfc48 100644 --- a/rpc/wrpc/server/src/connection.rs +++ b/rpc/wrpc/server/src/connection.rs @@ -16,6 +16,7 @@ use workflow_rpc::{ server::{prelude::*, result::Result as WrpcResult}, types::{MsgT, OpsT}, }; +use workflow_serializer::prelude::*; // // FIXME: Use workflow_rpc::encoding::Encoding directly in the ConnectionT implementation by deriving Hash, Eq and PartialEq in situ @@ -157,7 +158,7 @@ impl ConnectionT for Connection { fn into_message(notification: &Self::Notification, encoding: &Self::Encoding) -> Self::Message { let op: RpcApiOps = notification.event_type().into(); - Self::create_serialized_notification_message(encoding.clone().into(), op, notification.clone()).unwrap() + Self::create_serialized_notification_message(encoding.clone().into(), op, Serializable(notification.clone())).unwrap() } async fn send(&self, message: Self::Message) -> core::result::Result<(), Self::Error> { diff --git a/rpc/wrpc/server/src/router.rs b/rpc/wrpc/server/src/router.rs index f9e5942..68b4d4d 100644 --- a/rpc/wrpc/server/src/router.rs +++ b/rpc/wrpc/server/src/router.rs @@ -4,6 +4,7 @@ use spectre_rpc_core::{api::ops::RpcApiOps, prelude::*}; use spectre_rpc_macros::build_wrpc_server_interface; use std::sync::Arc; use workflow_rpc::server::prelude::*; +use workflow_serializer::prelude::*; /// A wrapper that creates an [`Interface`] instance and initializes /// RPC methods and notifications against this interface. The interface @@ -32,6 +33,7 @@ impl Router { Connection, RpcApiOps, [ + Ping, AddPeer, Ban, EstimateNetworkHashesPerSecond, @@ -55,15 +57,16 @@ impl Router { GetMempoolEntriesByAddresses, GetMempoolEntry, GetMetrics, + GetConnections, GetPeerAddresses, GetServerInfo, GetSink, GetSinkBlueScore, GetSubnetwork, GetSyncStatus, + GetSystemInfo, GetUtxosByAddresses, GetVirtualChainFromBlock, - Ping, ResolveFinalityConflict, Shutdown, SubmitBlock, @@ -75,22 +78,22 @@ impl Router { interface.method( RpcApiOps::Subscribe, - workflow_rpc::server::Method::new(move |manager: Server, connection: Connection, scope: Scope| { + workflow_rpc::server::Method::new(move |manager: Server, connection: Connection, scope: Serializable| { Box::pin(async move { - manager.start_notify(&connection, scope).await.map_err(|err| err.to_string())?; - Ok(SubscribeResponse::new(connection.id())) + manager.start_notify(&connection, scope.into_inner()).await.map_err(|err| err.to_string())?; + Ok(Serializable(SubscribeResponse::new(connection.id()))) }) }), ); interface.method( RpcApiOps::Unsubscribe, - workflow_rpc::server::Method::new(move |manager: Server, connection: Connection, scope: Scope| { + workflow_rpc::server::Method::new(move |manager: Server, connection: Connection, scope: Serializable| { Box::pin(async move { - manager.stop_notify(&connection, scope).await.unwrap_or_else(|err| { + manager.stop_notify(&connection, scope.into_inner()).await.unwrap_or_else(|err| { workflow_log::log_trace!("wRPC server -> error calling stop_notify(): {err}"); }); - Ok(UnsubscribeResponse {}) + Ok(Serializable(UnsubscribeResponse {})) }) }), ); diff --git a/rpc/wrpc/server/src/service.rs b/rpc/wrpc/server/src/service.rs index b84136d..ddff242 100644 --- a/rpc/wrpc/server/src/service.rs +++ b/rpc/wrpc/server/src/service.rs @@ -123,6 +123,7 @@ impl WrpcService { rpc_handler.clone(), router.interface.clone(), Some(counters), + false, ); WrpcService { options, server, rpc_handler, shutdown: SingleTrigger::default() } @@ -146,10 +147,15 @@ impl WrpcService { info!("WRPC Server starting on: {}", listen_address); tokio::spawn(async move { let config = WebSocketConfig { max_message_size: Some(MAX_WRPC_MESSAGE_SIZE), ..Default::default() }; - let serve_result = self.server.listen(&listen_address, Some(config)).await; - match serve_result { - Ok(_) => info!("WRPC Server stopped on: {}", listen_address), - Err(err) => panic!("WRPC Server {listen_address} stopped with error: {err:?}"), + match self.server.bind(&listen_address).await { + Ok(listener) => { + let serve_result = self.server.listen(listener, Some(config)).await; + match serve_result { + Ok(_) => info!("WRPC Server stopped on: {}", listen_address), + Err(err) => panic!("WRPC Server {listen_address} stopped with error: {err:?}"), + } + } + Err(err) => panic!("WRPC Server bind error on {listen_address}: {err:?}"), } }); diff --git a/rpc/wrpc/wasm/Cargo.toml b/rpc/wrpc/wasm/Cargo.toml index 54616d5..60a2e93 100644 --- a/rpc/wrpc/wasm/Cargo.toml +++ b/rpc/wrpc/wasm/Cargo.toml @@ -41,5 +41,5 @@ wasm-bindgen-futures.workspace = true workflow-core.workspace = true futures.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/rpc/wrpc/wasm/src/client.rs b/rpc/wrpc/wasm/src/client.rs index 2ea4ffb..72d8a7a 100644 --- a/rpc/wrpc/wasm/src/client.rs +++ b/rpc/wrpc/wasm/src/client.rs @@ -364,19 +364,7 @@ impl RpcClient { /// Optional: Resolver node id. #[wasm_bindgen(getter, js_name = "nodeId")] pub fn resolver_node_id(&self) -> Option { - self.inner.client.node_descriptor().map(|node| node.id.clone()) - } - - /// Optional: public node provider name. - #[wasm_bindgen(getter, js_name = "providerName")] - pub fn resolver_node_provider_name(&self) -> Option { - self.inner.client.node_descriptor().and_then(|node| node.provider_name.clone()) - } - - /// Optional: public node provider URL. - #[wasm_bindgen(getter, js_name = "providerUrl")] - pub fn resolver_node_provider_url(&self) -> Option { - self.inner.client.node_descriptor().and_then(|node| node.provider_url.clone()) + self.inner.client.node_descriptor().map(|node| node.uid.clone()) } /// Connect to the Spectre RPC server. This function starts a background @@ -796,7 +784,7 @@ impl RpcClient { #[wasm_bindgen(js_name = subscribeVirtualDaaScoreChanged)] pub async fn subscribe_daa_score(&self) -> Result<()> { if let Some(listener_id) = self.listener_id() { - self.inner.client.stop_notify(listener_id, Scope::VirtualDaaScoreChanged(VirtualDaaScoreChangedScope {})).await?; + self.inner.client.start_notify(listener_id, Scope::VirtualDaaScoreChanged(VirtualDaaScoreChangedScope {})).await?; } else { log_error!("RPC unsubscribe on a closed connection"); } @@ -957,6 +945,8 @@ build_wrpc_wasm_bindgen_interface!( /// performance and status of the Spectre node. /// Returned information: Memory usage, CPU usage, network activity. GetMetrics, + /// Retrieves current number of network connections + GetConnections, /// Retrieves the current sink block, which is the block with /// the highest cumulative difficulty in the Spectre BlockDAG. /// Returned information: Sink block hash, sink block height. @@ -1010,6 +1000,10 @@ build_wrpc_wasm_bindgen_interface!( /// score timestamp estimate. /// Returned information: DAA score timestamp estimate. GetDaaScoreTimestampEstimate, + /// Feerate estimates + GetFeeEstimate, + /// Feerate estimates (experimental) + GetFeeEstimateExperimental, /// Retrieves the current network configuration. /// Returned information: Current network configuration. GetCurrentNetwork, @@ -1042,8 +1036,11 @@ build_wrpc_wasm_bindgen_interface!( /// Returned information: None. SubmitBlock, /// Submits a transaction to the Spectre network. - /// Returned information: None. + /// Returned information: Submitted Transaction Id. SubmitTransaction, + /// Submits an RBF transaction to the Spectre network. + /// Returned information: Submitted Transaction Id, Transaction that was replaced. + SubmitTransactionReplacement, /// Unbans a previously banned peer, allowing it to connect /// to the Spectre node again. /// Returned information: None. diff --git a/rpc/wrpc/wasm/src/resolver.rs b/rpc/wrpc/wasm/src/resolver.rs index 56d4aab..142110f 100644 --- a/rpc/wrpc/wasm/src/resolver.rs +++ b/rpc/wrpc/wasm/src/resolver.rs @@ -21,6 +21,20 @@ declare! { * Optional URLs for one or multiple resolvers. */ urls?: string[]; + /** + * Use strict TLS for RPC connections. + * If not set or `false` (default), the resolver will + * provide the best available connection regardless of + * whether this connection supports TLS or not. + * If set to `true`, the resolver will only provide + * TLS-enabled connections. + * + * This setting is ignored in the browser environment + * when the browser navigator location is `https`. + * In which case the resolver will always use TLS-enabled + * connections. + */ + tls?: boolean; } "#, } @@ -130,8 +144,8 @@ impl Resolver { impl Resolver { /// List of public Spectre Resolver URLs. #[wasm_bindgen(getter)] - pub fn urls(&self) -> ResolverArrayT { - Array::from_iter(self.resolver.urls().iter().map(|v| JsValue::from(v.as_str()))).unchecked_into() + pub fn urls(&self) -> Option { + self.resolver.urls().map(|urls| Array::from_iter(urls.iter().map(|v| JsValue::from(v.as_str()))).unchecked_into()) } /// Fetches a public Spectre wRPC endpoint for the given encoding and network identifier. @@ -163,20 +177,27 @@ impl Resolver { impl TryFrom for NativeResolver { type Error = Error; fn try_from(config: IResolverConfig) -> Result { - let resolver = config + let tls = config.get_bool("tls").unwrap_or(false); + let urls = config .get_vec("urls") .map(|urls| urls.into_iter().map(|v| v.as_string()).collect::>>()) .or_else(|_| config.dyn_into::().map(|urls| urls.into_iter().map(|v| v.as_string()).collect::>>())) - .map_err(|_| Error::custom("Invalid or missing resolver URL"))? - .map(|urls| NativeResolver::new(urls.into_iter().map(Arc::new).collect())); + .map_err(|_| Error::custom("Invalid or missing resolver URL"))?; - Ok(resolver.unwrap_or_default()) + if let Some(urls) = urls { + Ok(NativeResolver::new(Some(urls.into_iter().map(Arc::new).collect()), tls)) + } else { + Ok(NativeResolver::new(None, tls)) + } } } impl TryCastFromJs for Resolver { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result> { + fn try_cast_from<'a, R>(value: &'a R) -> Result> + where + R: AsRef + 'a, + { Ok(Self::try_ref_from_js_value_as_cast(value)?) } } diff --git a/spectred/src/daemon.rs b/spectred/src/daemon.rs index fe0ca87..ec31c6c 100644 --- a/spectred/src/daemon.rs +++ b/spectred/src/daemon.rs @@ -13,7 +13,9 @@ use spectre_grpc_server::service::GrpcService; use spectre_notify::{address::tracker::Tracker, subscription::context::SubscriptionContext}; use spectre_rpc_service::service::RpcCoreService; use spectre_txscript::caches::TxScriptCacheCounters; +use spectre_utils::git; use spectre_utils::networking::ContextualNetAddress; +use spectre_utils::sysinfo::SystemInfo; use spectre_utils_tower::counters::TowerConnectionCounters; use spectre_addressmanager::AddressManager; @@ -233,7 +235,7 @@ pub fn create_core_with_runtime(runtime: &Runtime, args: &Args, fd_total_budget: let db_dir = app_dir.join(network.to_prefixed()).join(DEFAULT_DATA_DIR); // Print package name and version - info!("{} v{}", env!("CARGO_PKG_NAME"), version()); + info!("{} v{}", env!("CARGO_PKG_NAME"), git::with_short_hash(version())); assert!(!db_dir.to_str().unwrap().is_empty()); info!("Application directory: {}", app_dir.display()); @@ -408,6 +410,8 @@ do you confirm? (answer y/n or pass --yes to the Spectred command line to confir Arc::new(perf_monitor_builder.build()) }; + let system_info = SystemInfo::default(); + let notify_service = Arc::new(NotifyService::new(notification_root.clone(), notification_recv, subscription_context.clone())); let index_service: Option> = if args.utxoindex { // Use only a single thread for none-consensus databases @@ -472,6 +476,7 @@ do you confirm? (answer y/n or pass --yes to the Spectred command line to confir perf_monitor.clone(), p2p_tower_counters.clone(), grpc_tower_counters.clone(), + system_info, )); let grpc_service_broadcasters: usize = 3; // TODO: add a command line argument or derive from other arg/config/host-related fields let grpc_service = if !args.disable_grpc { diff --git a/testing/integration/src/common/utils.rs b/testing/integration/src/common/utils.rs index 96e39d9..2047f54 100644 --- a/testing/integration/src/common/utils.rs +++ b/testing/integration/src/common/utils.rs @@ -5,6 +5,7 @@ use secp256k1::Keypair; use spectre_addresses::Address; use spectre_consensus_core::{ constants::TX_VERSION, + header::Header, sign::sign, subnets::SUBNETWORK_ID_NATIVE, tx::{ @@ -18,7 +19,7 @@ use spectre_consensus_core::{ }; use spectre_core::info; use spectre_grpc_client::GrpcClient; -use spectre_rpc_core::{api::rpc::RpcApi, BlockAddedNotification, Notification, VirtualDaaScoreChangedNotification}; +use spectre_rpc_core::{api::rpc::RpcApi, BlockAddedNotification, Notification, RpcUtxoEntry, VirtualDaaScoreChangedNotification}; use spectre_txscript::pay_to_address_script; use std::{ collections::{hash_map::Entry::Occupied, HashMap, HashSet}, @@ -170,13 +171,13 @@ pub async fn fetch_spendable_utxos( { assert!(resp_entry.address.is_some()); assert_eq!(*resp_entry.address.as_ref().unwrap(), address); - utxos.push((resp_entry.outpoint, resp_entry.utxo_entry)); + utxos.push((TransactionOutpoint::from(resp_entry.outpoint), UtxoEntry::from(resp_entry.utxo_entry))); } utxos.sort_by(|a, b| b.1.amount.cmp(&a.1.amount)); utxos } -pub fn is_utxo_spendable(entry: &UtxoEntry, virtual_daa_score: u64, coinbase_maturity: u64) -> bool { +pub fn is_utxo_spendable(entry: &RpcUtxoEntry, virtual_daa_score: u64, coinbase_maturity: u64) -> bool { let needed_confirmations = if !entry.is_coinbase { 10 } else { coinbase_maturity }; entry.block_daa_score + needed_confirmations <= virtual_daa_score } @@ -187,7 +188,8 @@ pub async fn mine_block(pay_address: Address, submitting_client: &GrpcClient, li // Mine a block let template = submitting_client.get_block_template(pay_address.clone(), vec![]).await.unwrap(); - let block_hash = template.block.header.hash; + let header: Header = (&template.block.header).into(); + let block_hash = header.hash; submitting_client.submit_block(template.block, false).await.unwrap(); // Wait for each listening client to get notified the submitted block was added to the DAG diff --git a/testing/integration/src/daemon_integration_tests.rs b/testing/integration/src/daemon_integration_tests.rs index b01083b..ef934d8 100644 --- a/testing/integration/src/daemon_integration_tests.rs +++ b/testing/integration/src/daemon_integration_tests.rs @@ -8,6 +8,7 @@ use rand::thread_rng; use spectre_addresses::Address; use spectre_alloc::init_allocator_with_default_settings; use spectre_consensus::params::SIMNET_PARAMS; +use spectre_consensus_core::header::Header; use spectre_consensusmanager::ConsensusManager; use spectre_core::{task::runtime::AsyncRuntime, trace}; use spectre_grpc_client::GrpcClient; @@ -77,7 +78,8 @@ async fn daemon_mining_test() { .get_block_template(Address::new(spectred1.network.into(), spectre_addresses::Version::PubKey, &[0; 32]), vec![]) .await .unwrap(); - last_block_hash = Some(template.block.header.hash); + let header: Header = (&template.block.header).into(); + last_block_hash = Some(header.hash); rpc_client1.submit_block(template.block, false).await.unwrap(); while let Ok(notification) = match tokio::time::timeout(Duration::from_secs(1), event_receiver.recv()).await { @@ -180,7 +182,8 @@ async fn daemon_utxos_propagation_test() { let mut last_block_hash = None; for i in 0..initial_blocks { let template = rpc_client1.get_block_template(miner_address.clone(), vec![]).await.unwrap(); - last_block_hash = Some(template.block.header.hash); + let header: Header = (&template.block.header).into(); + last_block_hash = Some(header.hash); rpc_client1.submit_block(template.block, false).await.unwrap(); while let Ok(notification) = match tokio::time::timeout(Duration::from_secs(1), event_receiver1.recv()).await { diff --git a/testing/integration/src/rpc_tests.rs b/testing/integration/src/rpc_tests.rs index e8f63b0..86413d3 100644 --- a/testing/integration/src/rpc_tests.rs +++ b/testing/integration/src/rpc_tests.rs @@ -4,7 +4,7 @@ use crate::common::{client_notify::ChannelNotify, daemon::Daemon}; use futures_util::future::try_join_all; use spectre_addresses::{Address, Prefix, Version}; use spectre_consensus::params::SIMNET_GENESIS; -use spectre_consensus_core::{constants::MAX_SOMPI, subnets::SubnetworkId, tx::Transaction}; +use spectre_consensus_core::{constants::MAX_SOMPI, header::Header, subnets::SubnetworkId, tx::Transaction}; use spectre_core::info; use spectre_grpc_core::ops::SpectredPayloadOps; use spectre_hashes::Hash; @@ -64,7 +64,7 @@ async fn sanity_test() { // The intent of this for/match design (emphasizing the absence of an arm with fallback pattern in the match) // is to force any implementor of a new RpcApi method to add a matching arm here and to strongly incentivize // the adding of an actual sanity test of said new method. - for op in SpectredPayloadOps::list() { + for op in SpectredPayloadOps::iter() { let network_id = daemon.network; let task: JoinHandle<()> = match op { SpectredPayloadOps::SubmitBlock => { @@ -79,21 +79,24 @@ async fn sanity_test() { .unwrap(); // Before submitting a first block, the sink is the genesis, - let response = rpc_client.get_sink_call(GetSinkRequest {}).await.unwrap(); + let response = rpc_client.get_sink_call(None, GetSinkRequest {}).await.unwrap(); assert_eq!(response.sink, SIMNET_GENESIS.hash); - let response = rpc_client.get_sink_blue_score_call(GetSinkBlueScoreRequest {}).await.unwrap(); + let response = rpc_client.get_sink_blue_score_call(None, GetSinkBlueScoreRequest {}).await.unwrap(); assert_eq!(response.blue_score, 0); // the block count is 0 - let response = rpc_client.get_block_count_call(GetBlockCountRequest {}).await.unwrap(); + let response = rpc_client.get_block_count_call(None, GetBlockCountRequest {}).await.unwrap(); assert_eq!(response.block_count, 0); // and the virtual chain is the genesis only let response = rpc_client - .get_virtual_chain_from_block_call(GetVirtualChainFromBlockRequest { - start_hash: SIMNET_GENESIS.hash, - include_accepted_transaction_ids: false, - }) + .get_virtual_chain_from_block_call( + None, + GetVirtualChainFromBlockRequest { + start_hash: SIMNET_GENESIS.hash, + include_accepted_transaction_ids: false, + }, + ) .await .unwrap(); assert!(response.added_chain_block_hashes.is_empty()); @@ -101,14 +104,21 @@ async fn sanity_test() { // Get a block template let GetBlockTemplateResponse { block, is_synced } = rpc_client - .get_block_template_call(GetBlockTemplateRequest { - pay_address: Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32]), - extra_data: Vec::new(), - }) + .get_block_template_call( + None, + GetBlockTemplateRequest { + pay_address: Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32]), + extra_data: Vec::new(), + }, + ) .await .unwrap(); assert!(!is_synced); + // Compute the expected block hash for the received block + let header: Header = (&block.header).into(); + let block_hash = header.hash; + // Submit the template (no mining, in simnet PoW is skipped) let response = rpc_client.submit_block(block.clone(), false).await.unwrap(); assert_eq!(response.report, SubmitBlockReport::Success); @@ -131,22 +141,25 @@ async fn sanity_test() { } // After submitting a first block, the sink is the submitted block, - let response = rpc_client.get_sink_call(GetSinkRequest {}).await.unwrap(); - assert_eq!(response.sink, block.header.hash); + let response = rpc_client.get_sink_call(None, GetSinkRequest {}).await.unwrap(); + assert_eq!(response.sink, block_hash); // the block count is 1 - let response = rpc_client.get_block_count_call(GetBlockCountRequest {}).await.unwrap(); + let response = rpc_client.get_block_count_call(None, GetBlockCountRequest {}).await.unwrap(); assert_eq!(response.block_count, 1); // and the virtual chain from genesis contains the added block let response = rpc_client - .get_virtual_chain_from_block_call(GetVirtualChainFromBlockRequest { - start_hash: SIMNET_GENESIS.hash, - include_accepted_transaction_ids: false, - }) + .get_virtual_chain_from_block_call( + None, + GetVirtualChainFromBlockRequest { + start_hash: SIMNET_GENESIS.hash, + include_accepted_transaction_ids: false, + }, + ) .await .unwrap(); - assert!(response.added_chain_block_hashes.contains(&block.header.hash)); + assert!(response.added_chain_block_hashes.contains(&block_hash)); assert!(response.removed_chain_block_hashes.is_empty()); }) } @@ -158,7 +171,7 @@ async fn sanity_test() { SpectredPayloadOps::GetCurrentNetwork => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_current_network_call(GetCurrentNetworkRequest {}).await.unwrap(); + let response = rpc_client.get_current_network_call(None, GetCurrentNetworkRequest {}).await.unwrap(); assert_eq!(response.network, network_id.network_type); }) } @@ -166,11 +179,12 @@ async fn sanity_test() { SpectredPayloadOps::GetBlock => { let rpc_client = client.clone(); tst!(op, { - let result = rpc_client.get_block_call(GetBlockRequest { hash: 0.into(), include_transactions: false }).await; + let result = + rpc_client.get_block_call(None, GetBlockRequest { hash: 0.into(), include_transactions: false }).await; assert!(result.is_err()); let response = rpc_client - .get_block_call(GetBlockRequest { hash: SIMNET_GENESIS.hash, include_transactions: false }) + .get_block_call(None, GetBlockRequest { hash: SIMNET_GENESIS.hash, include_transactions: false }) .await .unwrap(); assert_eq!(response.block.header.hash, SIMNET_GENESIS.hash); @@ -181,7 +195,7 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response = rpc_client - .get_blocks_call(GetBlocksRequest { include_blocks: true, include_transactions: false, low_hash: None }) + .get_blocks_call(None, GetBlocksRequest { include_blocks: true, include_transactions: false, low_hash: None }) .await .unwrap(); assert_eq!(response.blocks.len(), 1, "genesis block should be returned"); @@ -193,7 +207,7 @@ async fn sanity_test() { SpectredPayloadOps::GetInfo => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_info_call(GetInfoRequest {}).await.unwrap(); + let response = rpc_client.get_info_call(None, GetInfoRequest {}).await.unwrap(); assert_eq!(response.server_version, spectre_core::spectred_env::version().to_string()); assert_eq!(response.mempool_size, 0); assert!(response.is_utxo_indexed); @@ -220,11 +234,14 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response_result = rpc_client - .get_mempool_entry_call(GetMempoolEntryRequest { - transaction_id: 0.into(), - include_orphan_pool: true, - filter_transaction_pool: false, - }) + .get_mempool_entry_call( + None, + GetMempoolEntryRequest { + transaction_id: 0.into(), + include_orphan_pool: true, + filter_transaction_pool: false, + }, + ) .await; // Test Get Mempool Entry: // TODO: Fix by adding actual mempool entries this can get because otherwise it errors out @@ -236,10 +253,10 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response = rpc_client - .get_mempool_entries_call(GetMempoolEntriesRequest { - include_orphan_pool: true, - filter_transaction_pool: false, - }) + .get_mempool_entries_call( + None, + GetMempoolEntriesRequest { include_orphan_pool: true, filter_transaction_pool: false }, + ) .await .unwrap(); assert!(response.mempool_entries.is_empty()); @@ -249,7 +266,7 @@ async fn sanity_test() { SpectredPayloadOps::GetConnectedPeerInfo => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_connected_peer_info_call(GetConnectedPeerInfoRequest {}).await.unwrap(); + let response = rpc_client.get_connected_peer_info_call(None, GetConnectedPeerInfoRequest {}).await.unwrap(); assert!(response.peer_info.is_empty()); }) } @@ -258,12 +275,12 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let peer_address = ContextualNetAddress::from_str("1.2.3.4").unwrap(); - let _ = rpc_client.add_peer_call(AddPeerRequest { peer_address, is_permanent: true }).await.unwrap(); + let _ = rpc_client.add_peer_call(None, AddPeerRequest { peer_address, is_permanent: true }).await.unwrap(); // Add peer only adds the IP to a connection request. It will only be added to known_addresses if it // actually can be connected to. So in this test we can't expect it to be added unless we set up an // actual peer. - let response = rpc_client.get_peer_addresses_call(GetPeerAddressesRequest {}).await.unwrap(); + let response = rpc_client.get_peer_addresses_call(None, GetPeerAddressesRequest {}).await.unwrap(); assert!(response.known_addresses.is_empty()); }) } @@ -274,14 +291,14 @@ async fn sanity_test() { let peer_address = ContextualNetAddress::from_str("5.6.7.8").unwrap(); let ip = peer_address.normalize(1).ip; - let _ = rpc_client.add_peer_call(AddPeerRequest { peer_address, is_permanent: false }).await.unwrap(); - let _ = rpc_client.ban_call(BanRequest { ip }).await.unwrap(); + let _ = rpc_client.add_peer_call(None, AddPeerRequest { peer_address, is_permanent: false }).await.unwrap(); + let _ = rpc_client.ban_call(None, BanRequest { ip }).await.unwrap(); - let response = rpc_client.get_peer_addresses_call(GetPeerAddressesRequest {}).await.unwrap(); + let response = rpc_client.get_peer_addresses_call(None, GetPeerAddressesRequest {}).await.unwrap(); assert!(response.banned_addresses.contains(&ip)); - let _ = rpc_client.unban_call(UnbanRequest { ip }).await.unwrap(); - let response = rpc_client.get_peer_addresses_call(GetPeerAddressesRequest {}).await.unwrap(); + let _ = rpc_client.unban_call(None, UnbanRequest { ip }).await.unwrap(); + let response = rpc_client.get_peer_addresses_call(None, GetPeerAddressesRequest {}).await.unwrap(); assert!(!response.banned_addresses.contains(&ip)); }) } @@ -316,7 +333,7 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let result = - rpc_client.get_subnetwork_call(GetSubnetworkRequest { subnetwork_id: SubnetworkId::from_byte(0) }).await; + rpc_client.get_subnetwork_call(None, GetSubnetworkRequest { subnetwork_id: SubnetworkId::from_byte(0) }).await; // Err because it's currently unimplemented assert!(result.is_err()); @@ -334,7 +351,7 @@ async fn sanity_test() { SpectredPayloadOps::GetBlockDagInfo => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_block_dag_info_call(GetBlockDagInfoRequest {}).await.unwrap(); + let response = rpc_client.get_block_dag_info_call(None, GetBlockDagInfoRequest {}).await.unwrap(); assert_eq!(response.network, network_id); }) } @@ -343,9 +360,10 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response_result = rpc_client - .resolve_finality_conflict_call(ResolveFinalityConflictRequest { - finality_block_hash: Hash::from_bytes([0; 32]), - }) + .resolve_finality_conflict_call( + None, + ResolveFinalityConflictRequest { finality_block_hash: Hash::from_bytes([0; 32]) }, + ) .await; // Err because it's currently unimplemented @@ -357,7 +375,7 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response_result = rpc_client - .get_headers_call(GetHeadersRequest { start_hash: SIMNET_GENESIS.hash, limit: 1, is_ascending: true }) + .get_headers_call(None, GetHeadersRequest { start_hash: SIMNET_GENESIS.hash, limit: 1, is_ascending: true }) .await; // Err because it's currently unimplemented @@ -369,7 +387,8 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let addresses = vec![Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32])]; - let response = rpc_client.get_utxos_by_addresses_call(GetUtxosByAddressesRequest { addresses }).await.unwrap(); + let response = + rpc_client.get_utxos_by_addresses_call(None, GetUtxosByAddressesRequest { addresses }).await.unwrap(); assert!(response.entries.is_empty()); }) } @@ -378,9 +397,10 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response = rpc_client - .get_balance_by_address_call(GetBalanceByAddressRequest { - address: Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32]), - }) + .get_balance_by_address_call( + None, + GetBalanceByAddressRequest { address: Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32]) }, + ) .await .unwrap(); assert_eq!(response.balance, 0); @@ -392,7 +412,7 @@ async fn sanity_test() { tst!(op, { let addresses = vec![Address::new(Prefix::Simnet, Version::PubKey, &[1u8; 32])]; let response = rpc_client - .get_balances_by_addresses_call(GetBalancesByAddressesRequest::new(addresses.clone())) + .get_balances_by_addresses_call(None, GetBalancesByAddressesRequest::new(addresses.clone())) .await .unwrap(); assert_eq!(response.entries.len(), 1); @@ -400,7 +420,7 @@ async fn sanity_test() { assert_eq!(response.entries[0].balance, Some(0)); let response = - rpc_client.get_balances_by_addresses_call(GetBalancesByAddressesRequest::new(vec![])).await.unwrap(); + rpc_client.get_balances_by_addresses_call(None, GetBalancesByAddressesRequest::new(vec![])).await.unwrap(); assert!(response.entries.is_empty()); }) } @@ -408,7 +428,7 @@ async fn sanity_test() { SpectredPayloadOps::GetSinkBlueScore => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_sink_blue_score_call(GetSinkBlueScoreRequest {}).await.unwrap(); + let response = rpc_client.get_sink_blue_score_call(None, GetSinkBlueScoreRequest {}).await.unwrap(); // A concurrent test may have added a single block so the blue score can be either 0 or 1 assert!(response.blue_score < 2); }) @@ -418,10 +438,10 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let response_result = rpc_client - .estimate_network_hashes_per_second_call(EstimateNetworkHashesPerSecondRequest { - window_size: 1000, - start_hash: None, - }) + .estimate_network_hashes_per_second_call( + None, + EstimateNetworkHashesPerSecondRequest { window_size: 1000, start_hash: None }, + ) .await; // The current DAA window is almost empty so an error is expected assert!(response_result.is_err()); @@ -433,11 +453,10 @@ async fn sanity_test() { tst!(op, { let addresses = vec![Address::new(Prefix::Simnet, Version::PubKey, &[0u8; 32])]; let response = rpc_client - .get_mempool_entries_by_addresses_call(GetMempoolEntriesByAddressesRequest::new( - addresses.clone(), - true, - false, - )) + .get_mempool_entries_by_addresses_call( + None, + GetMempoolEntriesByAddressesRequest::new(addresses.clone(), true, false), + ) .await .unwrap(); assert_eq!(response.entries.len(), 1); @@ -450,7 +469,7 @@ async fn sanity_test() { SpectredPayloadOps::GetCoinSupply => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_coin_supply_call(GetCoinSupplyRequest {}).await.unwrap(); + let response = rpc_client.get_coin_supply_call(None, GetCoinSupplyRequest {}).await.unwrap(); assert_eq!(response.circulating_sompi, 0); assert_eq!(response.max_sompi, MAX_SOMPI); }) @@ -459,7 +478,14 @@ async fn sanity_test() { SpectredPayloadOps::Ping => { let rpc_client = client.clone(); tst!(op, { - let _ = rpc_client.ping_call(PingRequest {}).await.unwrap(); + let _ = rpc_client.ping_call(None, PingRequest {}).await.unwrap(); + }) + } + + SpectredPayloadOps::GetConnections => { + let rpc_client = client.clone(); + tst!(op, { + let _ = rpc_client.get_connections_call(None, GetConnectionsRequest { include_profile_data: true }).await.unwrap(); }) } @@ -467,48 +493,68 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let get_metrics_call_response = rpc_client - .get_metrics_call(GetMetricsRequest { - consensus_metrics: true, - connection_metrics: true, - bandwidth_metrics: true, - process_metrics: true, - }) + .get_metrics_call( + None, + GetMetricsRequest { + consensus_metrics: true, + connection_metrics: true, + bandwidth_metrics: true, + process_metrics: true, + storage_metrics: true, + custom_metrics: true, + }, + ) .await .unwrap(); assert!(get_metrics_call_response.process_metrics.is_some()); assert!(get_metrics_call_response.consensus_metrics.is_some()); let get_metrics_call_response = rpc_client - .get_metrics_call(GetMetricsRequest { - consensus_metrics: false, - connection_metrics: true, - bandwidth_metrics: true, - process_metrics: true, - }) + .get_metrics_call( + None, + GetMetricsRequest { + consensus_metrics: false, + connection_metrics: true, + bandwidth_metrics: true, + process_metrics: true, + storage_metrics: true, + custom_metrics: true, + }, + ) .await .unwrap(); assert!(get_metrics_call_response.process_metrics.is_some()); assert!(get_metrics_call_response.consensus_metrics.is_none()); let get_metrics_call_response = rpc_client - .get_metrics_call(GetMetricsRequest { - consensus_metrics: true, - connection_metrics: true, - bandwidth_metrics: false, - process_metrics: false, - }) + .get_metrics_call( + None, + GetMetricsRequest { + consensus_metrics: true, + connection_metrics: true, + bandwidth_metrics: false, + process_metrics: false, + storage_metrics: false, + custom_metrics: true, + }, + ) .await .unwrap(); assert!(get_metrics_call_response.process_metrics.is_none()); assert!(get_metrics_call_response.consensus_metrics.is_some()); let get_metrics_call_response = rpc_client - .get_metrics_call(GetMetricsRequest { - consensus_metrics: false, - connection_metrics: true, - bandwidth_metrics: false, - process_metrics: false, - }) + .get_metrics_call( + None, + GetMetricsRequest { + consensus_metrics: false, + connection_metrics: true, + bandwidth_metrics: false, + process_metrics: false, + storage_metrics: false, + custom_metrics: true, + }, + ) .await .unwrap(); assert!(get_metrics_call_response.process_metrics.is_none()); @@ -516,10 +562,17 @@ async fn sanity_test() { }) } + SpectredPayloadOps::GetSystemInfo => { + let rpc_client = client.clone(); + tst!(op, { + let _response = rpc_client.get_system_info_call(None, GetSystemInfoRequest {}).await.unwrap(); + }) + } + SpectredPayloadOps::GetServerInfo => { let rpc_client = client.clone(); tst!(op, { - let response = rpc_client.get_server_info_call(GetServerInfoRequest {}).await.unwrap(); + let response = rpc_client.get_server_info_call(None, GetServerInfoRequest {}).await.unwrap(); assert!(response.has_utxo_index); // we set utxoindex above assert_eq!(response.network_id, network_id); }) @@ -528,7 +581,7 @@ async fn sanity_test() { SpectredPayloadOps::GetSyncStatus => { let rpc_client = client.clone(); tst!(op, { - let _ = rpc_client.get_sync_status_call(GetSyncStatusRequest {}).await.unwrap(); + let _ = rpc_client.get_sync_status_call(None, GetSyncStatusRequest {}).await.unwrap(); }) } @@ -536,9 +589,10 @@ async fn sanity_test() { let rpc_client = client.clone(); tst!(op, { let results = rpc_client - .get_daa_score_timestamp_estimate_call(GetDaaScoreTimestampEstimateRequest { - daa_scores: vec![0, 500, 2000, u64::MAX], - }) + .get_daa_score_timestamp_estimate_call( + None, + GetDaaScoreTimestampEstimateRequest { daa_scores: vec![0, 500, 2000, u64::MAX] }, + ) .await .unwrap(); @@ -547,7 +601,7 @@ async fn sanity_test() { } let results = rpc_client - .get_daa_score_timestamp_estimate_call(GetDaaScoreTimestampEstimateRequest { daa_scores: vec![] }) + .get_daa_score_timestamp_estimate_call(None, GetDaaScoreTimestampEstimateRequest { daa_scores: vec![] }) .await .unwrap(); @@ -670,7 +724,7 @@ async fn sanity_test() { // Shutdown should only be tested after everything let rpc_client = client.clone(); - let _ = rpc_client.shutdown_call(ShutdownRequest {}).await.unwrap(); + let _ = rpc_client.shutdown_call(None, ShutdownRequest {}).await.unwrap(); // // Fold-up diff --git a/testing/integration/src/tasks/block/miner.rs b/testing/integration/src/tasks/block/miner.rs index 74e3021..467db75 100644 --- a/testing/integration/src/tasks/block/miner.rs +++ b/testing/integration/src/tasks/block/miner.rs @@ -7,7 +7,7 @@ use rand_distr::{Distribution, Exp}; use spectre_addresses::Address; use spectre_core::warn; use spectre_grpc_client::GrpcClient; -use spectre_rpc_core::{api::rpc::RpcApi, GetBlockTemplateResponse, RpcBlock}; +use spectre_rpc_core::{api::rpc::RpcApi, GetBlockTemplateResponse, RpcRawBlock}; use spectre_utils::triggers::SingleTrigger; use std::{ cmp::max, @@ -25,7 +25,7 @@ pub struct BlockMinerTask { client: Arc, bps: u64, block_count: usize, - sender: Sender, + sender: Sender, template: Arc>, pay_address: Address, tx_counter: Arc, @@ -38,7 +38,7 @@ impl BlockMinerTask { client: Arc, bps: u64, block_count: usize, - sender: Sender, + sender: Sender, template: Arc>, pay_address: Address, stopper: Stopper, @@ -60,7 +60,7 @@ impl BlockMinerTask { client: Arc, bps: u64, block_count: usize, - sender: Sender, + sender: Sender, template: Arc>, pay_address: Address, stopper: Stopper, @@ -68,7 +68,7 @@ impl BlockMinerTask { Arc::new(Self::new(client, bps, block_count, sender, template, pay_address, stopper)) } - pub fn sender(&self) -> Sender { + pub fn sender(&self) -> Sender { self.sender.clone() } diff --git a/testing/integration/src/tasks/block/submitter.rs b/testing/integration/src/tasks/block/submitter.rs index 6207181..68e3ded 100644 --- a/testing/integration/src/tasks/block/submitter.rs +++ b/testing/integration/src/tasks/block/submitter.rs @@ -6,18 +6,18 @@ use async_channel::Sender; use async_trait::async_trait; use spectre_core::warn; use spectre_grpc_client::ClientPool; -use spectre_rpc_core::{api::rpc::RpcApi, RpcBlock}; +use spectre_rpc_core::{api::rpc::RpcApi, RpcRawBlock}; use spectre_utils::triggers::SingleTrigger; use std::{sync::Arc, time::Duration}; use tokio::{task::JoinHandle, time::sleep}; pub struct BlockSubmitterTask { - pool: ClientPool, + pool: ClientPool, stopper: Stopper, } impl BlockSubmitterTask { - pub fn new(pool: ClientPool, stopper: Stopper) -> Self { + pub fn new(pool: ClientPool, stopper: Stopper) -> Self { Self { pool, stopper } } @@ -26,7 +26,7 @@ impl BlockSubmitterTask { Arc::new(Self::new(pool, stopper)) } - pub fn sender(&self) -> Sender { + pub fn sender(&self) -> Sender { self.pool.sender() } } @@ -35,7 +35,7 @@ impl BlockSubmitterTask { impl Task for BlockSubmitterTask { fn start(&self, stop_signal: SingleTrigger) -> Vec> { warn!("Block submitter task starting..."); - let mut tasks = self.pool.start(|c, block: RpcBlock| async move { + let mut tasks = self.pool.start(|c, block: RpcRawBlock| async move { loop { match c.submit_block(block.clone(), false).await { Ok(response) => { diff --git a/utils/Cargo.toml b/utils/Cargo.toml index 2c49879..758dc9e 100644 --- a/utils/Cargo.toml +++ b/utils/Cargo.toml @@ -9,6 +9,9 @@ include.workspace = true license.workspace = true repository.workspace = true +[build-dependencies] +duct = "0.13.7" + [dependencies] arc-swap.workspace = true async-channel.workspace = true @@ -19,16 +22,20 @@ faster-hex.workspace = true ipnet.workspace = true itertools.workspace = true log.workspace = true +num_cpus.workspace = true once_cell.workspace = true parking_lot.workspace = true serde.workspace = true +sha2.workspace = true smallvec.workspace = true +sysinfo.workspace = true thiserror.workspace = true triggered.workspace = true uuid.workspace = true wasm-bindgen.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dependencies] +mac_address.workspace = true rlimit.workspace = true [dev-dependencies] diff --git a/utils/build.rs b/utils/build.rs new file mode 100644 index 0000000..505fa03 --- /dev/null +++ b/utils/build.rs @@ -0,0 +1,82 @@ +use duct::cmd; +use std::env; +use std::path::*; + +struct GitHead { + head_path: String, + head_ref_path: String, + full_hash: String, + short_hash: String, +} + +fn main() { + let success = if env::var("RUSTY_SPECTRE_NO_COMMIT_HASH").is_err() { + if let Some(GitHead { head_path, head_ref_path, full_hash, short_hash }) = try_git_head() { + println!("cargo::rerun-if-changed={head_path}"); + println!("cargo::rerun-if-changed={head_ref_path}"); + println!("cargo:rustc-env=RUSTY_SPECTRE_GIT_FULL_COMMIT_HASH={full_hash}"); + println!("cargo:rustc-env=RUSTY_SPECTRE_GIT_SHORT_COMMIT_HASH={short_hash}"); + true + } else { + false + } + } else { + false + }; + + if !success { + println!("cargo:rustc-env=RUSTY_SPECTRE_GIT_FULL_COMMIT_HASH="); + println!("cargo:rustc-env=RUSTY_SPECTRE_GIT_SHORT_COMMIT_HASH="); + } +} + +fn try_git_head() -> Option { + let cargo_manifest_dir = PathBuf::from(env::var("CARGO_MANIFEST_DIR").unwrap()); + let path = cargo_manifest_dir.as_path().parent()?; + + let full_hash = cmd!("git", "rev-parse", "HEAD").dir(path).read().ok().map(|full_hash| full_hash.trim().to_string()); + + let short_hash = cmd!("git", "rev-parse", "--short", "HEAD").dir(path).read().ok().map(|short_hash| short_hash.trim().to_string()); + + let git_folder = path.join(".git"); + if git_folder.is_dir() { + let head_path = git_folder.join("HEAD"); + if head_path.is_file() { + let head = std::fs::read_to_string(&head_path).ok()?; + if head.starts_with("ref: ") { + let head_ref_path = head.trim_start_matches("ref: "); + let head_ref_path = git_folder.join(head_ref_path.trim()); + if head_ref_path.is_file() { + if let (Some(full_hash), Some(short_hash)) = (full_hash, short_hash) { + return Some(GitHead { + head_path: head_path.to_str().unwrap().to_string(), + head_ref_path: head_ref_path.to_str().unwrap().to_string(), + full_hash, + short_hash, + }); + } else if let Ok(full_hash) = std::fs::read_to_string(&head_ref_path) { + let full_hash = full_hash.trim().to_string(); + let short_hash = if full_hash.len() >= 7 { + // this is not actually correct as short hash has a variable + // length based on commit short hash collisions (which is) + // why we attempt to use `git rev-parse` above. But since this + // is for reference purposes only, we can live with it. + full_hash[0..7].to_string() + } else { + full_hash.to_string() + }; + + return Some(GitHead { + head_path: head_path.to_str().unwrap().to_string(), + head_ref_path: head_ref_path.to_str().unwrap().to_string(), + full_hash, + short_hash, + }); + } + } + } + } + } + + None +} diff --git a/utils/src/git.rs b/utils/src/git.rs new file mode 100644 index 0000000..c80cd48 --- /dev/null +++ b/utils/src/git.rs @@ -0,0 +1,53 @@ +use crate::hex::FromHex; +use std::fmt::Display; + +const VERSION: &str = env!("CARGO_PKG_VERSION"); + +// generated by `build.rs` +const FULL_HASH: &str = env!("RUSTY_SPECTRE_GIT_FULL_COMMIT_HASH"); +const SHORT_HASH: &str = env!("RUSTY_SPECTRE_GIT_SHORT_COMMIT_HASH"); + +/// Check if the codebase is built under a Git repository +/// and return the hash of the current commit as `Vec`. +pub fn hash() -> Option> { + FromHex::from_hex(FULL_HASH).ok() +} + +pub fn short_hash() -> Option> { + FromHex::from_hex(SHORT_HASH).ok() +} + +pub fn hash_str() -> Option<&'static str> { + #[allow(clippy::const_is_empty)] + (!FULL_HASH.is_empty()).then_some(FULL_HASH) +} + +pub fn short_hash_str() -> Option<&'static str> { + #[allow(clippy::const_is_empty)] + (!SHORT_HASH.is_empty()).then_some(SHORT_HASH) +} + +pub fn version() -> String { + if let Some(short_hash) = short_hash_str() { + format!("v{VERSION}-{short_hash}") + } else { + format!("v{VERSION}") + } +} + +pub fn with_short_hash(version: V) -> impl Display +where + V: Display, +{ + if let Some(short_hash) = short_hash_str() { + format!("{version}-{short_hash}") + } else { + version.to_string() + } +} + +#[test] +fn test_git_hash() { + println!("FULL_HASH: {:?}", hash_str()); + println!("SHORT_HASH: {:?}", short_hash_str()); +} diff --git a/utils/src/lib.rs b/utils/src/lib.rs index 13aeba3..31cef4d 100644 --- a/utils/src/lib.rs +++ b/utils/src/lib.rs @@ -192,4 +192,9 @@ pub mod sync; pub mod triggers; pub mod vec; +pub mod git; + +#[cfg(not(target_arch = "wasm32"))] pub mod fd_budget; +#[cfg(not(target_arch = "wasm32"))] +pub mod sysinfo; diff --git a/utils/src/mem_size.rs b/utils/src/mem_size.rs index c7963a4..449f649 100644 --- a/utils/src/mem_size.rs +++ b/utils/src/mem_size.rs @@ -2,7 +2,7 @@ //! estimate sizes of run-time objects in memory, including deep heap allocations. See //! struct-level docs for more details. -use std::{collections::HashSet, mem::size_of, sync::Arc}; +use std::{collections::HashSet, sync::Arc}; use parking_lot::RwLock; diff --git a/utils/src/networking.rs b/utils/src/networking.rs index ebd72b2..b7a3397 100644 --- a/utils/src/networking.rs +++ b/utils/src/networking.rs @@ -179,7 +179,7 @@ impl Deref for IpAddress { // impl BorshSerialize for IpAddress { - fn serialize(&self, writer: &mut W) -> ::core::result::Result<(), borsh::maybestd::io::Error> { + fn serialize(&self, writer: &mut W) -> ::core::result::Result<(), std::io::Error> { let variant_idx: u8 = match self.0 { IpAddr::V4(..) => 0u8, IpAddr::V6(..) => 1u8, @@ -198,20 +198,20 @@ impl BorshSerialize for IpAddress { } impl BorshDeserialize for IpAddress { - fn deserialize(buf: &mut &[u8]) -> ::core::result::Result { - let variant_idx: u8 = BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> ::core::result::Result { + let variant_idx: u8 = BorshDeserialize::deserialize_reader(reader)?; let ip = match variant_idx { 0u8 => { - let octets: [u8; 4] = BorshDeserialize::deserialize(buf)?; + let octets: [u8; 4] = BorshDeserialize::deserialize_reader(reader)?; IpAddr::V4(Ipv4Addr::from(octets)) } 1u8 => { - let octets: [u8; 16] = BorshDeserialize::deserialize(buf)?; + let octets: [u8; 16] = BorshDeserialize::deserialize_reader(reader)?; IpAddr::V6(Ipv6Addr::from(octets)) } _ => { - let msg = borsh::maybestd::format!("Unexpected variant index: {:?}", variant_idx); - return Err(borsh::maybestd::io::Error::new(borsh::maybestd::io::ErrorKind::InvalidInput, msg)); + let msg = format!("Unexpected variant index: {:?}", variant_idx); + return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, msg)); } }; Ok(Self(ip)) @@ -275,6 +275,10 @@ impl ContextualNetAddress { Self { ip, port } } + pub fn has_port(&self) -> bool { + self.port.is_some() + } + pub fn normalize(&self, default_port: u16) -> NetAddress { NetAddress::new(self.ip, self.port.unwrap_or(default_port)) } @@ -389,15 +393,15 @@ impl Deref for PeerId { // impl BorshSerialize for PeerId { - fn serialize(&self, writer: &mut W) -> ::core::result::Result<(), borsh::maybestd::io::Error> { + fn serialize(&self, writer: &mut W) -> ::core::result::Result<(), std::io::Error> { borsh::BorshSerialize::serialize(&self.0.as_bytes(), writer)?; Ok(()) } } impl BorshDeserialize for PeerId { - fn deserialize(buf: &mut &[u8]) -> ::core::result::Result { - let bytes: uuid::Bytes = BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> ::core::result::Result { + let bytes: uuid::Bytes = BorshDeserialize::deserialize_reader(reader)?; Ok(Self::new(Uuid::from_bytes(bytes))) } } @@ -411,12 +415,12 @@ mod tests { fn test_ip_address_borsh() { // Tests for IpAddress Borsh ser/deser since we manually implemented them let ip: IpAddress = Ipv4Addr::from([44u8; 4]).into(); - let bin = ip.try_to_vec().unwrap(); + let bin = borsh::to_vec(&ip).unwrap(); let ip2: IpAddress = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(ip, ip2); let ip: IpAddress = Ipv6Addr::from([66u8; 16]).into(); - let bin = ip.try_to_vec().unwrap(); + let bin = borsh::to_vec(&ip).unwrap(); let ip2: IpAddress = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(ip, ip2); } @@ -425,12 +429,12 @@ mod tests { fn test_peer_id_borsh() { // Tests for PeerId Borsh ser/deser since we manually implemented them let id: PeerId = Uuid::new_v4().into(); - let bin = id.try_to_vec().unwrap(); + let bin = borsh::to_vec(&id).unwrap(); let id2: PeerId = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(id, id2); let id: PeerId = Uuid::from_bytes([123u8; 16]).into(); - let bin = id.try_to_vec().unwrap(); + let bin = borsh::to_vec(&id).unwrap(); let id2: PeerId = BorshDeserialize::try_from_slice(&bin).unwrap(); assert_eq!(id, id2); } diff --git a/utils/src/option.rs b/utils/src/option.rs index 9ccf96c..ff4779d 100644 --- a/utils/src/option.rs +++ b/utils/src/option.rs @@ -1,9 +1,10 @@ pub trait OptionExtensions { - fn is_none_or(&self, f: impl FnOnce(&T) -> bool) -> bool; + /// Substitute for unstable [Option::is_non_or] + fn is_none_or_ex(&self, f: impl FnOnce(&T) -> bool) -> bool; } impl OptionExtensions for Option { - fn is_none_or(&self, f: impl FnOnce(&T) -> bool) -> bool { + fn is_none_or_ex(&self, f: impl FnOnce(&T) -> bool) -> bool { match self { Some(v) => f(v), None => true, diff --git a/utils/src/sysinfo.rs b/utils/src/sysinfo.rs new file mode 100644 index 0000000..4e009d4 --- /dev/null +++ b/utils/src/sysinfo.rs @@ -0,0 +1,81 @@ +use crate::fd_budget; +use crate::git; +use crate::hex::ToHex; +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::io::Read; +use std::sync::OnceLock; + +static SYSTEM_INFO: OnceLock = OnceLock::new(); + +#[derive(Clone)] +pub struct SystemInfo { + pub system_id: Option>, + pub git_hash: Option>, + pub git_short_hash: Option>, + pub version: String, + pub cpu_physical_cores: u16, + pub total_memory: u64, + pub fd_limit: u32, +} + +// provide hex encoding for system_id, git_hash, and git_short_hash +impl std::fmt::Debug for SystemInfo { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SystemInfo") + .field("system_id", &self.system_id.as_ref().map(|id| id.to_hex())) + .field("git_hash", &self.git_hash.as_ref().map(|hash| hash.to_hex())) + .field("git_short_hash", &self.git_short_hash.as_ref().map(|hash| hash.to_hex())) + .field("version", &self.version) + .field("cpu_physical_cores", &self.cpu_physical_cores) + .field("total_memory", &self.total_memory) + .field("fd_limit", &self.fd_limit) + .finish() + } +} + +impl Default for SystemInfo { + fn default() -> Self { + let system_info = SYSTEM_INFO.get_or_init(|| { + let mut system = sysinfo::System::new(); + system.refresh_memory(); + let cpu_physical_cores = num_cpus::get() as u16; + let total_memory = system.total_memory(); + let fd_limit = fd_budget::limit() as u32; + let system_id = Self::try_system_id(); + let git_hash = git::hash(); + let git_short_hash = git::short_hash(); + let version = git::version(); + + SystemInfo { system_id, git_hash, git_short_hash, version, cpu_physical_cores, total_memory, fd_limit } + }); + (*system_info).clone() + } +} + +impl SystemInfo { + /// Obtain a unique system (machine) identifier. + fn try_system_id() -> Option> { + let some_id = if let Ok(mut file) = File::open("/etc/machine-id") { + // fetch the system id from /etc/machine-id + let mut machine_id = String::new(); + file.read_to_string(&mut machine_id).ok(); + machine_id.trim().to_string() + } else if let Ok(Some(mac)) = mac_address::get_mac_address() { + // fallback on the mac address + mac.to_string().trim().to_string() + } else { + // 🤷 + return None; + }; + let mut sha256 = Sha256::default(); + sha256.update(some_id.as_bytes()); + Some(sha256.finalize().to_vec()) + } +} + +impl AsRef for SystemInfo { + fn as_ref(&self) -> &SystemInfo { + self + } +} diff --git a/wallet/bip32/src/mnemonic/phrase.rs b/wallet/bip32/src/mnemonic/phrase.rs index f366baa..35515d7 100644 --- a/wallet/bip32/src/mnemonic/phrase.rs +++ b/wallet/bip32/src/mnemonic/phrase.rs @@ -23,7 +23,7 @@ pub type Entropy32 = [u8; KEY_SIZE]; pub type Entropy16 = [u8; 16]; /// Word count for a BIP39 mnemonic phrase. Identifies mnemonic as 12 or 24 word variants. -#[derive(Default, Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[derive(Default, PartialEq, Eq, Clone, Copy, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "kebab-case")] pub enum WordCount { #[default] @@ -85,8 +85,8 @@ impl Mnemonic { } #[wasm_bindgen(js_name = random)] - pub fn create_random_js(word_count: JsValue) -> Result { - let word_count = word_count.as_f64().unwrap_or(24.0) as usize; + pub fn create_random_js(word_count: Option) -> Result { + let word_count = word_count.unwrap_or(24) as usize; Mnemonic::random(word_count.try_into()?, Default::default()) } diff --git a/wallet/bip32/src/xpublic_key.rs b/wallet/bip32/src/xpublic_key.rs index a52d8f1..ac4eb72 100644 --- a/wallet/bip32/src/xpublic_key.rs +++ b/wallet/bip32/src/xpublic_key.rs @@ -174,8 +174,8 @@ impl BorshDeserialize for ExtendedPublicKey where K: PublicKey, { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { - let Header { version, magic } = Header::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let Header { version, magic } = Header::deserialize_reader(reader)?; if magic != Self::STORAGE_MAGIC { return Err(std::io::Error::new(std::io::ErrorKind::Other, "Invalid extended public key magic value")); } @@ -183,13 +183,11 @@ where return Err(std::io::Error::new(std::io::ErrorKind::Other, "Invalid extended public key version")); } - let public_key_bytes: [u8; KEY_SIZE + 1] = buf[..KEY_SIZE + 1] - .try_into() - .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Invalid extended public key"))?; + let mut public_key_bytes: [u8; KEY_SIZE + 1] = [0; KEY_SIZE + 1]; + reader.read_exact(&mut public_key_bytes)?; let public_key = K::from_bytes(public_key_bytes) .map_err(|_| std::io::Error::new(std::io::ErrorKind::Other, "Invalid extended public key"))?; - *buf = &buf[KEY_SIZE + 1..]; - let attrs = ExtendedKeyAttrs::deserialize(buf)?; + let attrs = ExtendedKeyAttrs::deserialize_reader(reader)?; Ok(Self { public_key, attrs }) } } diff --git a/wallet/core/Cargo.toml b/wallet/core/Cargo.toml index fabb6ec..a98845b 100644 --- a/wallet/core/Cargo.toml +++ b/wallet/core/Cargo.toml @@ -71,6 +71,7 @@ spectre-txscript.workspace = true spectre-utils.workspace = true spectre-wallet-keys.workspace = true spectre-wallet-macros.workspace = true +spectre-wallet-psst.workspace = true spectre-wasm-core.workspace = true spectre-wrpc-client.workspace = true spectre-wrpc-wasm.workspace = true @@ -125,5 +126,5 @@ serde_repr.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/wallet/core/src/account/descriptor.rs b/wallet/core/src/account/descriptor.rs index a25b6b2..6a811cf 100644 --- a/wallet/core/src/account/descriptor.rs +++ b/wallet/core/src/account/descriptor.rs @@ -17,6 +17,7 @@ pub struct AccountDescriptor { pub kind: AccountKind, pub account_id: AccountId, pub account_name: Option, + pub balance: Option, pub prv_key_data_ids: AssocPrvKeyDataIds, pub receive_address: Option
, pub change_address: Option
, @@ -29,11 +30,21 @@ impl AccountDescriptor { kind: AccountKind, account_id: AccountId, account_name: Option, + balance: Option, prv_key_data_ids: AssocPrvKeyDataIds, receive_address: Option
, change_address: Option
, ) -> Self { - Self { kind, account_id, account_name, prv_key_data_ids, receive_address, change_address, properties: BTreeMap::default() } + Self { + kind, + account_id, + account_name, + balance, + prv_key_data_ids, + receive_address, + change_address, + properties: BTreeMap::default(), + } } pub fn with_property(mut self, property: AccountDescriptorProperty, value: AccountDescriptorValue) -> Self { @@ -111,11 +122,12 @@ impl std::fmt::Display for AccountDescriptorValue { AccountDescriptorValue::Bool(value) => write!(f, "{}", value), AccountDescriptorValue::AddressDerivationMeta(value) => write!(f, "{}", value), AccountDescriptorValue::XPubKeys(value) => { - let mut s = String::new(); + let mut s = vec![]; for xpub in value.iter() { - s.push_str(&format!("{}\n", xpub)); + //s.push(xpub.to_string(None)); + s.push(format!("{}", xpub)); } - write!(f, "{}", s) + write!(f, "{}", s.join("\n")) } AccountDescriptorValue::Json(value) => write!(f, "{}", value), } @@ -225,6 +237,7 @@ declare! { receiveAddress? : Address, changeAddress? : Address, prvKeyDataIds : HexString[], + // balance? : Balance, [key: string]: any } "#, diff --git a/wallet/core/src/account/kind.rs b/wallet/core/src/account/kind.rs index 21330d0..20e863d 100644 --- a/wallet/core/src/account/kind.rs +++ b/wallet/core/src/account/kind.rs @@ -96,22 +96,17 @@ impl BorshSerialize for AccountKind { } impl BorshDeserialize for AccountKind { - fn deserialize(buf: &mut &[u8]) -> IoResult { - if buf.is_empty() { - Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid AccountKind length")) - } else { - let len = buf[0]; - if buf.len() < (len as usize + 1) { - Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid AccountKind length")) - } else { - let s = str64::make( - std::str::from_utf8(&buf[1..(len as usize + 1)]) - .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid UTF-8 sequence"))?, - ); - *buf = &buf[(len as usize + 1)..]; - Ok(Self(s)) - } - } + fn deserialize_reader(reader: &mut R) -> IoResult { + let len = ::deserialize_reader(reader)? as usize; + let mut buf = [0; 64]; + reader + .read_exact(&mut buf[0..len]) + .map_err(|err| std::io::Error::new(std::io::ErrorKind::InvalidData, format!("Invalid AccountKind length ({err:?})")))?; + let s = str64::make( + std::str::from_utf8(&buf[..len]) + .map_err(|_| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid UTF-8 sequence"))?, + ); + Ok(Self(s)) } } diff --git a/wallet/core/src/account/mod.rs b/wallet/core/src/account/mod.rs index 002f3c7..0c51386 100644 --- a/wallet/core/src/account/mod.rs +++ b/wallet/core/src/account/mod.rs @@ -5,8 +5,15 @@ pub mod descriptor; pub mod kind; +pub mod pssb; pub mod variants; pub use kind::*; +use pssb::{ + bundle_from_psst_generator, bundle_to_finalizer_stream, pssb_signer_for_address, psst_to_pending_transaction, PSSBSigner, + PSSTGenerator, +}; +use spectre_hashes::Hash; +use spectre_wallet_psst::bundle::Bundle; pub use variants::*; use crate::derivation::build_derivate_paths; @@ -356,6 +363,66 @@ pub trait Account: AnySync + Send + Sync + 'static { Ok((generator.summary(), ids)) } + async fn pssb_from_send_generator( + self: Arc, + destination: PaymentDestination, + priority_fee_sompi: Fees, + payload: Option>, + wallet_secret: Secret, + payment_secret: Option, + abortable: &Abortable, + ) -> Result { + let settings = GeneratorSettings::try_new_with_account(self.clone().as_dyn_arc(), destination, priority_fee_sompi, payload)?; + let keydata = self.prv_key_data(wallet_secret).await?; + let signer = Arc::new(PSSBSigner::new(self.clone().as_dyn_arc(), keydata, payment_secret)); + let generator = Generator::try_new(settings, None, Some(abortable))?; + let psst_generator = PSSTGenerator::new(generator, signer, self.wallet().address_prefix()?); + bundle_from_psst_generator(psst_generator).await + } + + async fn pssb_sign( + self: Arc, + bundle: &Bundle, + wallet_secret: Secret, + payment_secret: Option, + sign_for_address: Option<&Address>, + ) -> Result { + let keydata = self.prv_key_data(wallet_secret).await?; + let signer = Arc::new(PSSBSigner::new(self.clone().as_dyn_arc(), keydata.clone(), payment_secret.clone())); + + let network_id = self.wallet().clone().network_id()?; + let derivation = self.as_derivation_capable()?; + + let (derivation_path, _) = + build_derivate_paths(&derivation.account_kind(), derivation.account_index(), derivation.cosigner_index())?; + + let key_fingerprint = keydata.get_xprv(payment_secret.clone().as_ref())?.public_key().fingerprint(); + + match pssb_signer_for_address(bundle, signer, network_id, sign_for_address, derivation_path, key_fingerprint).await { + Ok(signer) => Ok(signer), + Err(e) => Err(Error::from(e.to_string())), + } + } + + async fn pssb_broadcast(self: Arc, bundle: &Bundle) -> Result, Error> { + let mut ids = Vec::new(); + let mut stream = bundle_to_finalizer_stream(bundle); + + while let Some(result) = stream.next().await { + match result { + Ok(psst) => { + let change = self.wallet().account()?.change_address()?; + let transaction = psst_to_pending_transaction(psst, self.wallet().network_id()?, change)?; + ids.push(transaction.try_submit(&self.wallet().rpc_api()).await?); + } + Err(e) => { + eprintln!("Error processing a PSST from bundle: {:?}", e); + } + } + } + Ok(ids) + } + /// Execute a transfer to another wallet account. async fn transfer( self: Arc, @@ -366,13 +433,14 @@ pub trait Account: AnySync + Send + Sync + 'static { payment_secret: Option, abortable: &Abortable, notifier: Option, + guard: &WalletGuard, ) -> Result<(GeneratorSummary, Vec)> { let keydata = self.prv_key_data(wallet_secret).await?; let signer = Arc::new(Signer::new(self.clone().as_dyn_arc(), keydata, payment_secret)); let destination_account = self .wallet() - .get_account_by_id(&destination_account_id) + .get_account_by_id(&destination_account_id, guard) .await? .ok_or_else(|| Error::AccountNotFound(destination_account_id))?; @@ -532,6 +600,7 @@ pub trait DerivationCapableAccount: Account { let settings = GeneratorSettings::try_new_with_iterator( self.wallet().network_id()?, Box::new(utxos.into_iter()), + None, change_address.clone(), 1, 1, @@ -545,7 +614,7 @@ pub trait DerivationCapableAccount: Account { let mut stream = generator.stream(); while let Some(transaction) = stream.try_next().await? { - transaction.try_sign_with_keys(&keys)?; + transaction.try_sign_with_keys(&keys, None)?; let id = transaction.try_submit(&rpc).await?; if let Some(notifier) = notifier { notifier(index, aggregate_utxo_count, balance, Some(id)); diff --git a/wallet/core/src/account/pssb.rs b/wallet/core/src/account/pssb.rs new file mode 100644 index 0000000..d321656 --- /dev/null +++ b/wallet/core/src/account/pssb.rs @@ -0,0 +1,360 @@ +pub use crate::error::Error; +use crate::imports::*; +use crate::tx::PaymentOutputs; +use futures::stream; +use secp256k1::schnorr; +use secp256k1::{Message, PublicKey}; +use spectre_bip32::{DerivationPath, KeyFingerprint, PrivateKey}; +use spectre_consensus_client::UtxoEntry as ClientUTXO; +use spectre_consensus_core::hashing::sighash::{calc_schnorr_signature_hash, SigHashReusedValues}; +use spectre_consensus_core::tx::VerifiableTransaction; +use spectre_consensus_core::tx::{TransactionInput, UtxoEntry}; +use spectre_txscript::extract_script_pub_key_address; +use spectre_txscript::opcodes::codes::OpData65; +use spectre_txscript::script_builder::ScriptBuilder; +use spectre_wallet_core::tx::{Generator, GeneratorSettings, PaymentDestination, PendingTransaction}; +pub use spectre_wallet_psst::bundle::Bundle; +use spectre_wallet_psst::prelude::KeySource; +use spectre_wallet_psst::prelude::{Finalizer, Inner, SignInputOk, Signature, Signer}; +pub use spectre_wallet_psst::psst::{Creator, PSST}; +use std::iter; + +struct PSSBSignerInner { + keydata: PrvKeyData, + account: Arc, + payment_secret: Option, + keys: Mutex>, +} + +pub struct PSSBSigner { + inner: Arc, +} + +impl PSSBSigner { + pub fn new(account: Arc, keydata: PrvKeyData, payment_secret: Option) -> Self { + Self { inner: Arc::new(PSSBSignerInner { keydata, account, payment_secret, keys: Mutex::new(AHashMap::new()) }) } + } + + pub fn ingest(&self, addresses: &[Address]) -> Result<()> { + let mut keys = self.inner.keys.lock()?; + + // Skip addresses that are already present in the key map. + let addresses = addresses.iter().filter(|a| !keys.contains_key(a)).collect::>(); + if !addresses.is_empty() { + let account = self.inner.account.clone().as_derivation_capable().expect("expecting derivation capable account"); + let (receive, change) = account.derivation().addresses_indexes(&addresses)?; + let private_keys = account.create_private_keys(&self.inner.keydata, &self.inner.payment_secret, &receive, &change)?; + for (address, private_key) in private_keys { + keys.insert(address.clone(), private_key.to_bytes()); + } + } + Ok(()) + } + + fn public_key(&self, for_address: &Address) -> Result { + let keys = self.inner.keys.lock()?; + match keys.get(for_address) { + Some(private_key) => { + let kp = secp256k1::Keypair::from_seckey_slice(secp256k1::SECP256K1, private_key)?; + Ok(kp.public_key()) + } + None => Err(Error::from("PSSBSigner address coverage error")), + } + } + + fn sign_schnorr(&self, for_address: &Address, message: Message) -> Result { + let keys = self.inner.keys.lock()?; + match keys.get(for_address) { + Some(private_key) => { + let schnorr_key = secp256k1::Keypair::from_seckey_slice(secp256k1::SECP256K1, private_key)?; + Ok(schnorr_key.sign_schnorr(message)) + } + None => Err(Error::from("PSSBSigner address coverage error")), + } + } +} + +pub struct PSSTGenerator { + generator: Generator, + signer: Arc, + prefix: Prefix, +} + +impl PSSTGenerator { + pub fn new(generator: Generator, signer: Arc, prefix: Prefix) -> Self { + Self { generator, signer, prefix } + } + + pub fn stream(&self) -> impl Stream, Error>> { + PSSTStream::new(self.generator.clone(), self.signer.clone(), self.prefix) + } +} + +struct PSSTStream { + generator_stream: Pin> + Send>>, + signer: Arc, + prefix: Prefix, +} + +impl PSSTStream { + fn new(generator: Generator, signer: Arc, prefix: Prefix) -> Self { + let generator_stream = generator.stream().map_err(Error::from); + Self { generator_stream: Box::pin(generator_stream), signer, prefix } + } +} + +impl Stream for PSSTStream { + type Item = Result, Error>; + + fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let this = self.as_ref(); + + let _prefix = this.prefix; + let _signer = this.signer.clone(); + + match self.get_mut().generator_stream.as_mut().poll_next(cx) { + Poll::Ready(Some(Ok(pending_tx))) => { + let psst = convert_pending_tx_to_psst(pending_tx); + Poll::Ready(Some(psst)) + } + Poll::Ready(Some(Err(e))) => Poll::Ready(Some(Err(e))), + Poll::Ready(None) => Poll::Ready(None), + Poll::Pending => Poll::Pending, + } + } +} + +fn convert_pending_tx_to_psst(pending_tx: PendingTransaction) -> Result, Error> { + let signable_tx = pending_tx.signable_transaction(); + let verifiable_tx = signable_tx.as_verifiable(); + let populated_inputs: Vec<(&TransactionInput, &UtxoEntry)> = verifiable_tx.populated_inputs().collect(); + let psst_inner = Inner::try_from((pending_tx.transaction(), populated_inputs.to_owned()))?; + Ok(PSST::::from(psst_inner)) +} + +pub async fn bundle_from_psst_generator(generator: PSSTGenerator) -> Result { + let mut bundle: Bundle = Bundle::new(); + let mut stream = generator.stream(); + + while let Some(psst_result) = stream.next().await { + match psst_result { + Ok(psst) => bundle.add_psst(psst), + Err(e) => return Err(e), + } + } + + Ok(bundle) +} + +pub async fn pssb_signer_for_address( + bundle: &Bundle, + signer: Arc, + network_id: NetworkId, + sign_for_address: Option<&Address>, + derivation_path: DerivationPath, + key_fingerprint: KeyFingerprint, +) -> Result { + let mut signed_bundle = Bundle::new(); + let mut reused_values = SigHashReusedValues::new(); + + // If set, sign-for address is used for signing. + // Else, all addresses from inputs are. + let addresses: Vec
= match sign_for_address { + Some(signer) => vec![signer.clone()], + None => bundle + .iter() + .flat_map(|inner| { + inner.inputs + .iter() + .filter_map(|input| input.utxo_entry.as_ref()) // Filter out None and get a reference to UtxoEntry if it exists + .filter_map(|utxo_entry| { + extract_script_pub_key_address(&utxo_entry.script_public_key.clone(), network_id.into()).ok() + }) + .collect::>() + }) + .collect(), + }; + + // Prepare the signer. + signer.ingest(addresses.as_ref())?; + + for psst_inner in bundle.iter().cloned() { + let psst: PSST = PSST::from(psst_inner); + + let mut sign = |signer_psst: PSST| { + signer_psst + .pass_signature_sync(|tx, sighash| -> Result, String> { + tx.tx + .inputs + .iter() + .enumerate() + .map(|(idx, _input)| { + let hash = calc_schnorr_signature_hash(&tx.as_verifiable(), idx, sighash[idx], &mut reused_values); + let msg = secp256k1::Message::from_digest_slice(hash.as_bytes().as_slice()).unwrap(); + + // When address represents a locked UTXO, no private key is available. + // Instead, use the account receive address' private key. + let address: &Address = match sign_for_address { + Some(address) => address, + None => addresses.get(idx).expect("Input indexed address"), + }; + + let public_key = signer.public_key(address).expect("Public key for input indexed address"); + + Ok(SignInputOk { + signature: Signature::Schnorr(signer.sign_schnorr(address, msg).unwrap()), + pub_key: public_key, + key_source: Some(KeySource { key_fingerprint, derivation_path: derivation_path.clone() }), + }) + }) + .collect() + }) + .unwrap() + }; + signed_bundle.add_psst(sign(psst.clone())); + } + Ok(signed_bundle) +} + +pub fn finalize_psst_one_or_more_sig_and_redeem_script(psst: PSST) -> Result, Error> { + let result = psst.finalize_sync(|inner: &Inner| -> Result>, String> { + Ok(inner + .inputs + .iter() + .map(|input| -> Vec { + let signatures: Vec<_> = input + .partial_sigs + .clone() + .into_iter() + .flat_map(|(_, signature)| iter::once(OpData65).chain(signature.into_bytes()).chain([input.sighash_type.to_u8()])) + .collect(); + + signatures + .into_iter() + .chain( + input + .redeem_script + .as_ref() + .map(|redeem_script| ScriptBuilder::new().add_data(redeem_script.as_slice()).unwrap().drain().to_vec()) + .unwrap_or_default(), + ) + .collect() + }) + .collect()) + }); + + match result { + Ok(finalized_psst) => Ok(finalized_psst), + Err(e) => Err(Error::from(e.to_string())), + } +} + +pub fn finalize_psst_no_sig_and_redeem_script(psst: PSST) -> Result, Error> { + let result = psst.finalize_sync(|inner: &Inner| -> Result>, String> { + Ok(inner + .inputs + .iter() + .map(|input| -> Vec { + input + .redeem_script + .as_ref() + .map(|redeem_script| ScriptBuilder::new().add_data(redeem_script.as_slice()).unwrap().drain().to_vec()) + .unwrap_or_default() + }) + .collect()) + }); + + match result { + Ok(finalized_psst) => Ok(finalized_psst), + Err(e) => Err(Error::from(e.to_string())), + } +} + +pub fn bundle_to_finalizer_stream(bundle: &Bundle) -> impl Stream, Error>> + Send { + stream::iter(bundle.iter().cloned().collect::>()).map(move |psst_inner| { + let psst: PSST = PSST::from(psst_inner); + let psst_finalizer = psst.constructor().updater().signer().finalizer(); + finalize_psst_one_or_more_sig_and_redeem_script(psst_finalizer) + }) +} + +pub fn psst_to_pending_transaction( + finalized_psst: PSST, + network_id: NetworkId, + change_address: Address, +) -> Result { + let mass = 10; + let (signed_tx, _) = match finalized_psst.clone().extractor() { + Ok(extractor) => match extractor.extract_tx() { + Ok(once_mass) => once_mass(mass), + Err(e) => return Err(Error::PendingTransactionFromPSSTError(e.to_string())), + }, + Err(e) => return Err(Error::PendingTransactionFromPSSTError(e.to_string())), + }; + + let inner_psst = finalized_psst.deref().clone(); + + let utxo_entries_ref: Vec = inner_psst + .inputs + .iter() + .filter_map(|input| { + if let Some(ue) = input.clone().utxo_entry { + return Some(UtxoEntryReference { + utxo: Arc::new(ClientUTXO { + address: Some(extract_script_pub_key_address(&ue.script_public_key, network_id.into()).unwrap()), + amount: ue.amount, + outpoint: input.previous_outpoint.into(), + script_public_key: ue.script_public_key, + block_daa_score: ue.block_daa_score, + is_coinbase: ue.is_coinbase, + }), + }); + } + None + }) + .collect(); + + let output: Vec = signed_tx.outputs.clone(); + let recipient = extract_script_pub_key_address(&output[0].script_public_key, network_id.into())?; + let fee_u: u64 = 0; + + let utxo_iterator: Box + Send + Sync + 'static> = + Box::new(utxo_entries_ref.clone().into_iter()); + + let final_transaction_destination = PaymentDestination::PaymentOutputs(PaymentOutputs::from((recipient.clone(), output[0].value))); + + let settings = GeneratorSettings { + network_id, + multiplexer: None, + sig_op_count: 1, + minimum_signatures: 1, + change_address, + utxo_iterator, + priority_utxo_entries: None, + source_utxo_context: None, + destination_utxo_context: None, + final_transaction_priority_fee: fee_u.into(), + final_transaction_destination, + final_transaction_payload: None, + }; + + // Create the Generator + let generator = Generator::try_new(settings, None, None)?; + + // Create PendingTransaction + let pending_tx = PendingTransaction::try_new( + &generator, + signed_tx.clone(), + utxo_entries_ref.clone(), + vec![], + None, + 0, + 0, + 0, + 0, + 0, + spectre_wallet_core::tx::DataKind::Final, + )?; + + Ok(pending_tx) +} diff --git a/wallet/core/src/account/variants/bip32.rs b/wallet/core/src/account/variants/bip32.rs index cb0c18d..d1e9043 100644 --- a/wallet/core/src/account/variants/bip32.rs +++ b/wallet/core/src/account/variants/bip32.rs @@ -70,13 +70,13 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let xpub_keys = BorshDeserialize::deserialize(buf)?; - let account_index = BorshDeserialize::deserialize(buf)?; - let ecdsa = BorshDeserialize::deserialize(buf)?; + let xpub_keys = BorshDeserialize::deserialize_reader(reader)?; + let account_index = BorshDeserialize::deserialize_reader(reader)?; + let ecdsa = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { xpub_keys, account_index, ecdsa }) } @@ -221,6 +221,7 @@ impl Account for Bip32 { BIP32_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), self.prv_key_data_id.into(), self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/account/variants/bip32watch.rs b/wallet/core/src/account/variants/bip32watch.rs index 178a8dd..1dabb52 100644 --- a/wallet/core/src/account/variants/bip32watch.rs +++ b/wallet/core/src/account/variants/bip32watch.rs @@ -68,12 +68,12 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> std::io::Result { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let xpub_keys = BorshDeserialize::deserialize(buf)?; - let ecdsa = BorshDeserialize::deserialize(buf)?; + let xpub_keys = BorshDeserialize::deserialize_reader(reader)?; + let ecdsa = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { xpub_keys, ecdsa }) } @@ -203,6 +203,7 @@ impl Account for Bip32Watch { BIP32_WATCH_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), AssocPrvKeyDataIds::None, self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/account/variants/keypair.rs b/wallet/core/src/account/variants/keypair.rs index 865af2d..8d53794 100644 --- a/wallet/core/src/account/variants/keypair.rs +++ b/wallet/core/src/account/variants/keypair.rs @@ -69,19 +69,17 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { use secp256k1::constants::PUBLIC_KEY_SIZE; let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let public_key_bytes: [u8; PUBLIC_KEY_SIZE] = buf[..PUBLIC_KEY_SIZE] - .try_into() - .map_err(|_| IoError::new(IoErrorKind::Other, "Unable to deserialize keypair account (public_key buffer try_into)"))?; + let mut public_key_bytes: [u8; PUBLIC_KEY_SIZE] = [0; PUBLIC_KEY_SIZE]; + reader.read_exact(&mut public_key_bytes)?; let public_key = secp256k1::PublicKey::from_slice(&public_key_bytes) .map_err(|_| IoError::new(IoErrorKind::Other, "Unable to deserialize keypair account (invalid public key)"))?; - *buf = &buf[PUBLIC_KEY_SIZE..]; - let ecdsa = BorshDeserialize::deserialize(buf)?; + let ecdsa = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { public_key, ecdsa }) } @@ -181,6 +179,7 @@ impl Account for Keypair { KEYPAIR_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), self.prv_key_data_id.into(), self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/account/variants/legacy.rs b/wallet/core/src/account/variants/legacy.rs index 645c3cd..dfaf346 100644 --- a/wallet/core/src/account/variants/legacy.rs +++ b/wallet/core/src/account/variants/legacy.rs @@ -59,9 +59,9 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; Ok(Self {}) } @@ -192,6 +192,7 @@ impl Account for Legacy { LEGACY_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), self.prv_key_data_id.into(), self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/account/variants/multisig.rs b/wallet/core/src/account/variants/multisig.rs index b146f9b..41de9e8 100644 --- a/wallet/core/src/account/variants/multisig.rs +++ b/wallet/core/src/account/variants/multisig.rs @@ -70,14 +70,14 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let xpub_keys = BorshDeserialize::deserialize(buf)?; - let cosigner_index = BorshDeserialize::deserialize(buf)?; - let minimum_signatures = BorshDeserialize::deserialize(buf)?; - let ecdsa = BorshDeserialize::deserialize(buf)?; + let xpub_keys = BorshDeserialize::deserialize_reader(reader)?; + let cosigner_index = BorshDeserialize::deserialize_reader(reader)?; + let minimum_signatures = BorshDeserialize::deserialize_reader(reader)?; + let ecdsa = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { xpub_keys, cosigner_index, minimum_signatures, ecdsa }) } @@ -232,6 +232,7 @@ impl Account for MultiSig { MULTISIG_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), self.prv_key_data_ids.clone().try_into()?, self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/account/variants/resident.rs b/wallet/core/src/account/variants/resident.rs index f6d348d..b3a28ac 100644 --- a/wallet/core/src/account/variants/resident.rs +++ b/wallet/core/src/account/variants/resident.rs @@ -77,6 +77,7 @@ impl Account for Resident { RESIDENT_ACCOUNT_KIND.into(), *self.id(), self.name(), + self.balance(), AssocPrvKeyDataIds::None, self.receive_address().ok(), self.change_address().ok(), diff --git a/wallet/core/src/api/message.rs b/wallet/core/src/api/message.rs index fe89b79..a1a610d 100644 --- a/wallet/core/src/api/message.rs +++ b/wallet/core/src/api/message.rs @@ -44,6 +44,47 @@ pub struct FlushResponse {} pub struct ConnectRequest { pub url: Option, pub network_id: NetworkId, + // retry on error, otherwise give up + pub retry_on_error: bool, + // block async call until connected, otherwise return immediately + // and continue attempting to connect in the background + pub block_async_connect: bool, + // require node to be synced, fail otherwise + pub require_sync: bool, +} + +impl Default for ConnectRequest { + fn default() -> Self { + Self { + url: None, + network_id: NetworkId::new(NetworkType::Mainnet), + retry_on_error: true, + block_async_connect: true, + require_sync: true, + } + } +} + +impl ConnectRequest { + pub fn with_url(self, url: Option) -> Self { + ConnectRequest { url, ..self } + } + + pub fn with_network_id(self, network_id: &NetworkId) -> Self { + ConnectRequest { network_id: *network_id, ..self } + } + + pub fn with_retry_on_error(self, retry_on_error: bool) -> Self { + ConnectRequest { retry_on_error, ..self } + } + + pub fn with_block_async_connect(self, block_async_connect: bool) -> Self { + ConnectRequest { block_async_connect, ..self } + } + + pub fn with_require_sync(self, require_sync: bool) -> Self { + ConnectRequest { require_sync, ..self } + } } #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] diff --git a/wallet/core/src/api/traits.rs b/wallet/core/src/api/traits.rs index b3b2935..8719061 100644 --- a/wallet/core/src/api/traits.rs +++ b/wallet/core/src/api/traits.rs @@ -42,8 +42,12 @@ pub trait WalletApi: Send + Sync + AnySync { /// - `is_wrpc_client` - whether the wallet is connected to a node via wRPC async fn get_status_call(self: Arc, request: GetStatusRequest) -> Result; - async fn connect(self: Arc, url: Option, network_id: NetworkId) -> Result<()> { - self.connect_call(ConnectRequest { url, network_id }).await?; + /// Synchronous connect call (blocking, single attempt, requires sync). + async fn connect(self: Arc, url: Option, network_id: &NetworkId) -> Result<()> { + let retry_on_error = false; + let block_async_connect = true; + let require_sync = true; + self.connect_call(ConnectRequest { url, network_id: *network_id, retry_on_error, block_async_connect, require_sync }).await?; Ok(()) } diff --git a/wallet/core/src/api/transport.rs b/wallet/core/src/api/transport.rs index 9328ba7..dfaea1c 100644 --- a/wallet/core/src/api/transport.rs +++ b/wallet/core/src/api/transport.rs @@ -18,7 +18,7 @@ use crate::imports::*; use crate::result::Result; use crate::wallet::Wallet; use async_trait::async_trait; -use borsh::{BorshDeserialize, BorshSerialize}; +use borsh::BorshDeserialize; use spectre_wallet_macros::{build_wallet_client_transport_interface, build_wallet_server_transport_interface}; use workflow_core::task::spawn; diff --git a/wallet/core/src/compat/gen1.rs b/wallet/core/src/compat/gen1.rs index 3a0c1c4..0475386 100644 --- a/wallet/core/src/compat/gen1.rs +++ b/wallet/core/src/compat/gen1.rs @@ -14,7 +14,7 @@ pub fn decrypt_mnemonic>( let mut aead = chacha20poly1305::XChaCha20Poly1305::new(Key::from_slice(&key)); let (nonce, ciphertext) = cipher.as_ref().split_at(24); - let decrypted = aead.decrypt(nonce.into(), ciphertext).unwrap(); + let decrypted = aead.decrypt(nonce.into(), ciphertext)?; Ok(unsafe { String::from_utf8_unchecked(decrypted) }) } @@ -36,8 +36,10 @@ mod test { ecdsa: false, }; - let decrypted = decrypt_mnemonic(8, file.encrypted_mnemonic, b"").unwrap(); - assert_eq!("dizzy uncover funny time weapon chat volume squirrel comic motion until diamond response remind hurt spider door strategy entire oyster hawk marriage soon fabric", decrypted); + let decrypted = decrypt_mnemonic(8, file.encrypted_mnemonic, b""); + log_info!("decrypted: {decrypted:?}"); + assert!(decrypted.is_ok(), "decrypt error"); + assert_eq!("dizzy uncover funny time weapon chat volume squirrel comic motion until diamond response remind hurt spider door strategy entire oyster hawk marriage soon fabric", decrypted.unwrap()); } #[tokio::test] diff --git a/wallet/core/src/derivation.rs b/wallet/core/src/derivation.rs index 2a438a2..aeb9a65 100644 --- a/wallet/core/src/derivation.rs +++ b/wallet/core/src/derivation.rs @@ -15,7 +15,7 @@ use crate::error::Error; use crate::imports::*; use crate::result::Result; use spectre_bip32::{AddressType, DerivationPath, ExtendedPrivateKey, ExtendedPublicKey, Language, Mnemonic, SecretKeyExt}; -use spectre_consensus_core::network::NetworkType; +use spectre_consensus_core::network::{NetworkType, NetworkTypeT}; use spectre_txscript::{ extract_script_pub_key_address, multisig_redeem_script, multisig_redeem_script_ecdsa, pay_to_script_hash_script, }; @@ -458,20 +458,26 @@ pub fn create_multisig_address( /// @category Wallet SDK #[wasm_bindgen(js_name=createAddress)] pub fn create_address_js( - key: PublicKeyT, - network_type: NetworkType, + key: &PublicKeyT, + network: &NetworkTypeT, ecdsa: Option, account_kind: Option, ) -> Result
{ let public_key = PublicKey::try_cast_from(key)?; - create_address(1, vec![public_key.as_ref().try_into()?], network_type.into(), ecdsa.unwrap_or(false), account_kind) + create_address( + 1, + vec![public_key.as_ref().try_into()?], + NetworkType::try_from(network)?.into(), + ecdsa.unwrap_or(false), + account_kind, + ) } /// @category Wallet SDK #[wasm_bindgen(js_name=createMultisigAddress)] pub fn create_multisig_address_js( minimum_signatures: usize, - keys: PublicKeyArrayT, + keys: &PublicKeyArrayT, network_type: NetworkType, ecdsa: Option, account_kind: Option, diff --git a/wallet/core/src/deterministic.rs b/wallet/core/src/deterministic.rs index 8fd2ff6..ca66f0d 100644 --- a/wallet/core/src/deterministic.rs +++ b/wallet/core/src/deterministic.rs @@ -101,7 +101,7 @@ where T: AsSlice + BorshSerialize, { let mut hashes: [Hash; N] = [Hash::default(); N]; - let bytes = hashable.try_to_vec().unwrap(); + let bytes = borsh::to_vec(&hashable).unwrap(); hashes[0] = Hash::from_slice(sha256_hash(&bytes).as_ref()); for i in 1..N { hashes[i] = Hash::from_slice(sha256_hash(&hashes[i - 1].as_bytes()).as_ref()); @@ -143,7 +143,7 @@ pub fn from_multisig(prv_key_data_ids: &Option( ecdsa: Some(data.ecdsa), account_index: None, secp256k1_public_key: None, - data: Some(data.xpub_keys.try_to_vec().unwrap()), + data: Some(borsh::to_vec(&data.xpub_keys).unwrap()), }; make_hashes(hashable) } @@ -177,26 +177,26 @@ pub(crate) fn from_keypair(prv_key_data_id: &PrvKeyDataId, data: make_hashes(hashable) } -/// Create deterministic hashes from bip32-watch. -pub fn from_bip32_watch(public_key: &PublicKey) -> [Hash; N] { +/// Create deterministic hashes from a public key. +pub fn from_public_key(account_kind: &AccountKind, public_key: &PublicKey) -> [Hash; N] { let hashable: DeterministicHashData<[PrvKeyDataId; 0]> = DeterministicHashData { - account_kind: &bip32watch::BIP32_WATCH_ACCOUNT_KIND.into(), + account_kind, prv_key_data_ids: &None, ecdsa: None, - account_index: Some(0), + account_index: None, secp256k1_public_key: Some(public_key.serialize().to_vec()), data: None, }; make_hashes(hashable) } -/// Create deterministic hashes from a public key. -pub fn from_public_key(account_kind: &AccountKind, public_key: &PublicKey) -> [Hash; N] { +/// Create deterministic hashes from bip32-watch. +pub fn from_bip32_watch(public_key: &PublicKey) -> [Hash; N] { let hashable: DeterministicHashData<[PrvKeyDataId; 0]> = DeterministicHashData { - account_kind, + account_kind: &bip32watch::BIP32_WATCH_ACCOUNT_KIND.into(), prv_key_data_ids: &None, ecdsa: None, - account_index: None, + account_index: Some(0), secp256k1_public_key: Some(public_key.serialize().to_vec()), data: None, }; diff --git a/wallet/core/src/encryption.rs b/wallet/core/src/encryption.rs index bee57a1..f07b49c 100644 --- a/wallet/core/src/encryption.rs +++ b/wallet/core/src/encryption.rs @@ -146,7 +146,7 @@ where } pub fn encrypt(&self, secret: &Secret, encryption_kind: EncryptionKind) -> Result { - let bytes = self.0.try_to_vec()?; + let bytes = borsh::to_vec(&self.0)?; let encrypted = match encryption_kind { EncryptionKind::XChaCha20Poly1305 => encrypt_xchacha20poly1305(bytes.as_slice(), secret)?, }; diff --git a/wallet/core/src/error.rs b/wallet/core/src/error.rs index 86f5b99..b34d2dc 100644 --- a/wallet/core/src/error.rs +++ b/wallet/core/src/error.rs @@ -13,6 +13,7 @@ use std::sync::PoisonError; use thiserror::Error; use wasm_bindgen::JsValue; use workflow_core::abortable::Aborted; +use workflow_core::channel::{RecvError, SendError, TrySendError}; use workflow_core::sendable::*; use workflow_rpc::client::error::Error as RpcError; use workflow_wasm::jserror::*; @@ -332,6 +333,14 @@ pub enum Error { #[error(transparent)] Metrics(#[from] spectre_metrics_core::error::Error), + + #[error("Connected node is not synced")] + NotSynced, + #[error(transparent)] + Psst(#[from] spectre_wallet_psst::error::Error), + + #[error("Error generating pending transaction from PSST: {0}")] + PendingTransactionFromPSSTError(String), } impl From for Error { @@ -415,8 +424,20 @@ impl From> for Error { } } -impl From> for Error { - fn from(e: workflow_core::channel::SendError) -> Self { +impl From> for Error { + fn from(e: SendError) -> Self { + Error::Custom(e.to_string()) + } +} + +impl From for Error { + fn from(e: RecvError) -> Self { + Error::Custom(e.to_string()) + } +} + +impl From> for Error { + fn from(e: TrySendError) -> Self { Error::Custom(e.to_string()) } } diff --git a/wallet/core/src/imports.rs b/wallet/core/src/imports.rs index 51c5822..a157e49 100644 --- a/wallet/core/src/imports.rs +++ b/wallet/core/src/imports.rs @@ -17,7 +17,6 @@ pub use crate::rpc::Rpc; pub use crate::rpc::{DynRpcApi, RpcCtl}; pub use crate::serializer::*; pub use crate::storage::*; -pub use crate::tx::MassCombinationStrategy; pub use crate::utxo::balance::Balance; pub use crate::utxo::scan::{Scan, ScanExtent}; pub use crate::utxo::{Maturity, NetworkParams, OutgoingTransaction, UtxoContext, UtxoEntryReference, UtxoProcessor}; @@ -49,6 +48,7 @@ pub use std::collections::{HashMap, HashSet}; pub use std::pin::Pin; pub use std::str::FromStr; pub use std::sync::atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}; +pub use std::sync::LazyLock; pub use std::sync::{Arc, Mutex, MutexGuard, RwLock}; pub use std::task::{Context, Poll}; pub use wasm_bindgen::prelude::*; diff --git a/wallet/core/src/lib.rs b/wallet/core/src/lib.rs index 8667380..4ba4b27 100644 --- a/wallet/core/src/lib.rs +++ b/wallet/core/src/lib.rs @@ -119,5 +119,10 @@ pub fn version() -> String { env!("CARGO_PKG_VERSION").to_string() } +/// Returns the version of the Wallet framework combined with short git hash. +pub fn version_with_git_hash() -> String { + spectre_utils::git::with_short_hash(env!("CARGO_PKG_VERSION")).to_string() +} + #[cfg(test)] pub mod tests; diff --git a/wallet/core/src/prelude.rs b/wallet/core/src/prelude.rs index 54bcd46..91bc7aa 100644 --- a/wallet/core/src/prelude.rs +++ b/wallet/core/src/prelude.rs @@ -14,9 +14,14 @@ pub use crate::rpc::{ConnectOptions, ConnectStrategy, DynRpcApi}; pub use crate::settings::WalletSettings; pub use crate::storage::{IdT, Interface, PrvKeyDataId, PrvKeyDataInfo, TransactionId, TransactionRecord, WalletDescriptor}; pub use crate::tx::{Fees, PaymentDestination, PaymentOutput, PaymentOutputs}; +pub use crate::utils::{ + sompi_to_spectre, sompi_to_spectre_string, sompi_to_spectre_string_with_suffix, spectre_suffix, spectre_to_sompi, + try_spectre_str_to_sompi, try_spectre_str_to_sompi_i64, +}; pub use crate::utxo::balance::{Balance, BalanceStrings}; pub use crate::wallet::args::*; pub use crate::wallet::Wallet; +pub use async_std::sync::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; pub use spectre_addresses::{Address, Prefix as AddressPrefix}; pub use spectre_bip32::{Language, Mnemonic, WordCount}; pub use spectre_wallet_keys::secret::Secret; diff --git a/wallet/core/src/serializer.rs b/wallet/core/src/serializer.rs index 391f813..d7451a3 100644 --- a/wallet/core/src/serializer.rs +++ b/wallet/core/src/serializer.rs @@ -59,8 +59,8 @@ impl BorshSerialize for StorageHeader { } impl BorshDeserialize for StorageHeader { - fn deserialize(buf: &mut &[u8]) -> std::io::Result { - let (magic, version): (u32, u32) = BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> std::io::Result { + let (magic, version): (u32, u32) = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { magic, version }) } } diff --git a/wallet/core/src/storage/account.rs b/wallet/core/src/storage/account.rs index da0c2df..a33585a 100644 --- a/wallet/core/src/storage/account.rs +++ b/wallet/core/src/storage/account.rs @@ -26,10 +26,10 @@ impl BorshSerialize for AccountSettings { } impl BorshDeserialize for AccountSettings { - fn deserialize(buf: &mut &[u8]) -> IoResult { - let _version: u32 = BorshDeserialize::deserialize(buf)?; - let name = BorshDeserialize::deserialize(buf)?; - let meta = BorshDeserialize::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> IoResult { + let _version: u32 = BorshDeserialize::deserialize_reader(reader)?; + let name = BorshDeserialize::deserialize_reader(reader)?; + let meta = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { name, meta }) } @@ -63,7 +63,7 @@ impl AccountStorage { where A: AccountStorable, { - Ok(Self { id: *id, storage_key: *storage_key, kind, prv_key_data_ids, settings, serialized: serialized.try_to_vec()? }) + Ok(Self { id: *id, storage_key: *storage_key, kind, prv_key_data_ids, settings, serialized: borsh::to_vec(&serialized)? }) } pub fn id(&self) -> &AccountId { @@ -107,16 +107,16 @@ impl BorshSerialize for AccountStorage { } impl BorshDeserialize for AccountStorage { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - - let kind = BorshDeserialize::deserialize(buf)?; - let id = BorshDeserialize::deserialize(buf)?; - let storage_key = BorshDeserialize::deserialize(buf)?; - let prv_key_data_ids = BorshDeserialize::deserialize(buf)?; - let settings = BorshDeserialize::deserialize(buf)?; - let serialized = BorshDeserialize::deserialize(buf)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + + let kind = BorshDeserialize::deserialize_reader(reader)?; + let id = BorshDeserialize::deserialize_reader(reader)?; + let storage_key = BorshDeserialize::deserialize_reader(reader)?; + let prv_key_data_ids = BorshDeserialize::deserialize_reader(reader)?; + let settings = BorshDeserialize::deserialize_reader(reader)?; + let serialized = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { kind, id, storage_key, prv_key_data_ids, settings, serialized }) } diff --git a/wallet/core/src/storage/binding.rs b/wallet/core/src/storage/binding.rs index 45eac4a..18f988e 100644 --- a/wallet/core/src/storage/binding.rs +++ b/wallet/core/src/storage/binding.rs @@ -5,6 +5,45 @@ use crate::imports::*; use crate::utxo::{UtxoContextBinding as UtxoProcessorBinding, UtxoContextId}; +#[wasm_bindgen(typescript_custom_section)] +const ITransactionRecord: &'static str = r#" + +/** + * Type of a binding record. + * @see {@link IBinding}, {@link ITransactionDataVariant}, {@link ITransactionRecord} + * @category Wallet SDK + */ +export enum BindingType { + /** + * The data structure is associated with a user-supplied id. + * @see {@link IBinding} + */ + Custom = "custom", + /** + * The data structure is associated with a wallet account. + * @see {@link IBinding}, {@link Account} + */ + Account = "account", +} + +/** + * Internal transaction data contained within the transaction record. + * @see {@link ITransactionRecord} + * @category Wallet SDK + */ +export interface IBinding { + type : BindingType; + data : HexString; +} +"#; + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(extends = Object, typescript_type = "IBinding")] + #[derive(Clone, Debug, PartialEq, Eq)] + pub type BindingT; +} + #[derive(Clone, Debug, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] #[serde(rename_all = "kebab-case")] #[serde(tag = "type", content = "id")] diff --git a/wallet/core/src/storage/keydata/data.rs b/wallet/core/src/storage/keydata/data.rs index 789f1c3..55ba019 100644 --- a/wallet/core/src/storage/keydata/data.rs +++ b/wallet/core/src/storage/keydata/data.rs @@ -44,11 +44,12 @@ impl BorshSerialize for PrvKeyDataVariant { } impl BorshDeserialize for PrvKeyDataVariant { - fn deserialize(buf: &mut &[u8]) -> IoResult { - let StorageHeader { version: _, .. } = StorageHeader::deserialize(buf)?.try_magic(Self::MAGIC)?.try_version(Self::VERSION)?; + fn deserialize_reader(reader: &mut R) -> IoResult { + let StorageHeader { version: _, .. } = + StorageHeader::deserialize_reader(reader)?.try_magic(Self::MAGIC)?.try_version(Self::VERSION)?; - let kind: PrvKeyDataVariantKind = BorshDeserialize::deserialize(buf)?; - let string: String = BorshDeserialize::deserialize(buf)?; + let kind: PrvKeyDataVariantKind = BorshDeserialize::deserialize_reader(reader)?; + let string: String = BorshDeserialize::deserialize_reader(reader)?; match kind { PrvKeyDataVariantKind::Mnemonic => Ok(Self::Mnemonic(string)), diff --git a/wallet/core/src/storage/local/interface.rs b/wallet/core/src/storage/local/interface.rs index 1e998eb..c4ada71 100644 --- a/wallet/core/src/storage/local/interface.rs +++ b/wallet/core/src/storage/local/interface.rs @@ -131,7 +131,7 @@ impl LocalStoreInner { async fn try_export(&self, wallet_secret: &Secret, _options: WalletExportOptions) -> Result> { let wallet = self.cache.read().unwrap().to_wallet(None, wallet_secret)?; - Ok(wallet.try_to_vec()?) + Ok(borsh::to_vec(&wallet)?) } fn storage(&self) -> Arc { diff --git a/wallet/core/src/storage/local/payload.rs b/wallet/core/src/storage/local/payload.rs index 2e701d9..b7fe86f 100644 --- a/wallet/core/src/storage/local/payload.rs +++ b/wallet/core/src/storage/local/payload.rs @@ -67,13 +67,13 @@ impl BorshSerialize for Payload { } impl BorshDeserialize for Payload { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let prv_key_data = BorshDeserialize::deserialize(buf)?; - let accounts = BorshDeserialize::deserialize(buf)?; - let address_book = BorshDeserialize::deserialize(buf)?; - let encrypt_transactions = BorshDeserialize::deserialize(buf)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + let prv_key_data = BorshDeserialize::deserialize_reader(reader)?; + let accounts = BorshDeserialize::deserialize_reader(reader)?; + let address_book = BorshDeserialize::deserialize_reader(reader)?; + let encrypt_transactions = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { prv_key_data, accounts, address_book, encrypt_transactions }) } diff --git a/wallet/core/src/storage/local/transaction/fsio.rs b/wallet/core/src/storage/local/transaction/fsio.rs index e95c82d..969034f 100644 --- a/wallet/core/src/storage/local/transaction/fsio.rs +++ b/wallet/core/src/storage/local/transaction/fsio.rs @@ -65,7 +65,10 @@ impl TransactionStore { match fs::readdir(folder, true).await { Ok(mut files) => { // we reverse the order of the files so that the newest files are first - files.sort_by_key(|f| std::cmp::Reverse(f.metadata().unwrap().created())); + files.sort_by_key(|f| { + let meta = f.metadata().expect("fsio: missing file metadata"); + std::cmp::Reverse(meta.created().or_else(|| meta.modified()).unwrap_or_default()) + }); for file in files { if let Ok(id) = TransactionId::from_hex(file.file_name()) { @@ -315,6 +318,6 @@ async fn write(path: &Path, record: &TransactionRecord, secret: Option<&Secret>, } else { Encryptable::from(record.clone()) }; - fs::write(path, &data.try_to_vec()?).await?; + fs::write(path, &borsh::to_vec(&data)?).await?; Ok(()) } diff --git a/wallet/core/src/storage/local/transaction/indexdb.rs b/wallet/core/src/storage/local/transaction/indexdb.rs index e4190aa..463508f 100644 --- a/wallet/core/src/storage/local/transaction/indexdb.rs +++ b/wallet/core/src/storage/local/transaction/indexdb.rs @@ -26,35 +26,104 @@ pub struct Inner { impl Inner { async fn open_db(&self, db_name: String) -> Result { call_async_no_send!(async move { - let mut db_req: OpenDbRequest = IdbDatabase::open_u32(&db_name, 1) + let mut db_req: OpenDbRequest = IdbDatabase::open_u32(&db_name, 2) .map_err(|err| Error::Custom(format!("Failed to open indexdb database {:?}", err)))?; - - fn on_upgrade_needed(evt: &IdbVersionChangeEvent) -> Result<(), JsValue> { - // Check if the object store exists; create it if it doesn't - if !evt.db().object_store_names().any(|n| n == TRANSACTIONS_STORE_NAME) { + let fix_timestamp = Arc::new(Mutex::new(false)); + let fix_timestamp_clone = fix_timestamp.clone(); + let on_upgrade_needed = move |evt: &IdbVersionChangeEvent| -> Result<(), JsValue> { + let old_version = evt.old_version(); + if old_version < 1.0 { let object_store = evt.db().create_object_store(TRANSACTIONS_STORE_NAME)?; + let db_index_params = IdbIndexParameters::new(); + db_index_params.set_unique(true); object_store.create_index_with_params( TRANSACTIONS_STORE_ID_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_ID_INDEX), - IdbIndexParameters::new().unique(true), + &db_index_params, )?; object_store.create_index_with_params( TRANSACTIONS_STORE_TIMESTAMP_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_TIMESTAMP_INDEX), - IdbIndexParameters::new().unique(false), + &db_index_params, )?; object_store.create_index_with_params( TRANSACTIONS_STORE_DATA_INDEX, &IdbKeyPath::str(TRANSACTIONS_STORE_DATA_INDEX), - IdbIndexParameters::new().unique(false), + &db_index_params, )?; + + // these changes are not required for new db + } else if old_version < 2.0 { + *fix_timestamp_clone.lock().unwrap() = true; } + // // Check if the object store exists; create it if it doesn't + // if !evt.db().object_store_names().any(|n| n == TRANSACTIONS_STORE_NAME) { + + // } Ok(()) - } + }; db_req.set_on_upgrade_needed(Some(on_upgrade_needed)); - db_req.await.map_err(|err| Error::Custom(format!("Open database request failed for indexdb database {:?}", err))) + let db = + db_req.await.map_err(|err| Error::Custom(format!("Open database request failed for indexdb database {:?}", err)))?; + + if *fix_timestamp.lock().unwrap() { + log_info!("DEBUG: fixing timestamp"); + let idb_tx = db + .transaction_on_one_with_mode(TRANSACTIONS_STORE_NAME, IdbTransactionMode::Readwrite) + .map_err(|err| Error::Custom(format!("Failed to open indexdb transaction for reading {:?}", err)))?; + let store = idb_tx + .object_store(TRANSACTIONS_STORE_NAME) + .map_err(|err| Error::Custom(format!("Failed to open indexdb object store for reading {:?}", err)))?; + let binding = store + .index(TRANSACTIONS_STORE_TIMESTAMP_INDEX) + .map_err(|err| Error::Custom(format!("Failed to open indexdb indexed store cursor {:?}", err)))?; + let cursor = binding + .open_cursor_with_range_and_direction(&JsValue::NULL, web_sys::IdbCursorDirection::Prev) + .map_err(|err| Error::Custom(format!("Failed to open indexdb store cursor for reading {:?}", err)))?; + let cursor = cursor.await.map_err(|err| Error::Custom(format!("Failed to open indexdb store cursor {:?}", err)))?; + + // let next_year_date = Date::new_0(); + // next_year_date.set_full_year(next_year_date.get_full_year() + 1); + // let next_year_ts = next_year_date.get_time(); + + if let Some(cursor) = cursor { + loop { + let js_value = cursor.value(); + if let Ok(record) = transaction_record_from_js_value(&js_value, None) { + if record.unixtime_msec.is_some() { + let new_js_value = transaction_record_to_js_value(&record, None, ENCRYPTION_KIND)?; + + //log_info!("DEBUG: new_js_value: {:?}", new_js_value); + + cursor + .update(&new_js_value) + .map_err(|err| Error::Custom(format!("Failed to update record timestamp {:?}", err)))? + .await + .map_err(|err| Error::Custom(format!("Failed to update record timestamp {:?}", err)))?; + } + } + if let Ok(b) = cursor.continue_cursor() { + match b.await { + Ok(b) => { + if !b { + break; + } + } + Err(err) => { + log_info!("DEBUG IDB: Loading transaction error, cursor.continue_cursor() {:?}", err); + break; + } + } + } else { + break; + } + } + } + } + + Ok(db) }) } } @@ -218,36 +287,73 @@ impl TransactionRecordStore for TransactionStore { binding: &Binding, network_id: &NetworkId, _filter: Option>, - _range: std::ops::Range, + range: std::ops::Range, ) -> Result { - // log_info!("DEBUG IDB: Loading transaction records for range {:?}", _range); - + log_info!("DEBUG IDB: Loading transaction records for range {:?}", range); let binding_str = binding.to_hex(); let network_id_str = network_id.to_string(); let db_name = self.make_db_name(&binding_str, &network_id_str); - let inner = self.inner().clone(); - call_async_no_send!(async move { let db = inner.open_db(db_name).await?; - let idb_tx = db .transaction_on_one_with_mode(TRANSACTIONS_STORE_NAME, IdbTransactionMode::Readonly) .map_err(|err| Error::Custom(format!("Failed to open indexdb transaction for reading {:?}", err)))?; - let store = idb_tx .object_store(TRANSACTIONS_STORE_NAME) .map_err(|err| Error::Custom(format!("Failed to open indexdb object store for reading {:?}", err)))?; - - let array = store - .get_all() - .map_err(|err| Error::Custom(format!("Failed to get transaction record from indexdb {:?}", err)))? + let total = store + .count() + .map_err(|err| Error::Custom(format!("Failed to count indexdb records {:?}", err)))? .await - .map_err(|err| Error::Custom(format!("Failed to get transaction record from indexdb {:?}", err)))?; - - let transactions = array + .map_err(|err| Error::Custom(format!("Failed to count indexdb records from future {:?}", err)))?; + + let binding = store + .index(TRANSACTIONS_STORE_TIMESTAMP_INDEX) + .map_err(|err| Error::Custom(format!("Failed to open indexdb indexed store cursor {:?}", err)))?; + let cursor = binding + .open_cursor_with_range_and_direction(&JsValue::NULL, web_sys::IdbCursorDirection::Prev) + .map_err(|err| Error::Custom(format!("Failed to open indexdb store cursor for reading {:?}", err)))?; + let mut records = vec![]; + let cursor = cursor.await.map_err(|err| Error::Custom(format!("Failed to open indexdb store cursor {:?}", err)))?; + if let Some(cursor) = cursor { + if range.start > 0 { + let res = cursor + .advance(range.start as u32) + .map_err(|err| Error::Custom(format!("Unable to advance indexdb cursor {:?}", err)))? + .await; + let _res = res.map_err(|err| Error::Custom(format!("Unable to advance indexdb cursor future {:?}", err)))?; + // if !res { + // //return Err(Error::Custom(format!("Unable to advance indexdb cursor future {:?}", err))); + // } + } + let count = range.end - range.start; + loop { + if records.len() < count { + records.push(cursor.value()); + if let Ok(b) = cursor.continue_cursor() { + match b.await { + Ok(b) => { + if !b { + break; + } + } + Err(err) => { + log_info!("DEBUG IDB: Loading transaction error, cursor.continue_cursor() {:?}", err); + break; + } + } + } else { + break; + } + } else { + break; + } + } + } + let transactions = records .iter() - .filter_map(|js_value| match transaction_record_from_js_value(&js_value, None) { + .filter_map(|js_value| match transaction_record_from_js_value(js_value, None) { Ok(transaction_record) => Some(Arc::new(transaction_record)), Err(err) => { log_error!("Failed to deserialize transaction record from indexdb {:?}", err); @@ -256,8 +362,7 @@ impl TransactionRecordStore for TransactionStore { }) .collect::>(); - let total = transactions.len() as u64; - Ok(TransactionRangeResult { transactions, total }) + Ok(TransactionRangeResult { transactions, total: total.into() }) }) } @@ -285,7 +390,7 @@ impl TransactionRecordStore for TransactionStore { let inner = inner_guard.lock().unwrap().clone(); call_async_no_send!(async move { - for (db_name, items) in &items.into_iter().group_by(|item| item.db_name.clone()) { + for (db_name, items) in &items.into_iter().chunk_by(|item| item.db_name.clone()) { let db = inner.open_db(db_name).await?; let idb_tx = db @@ -474,16 +579,14 @@ fn transaction_record_to_js_value( ) -> Result { let id = transaction_record.id.to_string(); let unixtime_msec = transaction_record.unixtime_msec; - let mut borsh_data = vec![]; - ::serialize(transaction_record, &mut borsh_data)?; let id_js_value = JsValue::from_str(&id); let timestamp_js_value = match unixtime_msec { Some(unixtime_msec) => { - let unixtime_sec = (unixtime_msec / 1000) as u32; + //let unixtime_sec = (unixtime_msec / 1000) as u32; let date = Date::new_0(); - date.set_utc_seconds(unixtime_sec); + date.set_time(unixtime_msec as f64); date.into() } None => JsValue::NULL, @@ -494,7 +597,7 @@ fn transaction_record_to_js_value( } else { Encryptable::from(transaction_record.clone()) }; - let encryped_data_vec = encryped_data.try_to_vec()?; + let encryped_data_vec = borsh::to_vec(&encryped_data)?; let borsh_data_uint8_arr = Uint8Array::from(encryped_data_vec.as_slice()); let borsh_data_js_value = borsh_data_uint8_arr.into(); @@ -519,6 +622,6 @@ fn transaction_record_from_js_value(js_value: &JsValue, secret: Option<&Secret>) Ok(transaction_record.0) } else { - Err(Error::Custom("supplied argument must be an object".to_string())) + Err(Error::Custom("supplied argument must be an object, found ({js_value:?})".to_string())) } } diff --git a/wallet/core/src/storage/local/wallet.rs b/wallet/core/src/storage/local/wallet.rs index 2f14080..321527f 100644 --- a/wallet/core/src/storage/local/wallet.rs +++ b/wallet/core/src/storage/local/wallet.rs @@ -61,7 +61,7 @@ impl WalletStorage { cfg_if! { if #[cfg(target_arch = "wasm32")] { - let serialized = BorshSerialize::try_to_vec(self)?; + let serialized = borsh::to_vec(self)?; fs::write(store.filename(), serialized.as_slice()).await?; } else { // make this platform-specific to avoid creating @@ -101,8 +101,8 @@ impl BorshSerialize for WalletStorage { } impl BorshDeserialize for WalletStorage { - fn deserialize(buf: &mut &[u8]) -> IoResult { - let StorageHeader { magic, version, .. } = StorageHeader::deserialize(buf)?; + fn deserialize_reader(reader: &mut R) -> IoResult { + let StorageHeader { magic, version, .. } = StorageHeader::deserialize_reader(reader)?; if magic != Self::STORAGE_MAGIC { return Err(IoError::new( @@ -118,12 +118,12 @@ impl BorshDeserialize for WalletStorage { )); } - let title = BorshDeserialize::deserialize(buf)?; - let user_hint = BorshDeserialize::deserialize(buf)?; - let encryption_kind = BorshDeserialize::deserialize(buf)?; - let payload = BorshDeserialize::deserialize(buf)?; - let metadata = BorshDeserialize::deserialize(buf)?; - let transactions = BorshDeserialize::deserialize(buf)?; + let title = BorshDeserialize::deserialize_reader(reader)?; + let user_hint = BorshDeserialize::deserialize_reader(reader)?; + let encryption_kind = BorshDeserialize::deserialize_reader(reader)?; + let payload = BorshDeserialize::deserialize_reader(reader)?; + let metadata = BorshDeserialize::deserialize_reader(reader)?; + let transactions = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { title, user_hint, encryption_kind, payload, metadata, transactions }) } diff --git a/wallet/core/src/storage/metadata.rs b/wallet/core/src/storage/metadata.rs index 0eacb90..d421a17 100644 --- a/wallet/core/src/storage/metadata.rs +++ b/wallet/core/src/storage/metadata.rs @@ -47,12 +47,12 @@ impl BorshSerialize for AccountMetadata { } impl BorshDeserialize for AccountMetadata { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let id = BorshDeserialize::deserialize(buf)?; - let indexes = BorshDeserialize::deserialize(buf)?; + let id = BorshDeserialize::deserialize_reader(reader)?; + let indexes = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { id, indexes }) } diff --git a/wallet/core/src/storage/mod.rs b/wallet/core/src/storage/mod.rs index 67f17af..3081dc5 100644 --- a/wallet/core/src/storage/mod.rs +++ b/wallet/core/src/storage/mod.rs @@ -18,7 +18,7 @@ pub mod transaction; pub use account::{AccountSettings, AccountStorable, AccountStorage}; pub use address::AddressBookEntry; -pub use binding::Binding; +pub use binding::{Binding, BindingT}; pub use hint::Hint; pub use id::IdT; pub use interface::{ diff --git a/wallet/core/src/storage/transaction/data.rs b/wallet/core/src/storage/transaction/data.rs index 1d41699..cef6ae6 100644 --- a/wallet/core/src/storage/transaction/data.rs +++ b/wallet/core/src/storage/transaction/data.rs @@ -282,42 +282,42 @@ impl BorshSerialize for TransactionData { } impl BorshDeserialize for TransactionData { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - let kind: TransactionKind = BorshDeserialize::deserialize(buf)?; + let kind: TransactionKind = BorshDeserialize::deserialize_reader(reader)?; match kind { TransactionKind::Reorg => { - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Reorg { utxo_entries, aggregate_input_value }) } TransactionKind::Incoming => { - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Incoming { utxo_entries, aggregate_input_value }) } TransactionKind::Stasis => { - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Stasis { utxo_entries, aggregate_input_value }) } TransactionKind::External => { - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::External { utxo_entries, aggregate_input_value }) } TransactionKind::Batch => { - let fees: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_output_value: u64 = BorshDeserialize::deserialize(buf)?; - let transaction: Transaction = BorshDeserialize::deserialize(buf)?; - let payment_value: Option = BorshDeserialize::deserialize(buf)?; - let change_value: u64 = BorshDeserialize::deserialize(buf)?; - let accepted_daa_score: Option = BorshDeserialize::deserialize(buf)?; - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; + let fees: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_output_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let transaction: Transaction = BorshDeserialize::deserialize_reader(reader)?; + let payment_value: Option = BorshDeserialize::deserialize_reader(reader)?; + let change_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let accepted_daa_score: Option = BorshDeserialize::deserialize_reader(reader)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Batch { fees, aggregate_input_value, @@ -330,14 +330,14 @@ impl BorshDeserialize for TransactionData { }) } TransactionKind::Outgoing => { - let fees: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_output_value: u64 = BorshDeserialize::deserialize(buf)?; - let transaction: Transaction = BorshDeserialize::deserialize(buf)?; - let payment_value: Option = BorshDeserialize::deserialize(buf)?; - let change_value: u64 = BorshDeserialize::deserialize(buf)?; - let accepted_daa_score: Option = BorshDeserialize::deserialize(buf)?; - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; + let fees: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_output_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let transaction: Transaction = BorshDeserialize::deserialize_reader(reader)?; + let payment_value: Option = BorshDeserialize::deserialize_reader(reader)?; + let change_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let accepted_daa_score: Option = BorshDeserialize::deserialize_reader(reader)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Outgoing { fees, aggregate_input_value, @@ -350,14 +350,14 @@ impl BorshDeserialize for TransactionData { }) } TransactionKind::TransferIncoming => { - let fees: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_output_value: u64 = BorshDeserialize::deserialize(buf)?; - let transaction: Transaction = BorshDeserialize::deserialize(buf)?; - let payment_value: Option = BorshDeserialize::deserialize(buf)?; - let change_value: u64 = BorshDeserialize::deserialize(buf)?; - let accepted_daa_score: Option = BorshDeserialize::deserialize(buf)?; - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; + let fees: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_output_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let transaction: Transaction = BorshDeserialize::deserialize_reader(reader)?; + let payment_value: Option = BorshDeserialize::deserialize_reader(reader)?; + let change_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let accepted_daa_score: Option = BorshDeserialize::deserialize_reader(reader)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::TransferIncoming { fees, aggregate_input_value, @@ -370,14 +370,14 @@ impl BorshDeserialize for TransactionData { }) } TransactionKind::TransferOutgoing => { - let fees: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_output_value: u64 = BorshDeserialize::deserialize(buf)?; - let transaction: Transaction = BorshDeserialize::deserialize(buf)?; - let payment_value: Option = BorshDeserialize::deserialize(buf)?; - let change_value: u64 = BorshDeserialize::deserialize(buf)?; - let accepted_daa_score: Option = BorshDeserialize::deserialize(buf)?; - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; + let fees: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_output_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let transaction: Transaction = BorshDeserialize::deserialize_reader(reader)?; + let payment_value: Option = BorshDeserialize::deserialize_reader(reader)?; + let change_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let accepted_daa_score: Option = BorshDeserialize::deserialize_reader(reader)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::TransferOutgoing { fees, aggregate_input_value, @@ -390,13 +390,13 @@ impl BorshDeserialize for TransactionData { }) } TransactionKind::Change => { - let aggregate_input_value: u64 = BorshDeserialize::deserialize(buf)?; - let aggregate_output_value: u64 = BorshDeserialize::deserialize(buf)?; - let transaction: Transaction = BorshDeserialize::deserialize(buf)?; - let payment_value: Option = BorshDeserialize::deserialize(buf)?; - let change_value: u64 = BorshDeserialize::deserialize(buf)?; - let accepted_daa_score: Option = BorshDeserialize::deserialize(buf)?; - let utxo_entries: Vec = BorshDeserialize::deserialize(buf)?; + let aggregate_input_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let aggregate_output_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let transaction: Transaction = BorshDeserialize::deserialize_reader(reader)?; + let payment_value: Option = BorshDeserialize::deserialize_reader(reader)?; + let change_value: u64 = BorshDeserialize::deserialize_reader(reader)?; + let accepted_daa_score: Option = BorshDeserialize::deserialize_reader(reader)?; + let utxo_entries: Vec = BorshDeserialize::deserialize_reader(reader)?; Ok(TransactionData::Change { aggregate_input_value, aggregate_output_value, diff --git a/wallet/core/src/storage/transaction/record.rs b/wallet/core/src/storage/transaction/record.rs index 6cb92a8..98b3c3f 100644 --- a/wallet/core/src/storage/transaction/record.rs +++ b/wallet/core/src/storage/transaction/record.rs @@ -4,7 +4,7 @@ use super::*; use crate::imports::*; -use crate::storage::Binding; +use crate::storage::{Binding, BindingT}; use crate::tx::PendingTransactionInner; use workflow_core::time::{unixtime_as_millis_u64, unixtime_to_locale_string}; use workflow_wasm::utils::try_get_js_value_prop; @@ -289,7 +289,9 @@ export interface ITransactionRecord { extern "C" { #[wasm_bindgen(extends = Object, typescript_type = "ITransactionRecord")] #[derive(Clone, Debug, PartialEq, Eq)] - pub type ITransactionRecord; + pub type TransactionRecordT; + #[wasm_bindgen(extends = Object, typescript_type = "ITransactionData")] + pub type TransactionDataT; } #[wasm_bindgen(inspectable)] @@ -318,11 +320,12 @@ pub struct TransactionRecord { #[serde(rename = "unixtimeMsec")] #[wasm_bindgen(js_name = unixtimeMsec)] pub unixtime_msec: Option, + #[wasm_bindgen(skip)] pub value: u64, #[wasm_bindgen(skip)] pub binding: Binding, #[serde(rename = "blockDaaScore")] - #[wasm_bindgen(js_name = blockDaaScore)] + #[wasm_bindgen(skip)] pub block_daa_score: u64, #[serde(rename = "network")] #[wasm_bindgen(js_name = network)] @@ -378,9 +381,9 @@ impl TransactionRecord { let params = NetworkParams::from(self.network_id); let maturity = if self.is_coinbase() { - params.coinbase_transaction_maturity_period_daa + params.coinbase_transaction_maturity_period_daa() } else { - params.user_transaction_maturity_period_daa + params.user_transaction_maturity_period_daa() }; if current_daa_score < self.block_daa_score() + maturity { @@ -431,9 +434,9 @@ impl TransactionRecord { pub fn maturity_progress(&self, current_daa_score: u64) -> Option { let params = NetworkParams::from(self.network_id); let maturity = if self.is_coinbase() { - params.coinbase_transaction_maturity_period_daa + params.coinbase_transaction_maturity_period_daa() } else { - params.user_transaction_maturity_period_daa + params.user_transaction_maturity_period_daa() }; if current_daa_score < self.block_daa_score + maturity { @@ -784,14 +787,24 @@ impl TransactionRecord { #[wasm_bindgen] impl TransactionRecord { + #[wasm_bindgen(getter, js_name = "value")] + pub fn value_as_js_bigint(&self) -> BigInt { + self.value.into() + } + + #[wasm_bindgen(getter, js_name = "blockDaaScore")] + pub fn block_daa_score_as_js_bigint(&self) -> BigInt { + self.block_daa_score.into() + } + #[wasm_bindgen(getter, js_name = "binding")] - pub fn binding_as_js_value(&self) -> JsValue { - serde_wasm_bindgen::to_value(&self.binding).unwrap() + pub fn binding_as_js_value(&self) -> BindingT { + serde_wasm_bindgen::to_value(&self.binding).unwrap().unchecked_into() } #[wasm_bindgen(getter, js_name = "data")] - pub fn data_as_js_value(&self) -> JsValue { - try_get_js_value_prop(&serde_wasm_bindgen::to_value(&self.transaction_data).unwrap(), "data").unwrap() + pub fn data_as_js_value(&self) -> TransactionDataT { + try_get_js_value_prop(&serde_wasm_bindgen::to_value(&self.transaction_data).unwrap(), "data").unwrap().unchecked_into() } #[wasm_bindgen(getter, js_name = "type")] @@ -837,19 +850,19 @@ impl BorshSerialize for TransactionRecord { } impl BorshDeserialize for TransactionRecord { - fn deserialize(buf: &mut &[u8]) -> IoResult { + fn deserialize_reader(reader: &mut R) -> IoResult { let StorageHeader { version: _, .. } = - StorageHeader::deserialize(buf)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; - - let id = BorshDeserialize::deserialize(buf)?; - let unixtime = BorshDeserialize::deserialize(buf)?; - let value = BorshDeserialize::deserialize(buf)?; - let binding = BorshDeserialize::deserialize(buf)?; - let block_daa_score = BorshDeserialize::deserialize(buf)?; - let network_id = BorshDeserialize::deserialize(buf)?; - let transaction_data = BorshDeserialize::deserialize(buf)?; - let note = BorshDeserialize::deserialize(buf)?; - let metadata = BorshDeserialize::deserialize(buf)?; + StorageHeader::deserialize_reader(reader)?.try_magic(Self::STORAGE_MAGIC)?.try_version(Self::STORAGE_VERSION)?; + + let id = BorshDeserialize::deserialize_reader(reader)?; + let unixtime = BorshDeserialize::deserialize_reader(reader)?; + let value = BorshDeserialize::deserialize_reader(reader)?; + let binding = BorshDeserialize::deserialize_reader(reader)?; + let block_daa_score = BorshDeserialize::deserialize_reader(reader)?; + let network_id = BorshDeserialize::deserialize_reader(reader)?; + let transaction_data = BorshDeserialize::deserialize_reader(reader)?; + let note = BorshDeserialize::deserialize_reader(reader)?; + let metadata = BorshDeserialize::deserialize_reader(reader)?; Ok(Self { id, unixtime_msec: unixtime, value, binding, block_daa_score, network_id, transaction_data, note, metadata }) } @@ -861,7 +874,7 @@ impl BorshDeserialize for TransactionRecord { // } // } -impl From for ITransactionRecord { +impl From for TransactionRecordT { fn from(record: TransactionRecord) -> Self { JsValue::from(record).unchecked_into() } diff --git a/wallet/core/src/tests/rpc_core_mock.rs b/wallet/core/src/tests/rpc_core_mock.rs index cffb021..e1a1eac 100644 --- a/wallet/core/src/tests/rpc_core_mock.rs +++ b/wallet/core/src/tests/rpc_core_mock.rs @@ -9,7 +9,7 @@ use spectre_notify::scope::Scope; use spectre_notify::subscription::context::SubscriptionContext; use spectre_notify::subscription::{MutationPolicies, UtxosChangedMutationPolicy}; use spectre_rpc_core::api::ctl::RpcCtl; -use spectre_rpc_core::{api::rpc::RpcApi, *}; +use spectre_rpc_core::{api::connection::DynRpcConnection, api::rpc::RpcApi, *}; use spectre_rpc_core::{notify::connection::ChannelConnection, RpcResult}; use std::sync::Arc; @@ -83,7 +83,7 @@ impl Default for RpcCoreMock { #[async_trait] impl RpcApi for RpcCoreMock { // This fn needs to succeed while the client connects - async fn get_info_call(&self, _request: GetInfoRequest) -> RpcResult { + async fn get_info_call(&self, _connection: Option<&DynRpcConnection>, _request: GetInfoRequest) -> RpcResult { Ok(GetInfoResponse { p2p_id: "wallet-mock".to_string(), mempool_size: 1234, @@ -95,140 +95,237 @@ impl RpcApi for RpcCoreMock { }) } - async fn ping_call(&self, _request: PingRequest) -> RpcResult { + async fn ping_call(&self, _connection: Option<&DynRpcConnection>, _request: PingRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_metrics_call(&self, _request: GetMetricsRequest) -> RpcResult { + async fn get_metrics_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMetricsRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_server_info_call(&self, _request: GetServerInfoRequest) -> RpcResult { + async fn get_connections_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetConnectionsRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sync_status_call(&self, _request: GetSyncStatusRequest) -> RpcResult { + async fn get_server_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetServerInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_current_network_call(&self, _request: GetCurrentNetworkRequest) -> RpcResult { + async fn get_system_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSystemInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn submit_block_call(&self, _request: SubmitBlockRequest) -> RpcResult { + async fn get_sync_status_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSyncStatusRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_template_call(&self, _request: GetBlockTemplateRequest) -> RpcResult { + async fn get_current_network_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetCurrentNetworkRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_peer_addresses_call(&self, _request: GetPeerAddressesRequest) -> RpcResult { + async fn submit_block_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: SubmitBlockRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sink_call(&self, _request: GetSinkRequest) -> RpcResult { + async fn get_block_template_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockTemplateRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_mempool_entry_call(&self, _request: GetMempoolEntryRequest) -> RpcResult { + async fn get_peer_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetPeerAddressesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_mempool_entries_call(&self, _request: GetMempoolEntriesRequest) -> RpcResult { + async fn get_sink_call(&self, _connection: Option<&DynRpcConnection>, _request: GetSinkRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_connected_peer_info_call(&self, _request: GetConnectedPeerInfoRequest) -> RpcResult { + async fn get_mempool_entry_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMempoolEntryRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn add_peer_call(&self, _request: AddPeerRequest) -> RpcResult { + async fn get_mempool_entries_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetMempoolEntriesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn submit_transaction_call(&self, _request: SubmitTransactionRequest) -> RpcResult { + async fn get_connected_peer_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetConnectedPeerInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn submit_transaction_replacement_call( &self, + _connection: Option<&DynRpcConnection>, _request: SubmitTransactionReplacementRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_call(&self, _request: GetBlockRequest) -> RpcResult { + async fn add_peer_call(&self, _connection: Option<&DynRpcConnection>, _request: AddPeerRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_subnetwork_call(&self, _request: GetSubnetworkRequest) -> RpcResult { + async fn submit_transaction_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: SubmitTransactionRequest, + ) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_block_call(&self, _connection: Option<&DynRpcConnection>, _request: GetBlockRequest) -> RpcResult { + Err(RpcError::NotImplemented) + } + + async fn get_subnetwork_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSubnetworkRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_virtual_chain_from_block_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetVirtualChainFromBlockRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_blocks_call(&self, _request: GetBlocksRequest) -> RpcResult { + async fn get_blocks_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlocksRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_count_call(&self, _request: GetBlockCountRequest) -> RpcResult { + async fn get_block_count_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockCountRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_block_dag_info_call(&self, _request: GetBlockDagInfoRequest) -> RpcResult { + async fn get_block_dag_info_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBlockDagInfoRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn resolve_finality_conflict_call( &self, + _connection: Option<&DynRpcConnection>, _request: ResolveFinalityConflictRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn shutdown_call(&self, _request: ShutdownRequest) -> RpcResult { + async fn shutdown_call(&self, _connection: Option<&DynRpcConnection>, _request: ShutdownRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_headers_call(&self, _request: GetHeadersRequest) -> RpcResult { + async fn get_headers_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetHeadersRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_balance_by_address_call(&self, _request: GetBalanceByAddressRequest) -> RpcResult { + async fn get_balance_by_address_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetBalanceByAddressRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_balances_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetBalancesByAddressesRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_utxos_by_addresses_call(&self, _request: GetUtxosByAddressesRequest) -> RpcResult { + async fn get_utxos_by_addresses_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetUtxosByAddressesRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_sink_blue_score_call(&self, _request: GetSinkBlueScoreRequest) -> RpcResult { + async fn get_sink_blue_score_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetSinkBlueScoreRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn ban_call(&self, _request: BanRequest) -> RpcResult { + async fn ban_call(&self, _connection: Option<&DynRpcConnection>, _request: BanRequest) -> RpcResult { Err(RpcError::NotImplemented) } - async fn unban_call(&self, _request: UnbanRequest) -> RpcResult { + async fn unban_call(&self, _connection: Option<&DynRpcConnection>, _request: UnbanRequest) -> RpcResult { Err(RpcError::NotImplemented) } async fn estimate_network_hashes_per_second_call( &self, + _connection: Option<&DynRpcConnection>, _request: EstimateNetworkHashesPerSecondRequest, ) -> RpcResult { Err(RpcError::NotImplemented) @@ -236,28 +333,39 @@ impl RpcApi for RpcCoreMock { async fn get_mempool_entries_by_addresses_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetMempoolEntriesByAddressesRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_coin_supply_call(&self, _request: GetCoinSupplyRequest) -> RpcResult { + async fn get_coin_supply_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetCoinSupplyRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_daa_score_timestamp_estimate_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetDaaScoreTimestampEstimateRequest, ) -> RpcResult { Err(RpcError::NotImplemented) } - async fn get_fee_estimate_call(&self, _request: GetFeeEstimateRequest) -> RpcResult { + async fn get_fee_estimate_call( + &self, + _connection: Option<&DynRpcConnection>, + _request: GetFeeEstimateRequest, + ) -> RpcResult { Err(RpcError::NotImplemented) } async fn get_fee_estimate_experimental_call( &self, + _connection: Option<&DynRpcConnection>, _request: GetFeeEstimateExperimentalRequest, ) -> RpcResult { Err(RpcError::NotImplemented) diff --git a/wallet/core/src/tests/storage.rs b/wallet/core/src/tests/storage.rs index 7257311..5af0bcb 100644 --- a/wallet/core/src/tests/storage.rs +++ b/wallet/core/src/tests/storage.rs @@ -20,11 +20,11 @@ where } pub fn validate(&self) -> Result { - let bytes = self.try_to_vec()?; + let bytes = borsh::to_vec(self)?; let transform = Self::try_from_slice(bytes.as_slice())?; assert_eq!(transform.before, 0xdeadbeef); assert_eq!(transform.after, 0xbaadf00d); - let transform_bytes = transform.try_to_vec()?; + let transform_bytes = borsh::to_vec(&transform)?; assert_eq!(bytes, transform_bytes); Ok(transform.storable) } diff --git a/wallet/core/src/tx/generator/generator.rs b/wallet/core/src/tx/generator/generator.rs index 4ae80cb..7be24f6 100644 --- a/wallet/core/src/tx/generator/generator.rs +++ b/wallet/core/src/tx/generator/generator.rs @@ -66,6 +66,7 @@ use crate::tx::{ use crate::utxo::{NetworkParams, UtxoContext, UtxoEntryReference}; use spectre_consensus_client::UtxoEntry; use spectre_consensus_core::constants::UNACCEPTED_DAA_SCORE; +use spectre_consensus_core::mass::Kip9Version; use spectre_consensus_core::subnets::SUBNETWORK_ID_NATIVE; use spectre_consensus_core::tx::{Transaction, TransactionInput, TransactionOutpoint, TransactionOutput}; use spectre_txscript::pay_to_address_script; @@ -88,6 +89,11 @@ const TRANSACTION_MASS_BOUNDARY_FOR_STAGE_INPUT_ACCUMULATION: u64 = MAXIMUM_STAN struct Context { /// iterator containing UTXO entries available for transaction generation utxo_source_iterator: Box + Send + Sync + 'static>, + /// List of priority UTXO entries, that are consumed before polling the iterator + priority_utxo_entries: Option>, + /// HashSet containing priority UTXO entries, used for filtering + /// for potential duplicates from the iterator + priority_utxo_entry_filter: Option>, /// total number of UTXOs consumed by the single generator instance aggregated_utxos: usize, /// total fees of all transactions issued by @@ -210,7 +216,7 @@ struct Data { impl Data { fn new(calc: &MassCalculator) -> Self { - let aggregate_mass = calc.blank_transaction_mass(); + let aggregate_mass = calc.blank_transaction_compute_mass(); Data { inputs: vec![], @@ -261,7 +267,7 @@ struct Inner { // Current network id network_id: NetworkId, // Current network params - network_params: NetworkParams, + network_params: &'static NetworkParams, // Source Utxo Context (Used for source UtxoEntry aggregation) source_utxo_context: Option, @@ -340,6 +346,7 @@ impl Generator { multiplexer, utxo_iterator, source_utxo_context: utxo_context, + priority_utxo_entries, sig_op_count, minimum_signatures, change_address, @@ -351,7 +358,7 @@ impl Generator { let network_type = NetworkType::from(network_id); let network_params = NetworkParams::from(network_id); - let mass_calculator = MassCalculator::new(&network_id.into(), &network_params); + let mass_calculator = MassCalculator::new(&network_id.into(), network_params); let (final_transaction_outputs, final_transaction_amount) = match final_transaction_destination { PaymentDestination::Change => { @@ -396,11 +403,11 @@ impl Generator { } let standard_change_output_mass = - mass_calculator.calc_mass_for_output(&TransactionOutput::new(0, pay_to_address_script(&change_address))); - let signature_mass_per_input = mass_calculator.calc_signature_mass(minimum_signatures); - let final_transaction_outputs_compute_mass = mass_calculator.calc_mass_for_outputs(&final_transaction_outputs); + mass_calculator.calc_compute_mass_for_output(&TransactionOutput::new(0, pay_to_address_script(&change_address))); + let signature_mass_per_input = mass_calculator.calc_compute_mass_for_signature(minimum_signatures); + let final_transaction_outputs_compute_mass = mass_calculator.calc_compute_mass_for_outputs(&final_transaction_outputs); let final_transaction_payload = final_transaction_payload.unwrap_or_default(); - let final_transaction_payload_mass = mass_calculator.calc_mass_for_payload(final_transaction_payload.len()); + let final_transaction_payload_mass = mass_calculator.calc_compute_mass_for_payload(final_transaction_payload.len()); let final_transaction_outputs_harmonic = mass_calculator.calc_storage_mass_output_harmonic(&final_transaction_outputs).ok_or(Error::MassCalculationError)?; @@ -415,8 +422,14 @@ impl Generator { return Err(Error::GeneratorTransactionOutputsAreTooHeavy { mass: mass_sanity_check, kind: "compute mass" }); } + let priority_utxo_entry_filter = priority_utxo_entries.as_ref().map(|entries| entries.iter().cloned().collect()); + // remap to VecDeque as this list gets drained + let priority_utxo_entries = priority_utxo_entries.map(|entries| entries.into_iter().collect::>()); + let context = Mutex::new(Context { utxo_source_iterator: utxo_iterator, + priority_utxo_entries, + priority_utxo_entry_filter, number_of_transactions: 0, aggregated_utxos: 0, aggregate_fees: 0, @@ -465,7 +478,7 @@ impl Generator { /// Returns current [`NetworkParams`] pub fn network_params(&self) -> &NetworkParams { - &self.inner.network_params + self.inner.network_params } /// The underlying [`UtxoContext`] (if available). @@ -528,15 +541,29 @@ impl Generator { } /// Get next UTXO entry. This function obtains UTXO in the following order: - /// 1. From the UTXO stash (used to store UTxOs that were not used in the previous transaction) + /// 1. From the UTXO stash (used to store UTxOs that were consumed during previous transaction generation but were rejected due to various conditions, such as mass overflow) /// 2. From the current stage - /// 3. From the UTXO source iterator + /// 3. From priority UTXO entries + /// 4. From the UTXO source iterator (while filtering against priority UTXO entries) fn get_utxo_entry(&self, context: &mut Context, stage: &mut Stage) -> Option { context .utxo_stash .pop_front() .or_else(|| stage.utxo_iterator.as_mut().and_then(|utxo_stage_iterator| utxo_stage_iterator.next())) - .or_else(|| context.utxo_source_iterator.next()) + .or_else(|| context.priority_utxo_entries.as_mut().and_then(|entries| entries.pop_front())) + .or_else(|| loop { + let utxo_entry = context.utxo_source_iterator.next()?; + + if let Some(filter) = context.priority_utxo_entry_filter.as_ref() { + if filter.contains(&utxo_entry) { + // skip the entry from the iterator intake + // if it has been supplied as a priority entry + continue; + } + } + + break Some(utxo_entry); + }) } /// Calculate relay transaction mass for the current transaction `data` @@ -634,14 +661,14 @@ impl Generator { let input = TransactionInput::new(utxo.outpoint.clone().into(), vec![], 0, self.inner.sig_op_count); let input_amount = utxo.amount(); - let input_compute_mass = calc.calc_mass_for_input(&input) + self.inner.signature_mass_per_input; + let input_compute_mass = calc.calc_compute_mass_for_input(&input) + self.inner.signature_mass_per_input; // NOTE: relay transactions have no storage mass // mass threshold reached, yield transaction if data.aggregate_mass + input_compute_mass + self.inner.standard_change_output_compute_mass - + self.inner.network_params.additional_compound_transaction_mass + + self.inner.network_params.additional_compound_transaction_mass() > MAXIMUM_STANDARD_TRANSACTION_MASS { // note, we've used input for mass boundary calc and now abandon it @@ -649,7 +676,7 @@ impl Generator { context.utxo_stash.push_back(utxo_entry_reference); data.aggregate_mass += - self.inner.standard_change_output_compute_mass + self.inner.network_params.additional_compound_transaction_mass; + self.inner.standard_change_output_compute_mass + self.inner.network_params.additional_compound_transaction_mass(); data.transaction_fees = self.calc_relay_transaction_compute_fees(data); stage.aggregate_fees += data.transaction_fees; context.aggregate_fees += data.transaction_fees; @@ -837,8 +864,11 @@ impl Generator { calc.calc_storage_mass_output_harmonic_single(change_value) + self.inner.final_transaction_outputs_harmonic; let storage_mass_with_change = self.calc_storage_mass(data, output_harmonic_with_change); + // TODO - review and potentially simplify: + // this profiles the storage mass with change and without change + // and decides which one to use based on the fees if storage_mass_with_change == 0 - || (self.inner.network_params.mass_combination_strategy == MassCombinationStrategy::Max + || (self.inner.network_params.kip9_version() == Kip9Version::Beta // max(compute vs storage) && storage_mass_with_change < compute_mass_with_change) { 0 @@ -879,7 +909,7 @@ impl Generator { let compute_mass = data.aggregate_mass + self.inner.standard_change_output_compute_mass - + self.inner.network_params.additional_compound_transaction_mass; + + self.inner.network_params.additional_compound_transaction_mass(); let compute_fees = calc.calc_minimum_transaction_fee_from_mass(compute_mass); // TODO - consider removing this as calculated storage mass should produce `0` value diff --git a/wallet/core/src/tx/generator/pending.rs b/wallet/core/src/tx/generator/pending.rs index 23afeda..997e6ef 100644 --- a/wallet/core/src/tx/generator/pending.rs +++ b/wallet/core/src/tx/generator/pending.rs @@ -8,7 +8,8 @@ use crate::result::Result; use crate::rpc::DynRpcApi; use crate::tx::{DataKind, Generator}; use crate::utxo::{UtxoContext, UtxoEntryId, UtxoEntryReference}; -use spectre_consensus_core::sign::sign_with_multiple_v2; +use spectre_consensus_core::hashing::sighash_type::SigHashType; +use spectre_consensus_core::sign::{sign_input, sign_with_multiple_v2, Signed}; use spectre_consensus_core::tx::{SignableTransaction, Transaction, TransactionId}; use spectre_rpc_core::{RpcTransaction, RpcTransactionId}; @@ -223,9 +224,50 @@ impl PendingTransaction { Ok(()) } - pub fn try_sign_with_keys(&self, privkeys: &[[u8; 32]]) -> Result<()> { + pub fn create_input_signature(&self, input_index: usize, private_key: &[u8; 32], hash_type: SigHashType) -> Result> { let mutable_tx = self.inner.signable_tx.lock()?.clone(); - let signed_tx = sign_with_multiple_v2(mutable_tx, privkeys).fully_signed()?; + let verifiable_tx = mutable_tx.as_verifiable(); + + Ok(sign_input(&verifiable_tx, input_index, private_key, hash_type)) + } + + pub fn fill_input(&self, input_index: usize, signature_script: Vec) -> Result<()> { + let mut mutable_tx = self.inner.signable_tx.lock()?.clone(); + mutable_tx.tx.inputs[input_index].signature_script = signature_script; + *self.inner.signable_tx.lock().unwrap() = mutable_tx; + + Ok(()) + } + + pub fn sign_input(&self, input_index: usize, private_key: &[u8; 32], hash_type: SigHashType) -> Result<()> { + let mut mutable_tx = self.inner.signable_tx.lock()?.clone(); + + let signature_script = { + let verifiable_tx = &mutable_tx.as_verifiable(); + sign_input(verifiable_tx, input_index, private_key, hash_type) + }; + + mutable_tx.tx.inputs[input_index].signature_script = signature_script; + *self.inner.signable_tx.lock().unwrap() = mutable_tx; + + Ok(()) + } + + pub fn try_sign_with_keys(&self, privkeys: &[[u8; 32]], check_fully_signed: Option) -> Result<()> { + let mutable_tx = self.inner.signable_tx.lock()?.clone(); + let signed = sign_with_multiple_v2(mutable_tx, privkeys); + + let signed_tx = match signed { + Signed::Fully(tx) => tx, + Signed::Partially(_) => { + if check_fully_signed.unwrap_or(true) { + signed.fully_signed()? + } else { + signed.unwrap() + } + } + }; + *self.inner.signable_tx.lock().unwrap() = signed_tx; Ok(()) } diff --git a/wallet/core/src/tx/generator/settings.rs b/wallet/core/src/tx/generator/settings.rs index fb8cefb..d83bb7d 100644 --- a/wallet/core/src/tx/generator/settings.rs +++ b/wallet/core/src/tx/generator/settings.rs @@ -20,6 +20,8 @@ pub struct GeneratorSettings { pub utxo_iterator: Box + Send + Sync + 'static>, // Utxo Context pub source_utxo_context: Option, + // Priority utxo entries that are consumed before others + pub priority_utxo_entries: Option>, // typically a number of keys required to sign the transaction pub sig_op_count: u8, // number of minimum signatures required to sign the transaction @@ -77,6 +79,7 @@ impl GeneratorSettings { change_address, utxo_iterator: Box::new(utxo_iterator), source_utxo_context: Some(account.utxo_context().clone()), + priority_utxo_entries: None, final_transaction_priority_fee: final_priority_fee, final_transaction_destination, @@ -89,6 +92,7 @@ impl GeneratorSettings { pub fn try_new_with_context( utxo_context: UtxoContext, + priority_utxo_entries: Option>, change_address: Address, sig_op_count: u8, minimum_signatures: u16, @@ -108,6 +112,7 @@ impl GeneratorSettings { change_address, utxo_iterator: Box::new(utxo_iterator), source_utxo_context: Some(utxo_context), + priority_utxo_entries, final_transaction_priority_fee: final_priority_fee, final_transaction_destination, @@ -121,6 +126,7 @@ impl GeneratorSettings { pub fn try_new_with_iterator( network_id: NetworkId, utxo_iterator: Box + Send + Sync + 'static>, + priority_utxo_entries: Option>, change_address: Address, sig_op_count: u8, minimum_signatures: u16, @@ -137,6 +143,7 @@ impl GeneratorSettings { change_address, utxo_iterator: Box::new(utxo_iterator), source_utxo_context: None, + priority_utxo_entries, final_transaction_priority_fee: final_priority_fee, final_transaction_destination, diff --git a/wallet/core/src/tx/generator/test.rs b/wallet/core/src/tx/generator/test.rs index b24df64..3869611 100644 --- a/wallet/core/src/tx/generator/test.rs +++ b/wallet/core/src/tx/generator/test.rs @@ -169,11 +169,11 @@ fn validate(pt: &PendingTransaction) { ); let calc = MassCalculator::new(&pt.network_type().into(), network_params); - let additional_mass = if pt.is_final() { 0 } else { network_params.additional_compound_transaction_mass }; - let compute_mass = calc.calc_mass_for_signed_transaction(&tx, 1); + let additional_mass = if pt.is_final() { 0 } else { network_params.additional_compound_transaction_mass() }; + let compute_mass = calc.calc_compute_mass_for_signed_transaction(&tx, 1); let utxo_entries = pt.utxo_entries().values().cloned().collect::>(); - let storage_mass = calc.calc_storage_mass_for_transaction(false, &utxo_entries, &tx.outputs).unwrap_or_default(); + let storage_mass = calc.calc_storage_mass_for_transaction_parts(&utxo_entries, &tx.outputs).unwrap_or_default(); let calculated_mass = calc.combine_mass(compute_mass, storage_mass) + additional_mass; @@ -199,12 +199,12 @@ where let pt_fees = pt.fees(); let calc = MassCalculator::new(&pt.network_type().into(), network_params); - let additional_mass = if pt.is_final() { 0 } else { network_params.additional_compound_transaction_mass }; + let additional_mass = if pt.is_final() { 0 } else { network_params.additional_compound_transaction_mass() }; - let compute_mass = calc.calc_mass_for_signed_transaction(&tx, 1); + let compute_mass = calc.calc_compute_mass_for_signed_transaction(&tx, 1); let utxo_entries = pt.utxo_entries().values().cloned().collect::>(); - let storage_mass = calc.calc_storage_mass_for_transaction(false, &utxo_entries, &tx.outputs).unwrap_or_default(); + let storage_mass = calc.calc_storage_mass_for_transaction_parts(&utxo_entries, &tx.outputs).unwrap_or_default(); if DISPLAY_LOGS && storage_mass != 0 { println!( "calculated storage mass: {} calculated_compute_mass: {} total: {}", @@ -392,6 +392,7 @@ where let sig_op_count = 1; let minimum_signatures = 1; let utxo_iterator: Box + Send + Sync + 'static> = Box::new(utxo_entries.into_iter()); + let priority_utxo_entries = None; let source_utxo_context = None; let destination_utxo_context = None; let final_priority_fee = fees; @@ -406,6 +407,7 @@ where change_address, utxo_iterator, source_utxo_context, + priority_utxo_entries, destination_utxo_context, final_transaction_priority_fee: final_priority_fee, final_transaction_destination, diff --git a/wallet/core/src/tx/mass.rs b/wallet/core/src/tx/mass.rs index 631057f..ef93160 100644 --- a/wallet/core/src/tx/mass.rs +++ b/wallet/core/src/tx/mass.rs @@ -2,20 +2,15 @@ //! Transaction mass calculator. //! +use crate::result::Result; use crate::utxo::NetworkParams; +use spectre_consensus_client as kcc; use spectre_consensus_client::UtxoEntryReference; +use spectre_consensus_core::mass::Kip9Version; use spectre_consensus_core::tx::{Transaction, TransactionInput, TransactionOutput, SCRIPT_VECTOR_SIZE}; use spectre_consensus_core::{config::params::Params, constants::*, subnets::SUBNETWORK_ID_SIZE}; use spectre_hashes::HASH_SIZE; -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum MassCombinationStrategy { - /// `MassCombinator::Add` adds the storage and compute mass. - Add, - /// `MassCombinator::Max` returns the maximum of the storage and compute mass. - Max, -} - // pub const ECDSA_SIGNATURE_SIZE: u64 = 64; // pub const SCHNORR_SIGNATURE_SIZE: u64 = 64; pub const SIGNATURE_SIZE: u64 = 1 + 64 + 1; //1 byte for OP_DATA_65 + 64 (length of signature) + 1 byte for sig hash type @@ -222,7 +217,7 @@ pub struct MassCalculator { mass_per_script_pub_key_byte: u64, mass_per_sig_op: u64, storage_mass_parameter: u64, - mass_combination_strategy: MassCombinationStrategy, + kip9_version: Kip9Version, } impl MassCalculator { @@ -232,7 +227,7 @@ impl MassCalculator { mass_per_script_pub_key_byte: consensus_params.mass_per_script_pub_key_byte, mass_per_sig_op: consensus_params.mass_per_sig_op, storage_mass_parameter: consensus_params.storage_mass_parameter, - mass_combination_strategy: network_params.mass_combination_strategy, + kip9_version: network_params.kip9_version(), } } @@ -243,44 +238,45 @@ impl MassCalculator { } } - pub fn calc_mass_for_transaction(&self, tx: &Transaction) -> u64 { - self.blank_transaction_mass() - + self.calc_mass_for_payload(tx.payload.len()) - + self.calc_mass_for_outputs(&tx.outputs) - + self.calc_mass_for_inputs(&tx.inputs) + pub fn calc_transaction_compute_mass(&self, tx: &Transaction) -> u64 { + let payload_len = tx.payload.len(); + self.blank_transaction_compute_mass() + + self.calc_compute_mass_for_payload(payload_len) + + self.calc_compute_mass_for_outputs(&tx.outputs) + + self.calc_compute_mass_for_inputs(&tx.inputs) } - pub fn blank_transaction_mass(&self) -> u64 { + pub(crate) fn blank_transaction_compute_mass(&self) -> u64 { blank_transaction_serialized_byte_size() * self.mass_per_tx_byte } - pub fn calc_mass_for_payload(&self, payload_byte_size: usize) -> u64 { + pub(crate) fn calc_compute_mass_for_payload(&self, payload_byte_size: usize) -> u64 { payload_byte_size as u64 * self.mass_per_tx_byte } - pub fn calc_mass_for_outputs(&self, outputs: &[TransactionOutput]) -> u64 { - outputs.iter().map(|output| self.calc_mass_for_output(output)).sum() + pub(crate) fn calc_compute_mass_for_outputs(&self, outputs: &[TransactionOutput]) -> u64 { + outputs.iter().map(|output| self.calc_compute_mass_for_output(output)).sum() } - pub fn calc_mass_for_inputs(&self, inputs: &[TransactionInput]) -> u64 { - inputs.iter().map(|input| self.calc_mass_for_input(input)).sum::() + pub(crate) fn calc_compute_mass_for_inputs(&self, inputs: &[TransactionInput]) -> u64 { + inputs.iter().map(|input| self.calc_compute_mass_for_input(input)).sum::() } - pub fn calc_mass_for_output(&self, output: &TransactionOutput) -> u64 { + pub(crate) fn calc_compute_mass_for_output(&self, output: &TransactionOutput) -> u64 { self.mass_per_script_pub_key_byte * (2 + output.script_public_key.script().len() as u64) + transaction_output_serialized_byte_size(output) * self.mass_per_tx_byte } - pub fn calc_mass_for_input(&self, input: &TransactionInput) -> u64 { + pub(crate) fn calc_compute_mass_for_input(&self, input: &TransactionInput) -> u64 { input.sig_op_count as u64 * self.mass_per_sig_op + transaction_input_serialized_byte_size(input) * self.mass_per_tx_byte } - pub fn calc_signature_mass(&self, minimum_signatures: u16) -> u64 { + pub(crate) fn calc_compute_mass_for_signature(&self, minimum_signatures: u16) -> u64 { let minimum_signatures = std::cmp::max(1, minimum_signatures); SIGNATURE_SIZE * self.mass_per_tx_byte * minimum_signatures as u64 } - pub fn calc_signature_mass_for_inputs(&self, number_of_inputs: usize, minimum_signatures: u16) -> u64 { + pub fn calc_signature_compute_mass_for_inputs(&self, number_of_inputs: usize, minimum_signatures: u16) -> u64 { let minimum_signatures = std::cmp::max(1, minimum_signatures); SIGNATURE_SIZE * self.mass_per_tx_byte * minimum_signatures as u64 * number_of_inputs as u64 } @@ -289,48 +285,66 @@ impl MassCalculator { calc_minimum_required_transaction_relay_fee(mass) } - pub fn calc_mass_for_signed_transaction(&self, tx: &Transaction, minimum_signatures: u16) -> u64 { - self.calc_mass_for_transaction(tx) + self.calc_signature_mass_for_inputs(tx.inputs.len(), minimum_signatures) + pub fn calc_compute_mass_for_signed_transaction(&self, tx: &Transaction, minimum_signatures: u16) -> u64 { + self.calc_transaction_compute_mass(tx) + self.calc_signature_compute_mass_for_inputs(tx.inputs.len(), minimum_signatures) } - pub fn calc_minium_transaction_relay_fee(&self, tx: &Transaction, minimum_signatures: u16) -> u64 { - let mass = self.calc_mass_for_transaction(tx) + self.calc_signature_mass_for_inputs(tx.inputs.len(), minimum_signatures); - calc_minimum_required_transaction_relay_fee(mass) + pub fn calc_transaction_storage_fee(&self, inputs: &[UtxoEntryReference], outputs: &[TransactionOutput]) -> u64 { + self.calc_fee_for_storage_mass(self.calc_storage_mass_for_transaction_parts(inputs, outputs).unwrap_or(u64::MAX)) } - pub fn calc_tx_storage_fee(&self, is_coinbase: bool, inputs: &[UtxoEntryReference], outputs: &[TransactionOutput]) -> u64 { - self.calc_fee_for_storage_mass(self.calc_storage_mass_for_transaction(is_coinbase, inputs, outputs).unwrap_or(u64::MAX)) + // provisional + pub fn calc_fee_for_storage_mass(&self, mass: u64) -> u64 { + mass } - pub fn calc_fee_for_storage_mass(&self, mass: u64) -> u64 { + // provisional + pub fn calc_fee_for_mass(&self, mass: u64) -> u64 { mass } pub fn combine_mass(&self, compute_mass: u64, storage_mass: u64) -> u64 { - match self.mass_combination_strategy { - MassCombinationStrategy::Add => compute_mass + storage_mass, - MassCombinationStrategy::Max => std::cmp::max(compute_mass, storage_mass), + match self.kip9_version { + Kip9Version::Alpha => compute_mass + storage_mass, + Kip9Version::Beta => std::cmp::max(compute_mass, storage_mass), } } - pub fn calc_storage_mass_for_transaction( + /// Calculates the overall mass of this transaction, combining both compute and storage masses. + pub fn calc_tx_overall_mass(&self, tx: &kcc::Transaction) -> Result> { + let cctx = Transaction::from(tx); + let mass = match self.kip9_version { + Kip9Version::Alpha => self + .calc_storage_mass_for_transaction(tx)? + .and_then(|mass| mass.checked_add(self.calc_transaction_compute_mass(&cctx))), + Kip9Version::Beta => { + self.calc_storage_mass_for_transaction(tx)?.map(|mass| mass.max(self.calc_transaction_compute_mass(&cctx))) + } + }; + + Ok(mass) + } + + pub fn calc_storage_mass_for_transaction(&self, tx: &kcc::Transaction) -> Result> { + let utxos = tx.utxo_entry_references()?; + let outputs = tx.outputs(); + Ok(self.calc_storage_mass_for_transaction_parts(&utxos, &outputs)) + } + + pub fn calc_storage_mass_for_transaction_parts( &self, - is_coinbase: bool, inputs: &[UtxoEntryReference], outputs: &[TransactionOutput], ) -> Option { - if is_coinbase { - return Some(0); - } /* The code below computes the following formula: - max( 0 , C·( |O|/H(O) - |I|/A(I) ) ) + max( 0 , C·( |O|/H(O) - |I|/A(I) ) ) where C is the mass storage parameter, O is the set of output values, I is the set of input values, H(S) := |S|/sum_{s in S} 1 / s is the harmonic mean over the set S and A(S) := sum_{s in S} / |S| is the arithmetic mean. - See the (to date unpublished) KIP-0009 for more details + See KIP-0009 for more details */ // Since we are doing integer division, we perform the multiplication with C over the inner @@ -338,15 +352,36 @@ impl MassCalculator { // // If sum of fractions overflowed (nearly impossible, requires 10^7 outputs for C = 10^12), // we return `None` indicating mass is incomputable + // + // Note: in theory this can be tighten by subtracting input mass in the process (possibly avoiding the overflow), + // however the overflow case is so unpractical with current mass limits so we avoid the hassle let harmonic_outs = outputs .iter() .map(|out| self.storage_mass_parameter / out.value) .try_fold(0u64, |total, current| total.checked_add(current))?; // C·|O|/H(O) + let outs_len = outputs.len() as u64; + let ins_len = inputs.len() as u64; + + /* + KIP-0009 relaxed formula for the cases |O| = 1 OR |O| <= |I| <= 2: + max( 0 , C·( |O|/H(O) - |I|/H(I) ) ) + + Note: in the case |I| = 1 both formulas are equal, yet the following code (harmonic_ins) is a bit more efficient. + Hence, we transform the condition to |O| = 1 OR |I| = 1 OR |O| = |I| = 2 which is equivalent (and faster). + */ + + if self.kip9_version == Kip9Version::Beta && (outs_len == 1 || ins_len == 1 || (outs_len == 2 && ins_len == 2)) { + let harmonic_ins = inputs + .iter() + .map(|entry| self.storage_mass_parameter / entry.amount()) + .fold(0u64, |total, current| total.saturating_add(current)); // C·|I|/H(I) + return Some(harmonic_outs.saturating_sub(harmonic_ins)); // max( 0 , C·( |O|/H(O) - |I|/H(I) ) ); + } + // Total supply is bounded, so a sum of existing UTXO entries cannot overflow (nor can it be zero) let sum_ins = inputs.iter().map(|entry| entry.amount()).sum::(); // |I|·A(I) - let ins_len = inputs.len() as u64; let mean_ins = sum_ins / ins_len; // Inner fraction must be with C and over the mean value, in order to maximize precision. diff --git a/wallet/core/src/tx/payment.rs b/wallet/core/src/tx/payment.rs index 48c19ce..c092e1b 100644 --- a/wallet/core/src/tx/payment.rs +++ b/wallet/core/src/tx/payment.rs @@ -62,8 +62,11 @@ pub struct PaymentOutput { impl TryCastFromJs for PaymentOutput { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(array) = value.as_ref().dyn_ref::() { let length = array.length(); if length != 2 { @@ -74,7 +77,7 @@ impl TryCastFromJs for PaymentOutput { Ok(Self { address, amount }) } } else if let Some(object) = Object::try_from(value.as_ref()) { - let address = object.get_cast::
("address")?.into_owned(); + let address = object.cast_into::
("address")?; let amount = object.get_u64("amount")?; Ok(Self { address, amount }) } else { @@ -145,8 +148,11 @@ impl PaymentOutputs { impl TryCastFromJs for PaymentOutputs { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let outputs = if let Some(output_array) = value.as_ref().dyn_ref::() { let vec = output_array.to_vec(); vec.into_iter().map(PaymentOutput::try_owned_from).collect::, _>>()? diff --git a/wallet/core/src/utxo/balance.rs b/wallet/core/src/utxo/balance.rs index 416f0ff..c262b93 100644 --- a/wallet/core/src/utxo/balance.rs +++ b/wallet/core/src/utxo/balance.rs @@ -10,6 +10,7 @@ pub enum DeltaStyle { } #[derive(Default, Debug, Clone, Serialize, Deserialize, BorshSerialize, BorshDeserialize)] +#[borsh(use_discriminant = true)] pub enum Delta { #[default] NoChange = 0, diff --git a/wallet/core/src/utxo/context.rs b/wallet/core/src/utxo/context.rs index a75587e..931df3a 100644 --- a/wallet/core/src/utxo/context.rs +++ b/wallet/core/src/utxo/context.rs @@ -299,7 +299,7 @@ impl UtxoContext { context.mature.sorted_insert_binary_asc_by_key(utxo_entry.clone(), |entry| entry.amount_as_ref()); } else { let params = NetworkParams::from(self.processor().network_id()?); - match utxo_entry.maturity(¶ms, current_daa_score) { + match utxo_entry.maturity(params, current_daa_score) { Maturity::Stasis => { context.stasis.insert(utxo_entry.id().clone(), utxo_entry.clone()); self.processor() @@ -428,7 +428,7 @@ impl UtxoContext { for utxo_entry in utxo_entries.into_iter() { if let std::collections::hash_map::Entry::Vacant(e) = context.map.entry(utxo_entry.id()) { e.insert(utxo_entry.clone()); - match utxo_entry.maturity(¶ms, current_daa_score) { + match utxo_entry.maturity(params, current_daa_score) { Maturity::Stasis => { context.stasis.insert(utxo_entry.id().clone(), utxo_entry.clone()); self.processor() @@ -513,7 +513,6 @@ impl UtxoContext { } let mature = (mature + consumed).saturating_sub(outgoing); - Balance::new(mature, pending, outgoing_without_batch_tx, context.mature.len(), context.pending.len(), context.stasis.len()) } @@ -532,7 +531,7 @@ impl UtxoContext { let force_maturity_if_outgoing = outgoing_transaction.is_some(); let is_coinbase_stasis = - utxos.first().map(|utxo| matches!(utxo.maturity(¶ms, current_daa_score), Maturity::Stasis)).unwrap_or_default(); + utxos.first().map(|utxo| matches!(utxo.maturity(params, current_daa_score), Maturity::Stasis)).unwrap_or_default(); let is_batch = outgoing_transaction.as_ref().map_or_else(|| false, |tx| tx.is_batch()); if !is_batch { for utxo in utxos.iter() { @@ -572,22 +571,20 @@ impl UtxoContext { Ok(()) } - pub(crate) async fn handle_utxo_removed(&self, mut utxos: Vec, current_daa_score: u64) -> Result<()> { + pub(crate) async fn handle_utxo_removed(&self, utxos: Vec, current_daa_score: u64) -> Result<()> { // remove UTXOs from account set let outgoing_transactions = self.processor().outgoing(); #[allow(clippy::mutable_key_type)] let mut accepted_outgoing_transactions = HashSet::::new(); - utxos.retain(|utxo| { + for utxo in &utxos { for outgoing_transaction in outgoing_transactions.iter() { if outgoing_transaction.utxo_entries().contains_key(&utxo.id()) { accepted_outgoing_transactions.insert((*outgoing_transaction).clone()); - return false; } } - true - }); + } for accepted_outgoing_transaction in accepted_outgoing_transactions.into_iter() { if accepted_outgoing_transaction.is_batch() { diff --git a/wallet/core/src/utxo/processor.rs b/wallet/core/src/utxo/processor.rs index 13db7f5..1ddfdce 100644 --- a/wallet/core/src/utxo/processor.rs +++ b/wallet/core/src/utxo/processor.rs @@ -14,13 +14,13 @@ use spectre_notify::{ use spectre_rpc_core::{ api::{ ctl::{RpcCtl, RpcState}, - ops::RPC_API_VERSION, + ops::{RPC_API_REVISION, RPC_API_VERSION}, }, message::UtxosChangedNotification, GetServerInfoResponse, }; use spectre_wrpc_client::SpectreRpcClient; -use workflow_core::channel::{Channel, DuplexChannel}; +use workflow_core::channel::{Channel, DuplexChannel, Sender}; use workflow_core::task::spawn; use crate::events::Events; @@ -62,6 +62,7 @@ pub struct Inner { connect_disconnect_guard: AsyncMutex<()>, metrics: Arc, metrics_kinds: Mutex>, + connection_signaler: Mutex>>>, } impl Inner { @@ -91,6 +92,7 @@ impl Inner { connect_disconnect_guard: Default::default(), metrics: Arc::new(Metrics::default()), metrics_kinds: Mutex::new(vec![]), + connection_signaler: Mutex::new(None), } } } @@ -180,8 +182,10 @@ impl UtxoProcessor { } pub fn network_params(&self) -> Result<&'static NetworkParams> { + // pub fn network_params(&self) -> Result { let network_id = (*self.inner.network_id.lock().unwrap()).ok_or(Error::MissingNetworkId)?; - Ok(network_id.into()) + Ok(NetworkParams::from(network_id)) + // Ok(network_id.into()) } pub fn pending(&self) -> &DashMap { @@ -330,7 +334,7 @@ impl UtxoProcessor { } async fn handle_outgoing(&self, current_daa_score: u64) -> Result<()> { - let longevity = self.network_params()?.user_transaction_maturity_period_daa; + let longevity = self.network_params()?.user_transaction_maturity_period_daa(); self.inner.outgoing.retain(|_, outgoing| { if outgoing.acceptance_daa_score() != 0 && (outgoing.acceptance_daa_score() + longevity) < current_daa_score { @@ -439,14 +443,21 @@ impl UtxoProcessor { pub async fn init_state_from_server(&self) -> Result { let GetServerInfoResponse { + rpc_api_version, + rpc_api_revision, server_version, network_id: server_network_id, has_utxo_index, is_synced, virtual_daa_score, - rpc_api_version, } = self.rpc_api().get_server_info().await?; + if rpc_api_version > RPC_API_VERSION { + let current = format!("{RPC_API_VERSION}.{RPC_API_REVISION}"); + let connected = format!("{rpc_api_version}.{rpc_api_revision}"); + return Err(Error::RpcApiVersion(current, connected)); + } + if !has_utxo_index { self.notify(Events::UtxoIndexNotEnabled { url: self.rpc_url() }).await?; return Err(Error::MissingUtxoIndex); @@ -457,12 +468,6 @@ impl UtxoProcessor { return Err(Error::InvalidNetworkType(network_id.to_string(), server_network_id.to_string())); } - if rpc_api_version[0] > RPC_API_VERSION[0] || rpc_api_version[1] > RPC_API_VERSION[1] { - let current = RPC_API_VERSION.iter().map(|v| v.to_string()).collect::>().join("."); - let connected = rpc_api_version.iter().map(|v| v.to_string()).collect::>().join("."); - return Err(Error::RpcApiVersion(current, connected)); - } - self.inner.current_daa_score.store(virtual_daa_score, Ordering::SeqCst); log_trace!("Connected to spectred: '{server_version}' on '{server_network_id}'; SYNC: {is_synced} DAA: {virtual_daa_score}"); @@ -489,12 +494,30 @@ impl UtxoProcessor { Ok(()) } + /// Allows use to supply a channel Sender that will + /// receive the result of the wRPC connection attempt. + pub fn set_connection_signaler(&self, signal: Sender>) { + *self.inner.connection_signaler.lock().unwrap() = Some(signal); + } + + fn signal_connection(&self, result: std::result::Result<(), String>) -> bool { + let signal = self.inner.connection_signaler.lock().unwrap().take(); + if let Some(signal) = signal.as_ref() { + let _ = signal.try_send(result); + true + } else { + false + } + } + pub async fn handle_connect(&self) -> Result<()> { let _ = self.inner.connect_disconnect_guard.lock().await; match self.handle_connect_impl().await { Err(err) => { - log_error!("UtxoProcessor: error while connecting to node: {err}"); + if !self.signal_connection(Err(err.to_string())) { + log_error!("UtxoProcessor: error while connecting to node: {err}"); + } self.notify(Events::UtxoProcError { message: err.to_string() }).await?; if let Some(client) = self.rpc_client() { // try force disconnect the client if we have failed @@ -503,7 +526,10 @@ impl UtxoProcessor { } Err(err) } - Ok(_) => Ok(()), + Ok(_) => { + self.signal_connection(Ok(())); + Ok(()) + } } } @@ -633,15 +659,11 @@ impl UtxoProcessor { // handle RPC channel connection and disconnection events match msg { RpcState::Connected => { - if !this.is_connected() { - if let Err(err) = this.handle_connect().await { - log_error!("UtxoProcessor error: {err}"); - } else { - this.inner.multiplexer.try_broadcast(Box::new(Events::Connect { - network_id : this.network_id().expect("network id expected during connection"), - url : this.rpc_url() - })).unwrap_or_else(|err| log_error!("{err}")); - } + if !this.is_connected() && this.handle_connect().await.is_ok() { + this.inner.multiplexer.try_broadcast(Box::new(Events::Connect { + network_id : this.network_id().expect("network id expected during connection"), + url : this.rpc_url() + })).unwrap_or_else(|err| log_error!("{err}")); } }, RpcState::Disconnected => { diff --git a/wallet/core/src/utxo/reference.rs b/wallet/core/src/utxo/reference.rs index aaa1fe4..3ad3cef 100644 --- a/wallet/core/src/utxo/reference.rs +++ b/wallet/core/src/utxo/reference.rs @@ -34,14 +34,14 @@ pub trait UtxoEntryReferenceExtension { impl UtxoEntryReferenceExtension for UtxoEntryReference { fn maturity(&self, params: &NetworkParams, current_daa_score: u64) -> Maturity { if self.is_coinbase() { - if self.block_daa_score() + params.coinbase_transaction_stasis_period_daa > current_daa_score { + if self.block_daa_score() + params.coinbase_transaction_stasis_period_daa() > current_daa_score { Maturity::Stasis - } else if self.block_daa_score() + params.coinbase_transaction_maturity_period_daa > current_daa_score { + } else if self.block_daa_score() + params.coinbase_transaction_maturity_period_daa() > current_daa_score { Maturity::Pending } else { Maturity::Confirmed } - } else if self.block_daa_score() + params.user_transaction_maturity_period_daa > current_daa_score { + } else if self.block_daa_score() + params.user_transaction_maturity_period_daa() > current_daa_score { Maturity::Pending } else { Maturity::Confirmed diff --git a/wallet/core/src/utxo/settings.rs b/wallet/core/src/utxo/settings.rs index 3890263..9716418 100644 --- a/wallet/core/src/utxo/settings.rs +++ b/wallet/core/src/utxo/settings.rs @@ -4,58 +4,94 @@ //! use crate::imports::*; +use spectre_consensus_core::mass::Kip9Version; #[derive(Debug)] pub struct NetworkParams { - pub coinbase_transaction_maturity_period_daa: u64, + pub coinbase_transaction_maturity_period_daa: AtomicU64, pub coinbase_transaction_stasis_period_daa: u64, - pub user_transaction_maturity_period_daa: u64, - pub mass_combination_strategy: MassCombinationStrategy, + pub user_transaction_maturity_period_daa: AtomicU64, + pub kip9_version: Kip9Version, pub additional_compound_transaction_mass: u64, } -pub const MAINNET_NETWORK_PARAMS: NetworkParams = NetworkParams { - coinbase_transaction_maturity_period_daa: 100, +impl NetworkParams { + #[inline] + pub fn coinbase_transaction_maturity_period_daa(&self) -> u64 { + self.coinbase_transaction_maturity_period_daa.load(Ordering::Relaxed) + } + + #[inline] + pub fn coinbase_transaction_stasis_period_daa(&self) -> u64 { + self.coinbase_transaction_stasis_period_daa + } + + #[inline] + pub fn user_transaction_maturity_period_daa(&self) -> u64 { + self.user_transaction_maturity_period_daa.load(Ordering::Relaxed) + } + + #[inline] + pub fn kip9_version(&self) -> Kip9Version { + self.kip9_version + } + + #[inline] + pub fn additional_compound_transaction_mass(&self) -> u64 { + self.additional_compound_transaction_mass + } + + pub fn set_coinbase_transaction_maturity_period_daa(&self, value: u64) { + self.coinbase_transaction_maturity_period_daa.store(value, Ordering::Relaxed); + } + + pub fn set_user_transaction_maturity_period_daa(&self, value: u64) { + self.user_transaction_maturity_period_daa.store(value, Ordering::Relaxed); + } +} + +static MAINNET_NETWORK_PARAMS: LazyLock = LazyLock::new(|| NetworkParams { + coinbase_transaction_maturity_period_daa: AtomicU64::new(100), coinbase_transaction_stasis_period_daa: 50, - user_transaction_maturity_period_daa: 10, - mass_combination_strategy: MassCombinationStrategy::Add, - additional_compound_transaction_mass: 0, -}; + user_transaction_maturity_period_daa: AtomicU64::new(10), + kip9_version: Kip9Version::Beta, + additional_compound_transaction_mass: 100, +}); -pub const TESTNET10_NETWORK_PARAMS: NetworkParams = NetworkParams { - coinbase_transaction_maturity_period_daa: 100, +static TESTNET10_NETWORK_PARAMS: LazyLock = LazyLock::new(|| NetworkParams { + coinbase_transaction_maturity_period_daa: AtomicU64::new(100), coinbase_transaction_stasis_period_daa: 50, - user_transaction_maturity_period_daa: 10, - mass_combination_strategy: MassCombinationStrategy::Add, + user_transaction_maturity_period_daa: AtomicU64::new(10), + kip9_version: Kip9Version::Beta, additional_compound_transaction_mass: 100, -}; +}); -pub const TESTNET11_NETWORK_PARAMS: NetworkParams = NetworkParams { - coinbase_transaction_maturity_period_daa: 1_000, +static TESTNET11_NETWORK_PARAMS: LazyLock = LazyLock::new(|| NetworkParams { + coinbase_transaction_maturity_period_daa: AtomicU64::new(1_000), coinbase_transaction_stasis_period_daa: 500, - user_transaction_maturity_period_daa: 100, - mass_combination_strategy: MassCombinationStrategy::Add, + user_transaction_maturity_period_daa: AtomicU64::new(100), + kip9_version: Kip9Version::Alpha, additional_compound_transaction_mass: 100, -}; +}); -pub const DEVNET_NETWORK_PARAMS: NetworkParams = NetworkParams { - coinbase_transaction_maturity_period_daa: 100, +static SIMNET_NETWORK_PARAMS: LazyLock = LazyLock::new(|| NetworkParams { + coinbase_transaction_maturity_period_daa: AtomicU64::new(100), coinbase_transaction_stasis_period_daa: 50, - user_transaction_maturity_period_daa: 10, - mass_combination_strategy: MassCombinationStrategy::Add, + user_transaction_maturity_period_daa: AtomicU64::new(10), + kip9_version: Kip9Version::Alpha, additional_compound_transaction_mass: 0, -}; +}); -pub const SIMNET_NETWORK_PARAMS: NetworkParams = NetworkParams { - coinbase_transaction_maturity_period_daa: 100, +static DEVNET_NETWORK_PARAMS: LazyLock = LazyLock::new(|| NetworkParams { + coinbase_transaction_maturity_period_daa: AtomicU64::new(100), coinbase_transaction_stasis_period_daa: 50, - user_transaction_maturity_period_daa: 10, - mass_combination_strategy: MassCombinationStrategy::Add, + user_transaction_maturity_period_daa: AtomicU64::new(10), + kip9_version: Kip9Version::Beta, additional_compound_transaction_mass: 0, -}; +}); -impl From for &'static NetworkParams { - fn from(value: NetworkId) -> Self { +impl NetworkParams { + pub fn from(value: NetworkId) -> &'static NetworkParams { match value.network_type { NetworkType::Mainnet => &MAINNET_NETWORK_PARAMS, NetworkType::Testnet => match value.suffix { @@ -70,18 +106,27 @@ impl From for &'static NetworkParams { } } -impl From for NetworkParams { - fn from(value: NetworkId) -> Self { - match value.network_type { - NetworkType::Mainnet => MAINNET_NETWORK_PARAMS, - NetworkType::Testnet => match value.suffix { - Some(10) => TESTNET10_NETWORK_PARAMS, - Some(11) => TESTNET11_NETWORK_PARAMS, - Some(x) => panic!("Testnet suffix {} is not supported", x), - None => panic!("Testnet suffix not provided"), - }, - NetworkType::Devnet => DEVNET_NETWORK_PARAMS, - NetworkType::Simnet => SIMNET_NETWORK_PARAMS, - } +/// Set the coinbase transaction maturity period DAA score for a given network. +/// This controls the DAA period after which the user transactions are considered mature +/// and the wallet subsystem emits the transaction maturity event. +pub fn set_coinbase_transaction_maturity_period_daa(network_id: &NetworkId, value: u64) { + let network_params = NetworkParams::from(*network_id); + if value <= network_params.coinbase_transaction_stasis_period_daa() { + panic!( + "Coinbase transaction maturity period must be greater than the stasis period of {} DAA", + network_params.coinbase_transaction_stasis_period_daa() + ); + } + network_params.set_coinbase_transaction_maturity_period_daa(value); +} + +/// Set the user transaction maturity period DAA score for a given network. +/// This controls the DAA period after which the user transactions are considered mature +/// and the wallet subsystem emits the transaction maturity event. +pub fn set_user_transaction_maturity_period_daa(network_id: &NetworkId, value: u64) { + let network_params = NetworkParams::from(*network_id); + if value == 0 { + panic!("User transaction maturity period must be greater than 0"); } + network_params.set_user_transaction_maturity_period_daa(value); } diff --git a/wallet/core/src/wallet/api.rs b/wallet/core/src/wallet/api.rs index 313759b..adeb000 100644 --- a/wallet/core/src/wallet/api.rs +++ b/wallet/core/src/wallet/api.rs @@ -20,6 +20,9 @@ impl WalletApi for super::Wallet { } async fn get_status_call(self: Arc, request: GetStatusRequest) -> Result { + let guard = self.guard(); + let guard = guard.lock().await; + let GetStatusRequest { name } = request; let context = name.and_then(|name| self.inner.retained_contexts.lock().unwrap().get(&name).cloned()); @@ -34,7 +37,7 @@ impl WalletApi for super::Wallet { let (wallet_descriptor, account_descriptors) = if self.is_open() { let wallet_descriptor = self.descriptor(); - let account_descriptors = self.account_descriptors().await.ok(); + let account_descriptors = self.account_descriptors(&guard).await.ok(); (wallet_descriptor, account_descriptors) } else { (None, None) @@ -75,22 +78,37 @@ impl WalletApi for super::Wallet { async fn connect_call(self: Arc, request: ConnectRequest) -> Result { use workflow_rpc::client::{ConnectOptions, ConnectStrategy}; - let ConnectRequest { url, network_id } = request; + let ConnectRequest { url, network_id, retry_on_error, block_async_connect, require_sync } = request; if let Some(wrpc_client) = self.try_wrpc_client().as_ref() { - // self.set_network_id(network_id)?; + let strategy = if retry_on_error { ConnectStrategy::Retry } else { ConnectStrategy::Fallback }; - // let network_type = NetworkType::from(network_id); let url = url .map(|url| wrpc_client.parse_url_with_network_type(url, network_id.into()).map_err(|e| e.to_string())) .transpose()?; - let options = ConnectOptions { block_async_connect: false, strategy: ConnectStrategy::Retry, url, ..Default::default() }; + let options = ConnectOptions { block_async_connect, strategy, url, ..Default::default() }; wrpc_client.disconnect().await?; self.set_network_id(&network_id)?; + let processor = self.utxo_processor().clone(); + let (sender, receiver) = oneshot(); + + // set connection signaler that gets triggered + // by utxo processor when connection occurs + processor.set_connection_signaler(sender); + + // connect rpc wrpc_client.connect(Some(options)).await.map_err(|e| e.to_string())?; - Ok(ConnectResponse {}) + + // wait for connection signal, cascade if error + receiver.recv().await?.map_err(Error::custom)?; + + if require_sync && !self.is_synced() { + Err(Error::NotSynced) + } else { + Ok(ConnectResponse {}) + } } else { Err(Error::NotWrpcClient) } @@ -143,9 +161,12 @@ impl WalletApi for super::Wallet { } async fn wallet_open_call(self: Arc, request: WalletOpenRequest) -> Result { + let guard = self.guard(); + let guard = guard.lock().await; + let WalletOpenRequest { wallet_secret, filename, account_descriptors, legacy_accounts } = request; let args = WalletOpenArgs { account_descriptors, legacy_accounts: legacy_accounts.unwrap_or_default() }; - let account_descriptors = self.open(&wallet_secret, filename, args).await?; + let account_descriptors = self.open(&wallet_secret, filename, args, &guard).await?; Ok(WalletOpenResponse { account_descriptors }) } @@ -159,7 +180,11 @@ impl WalletApi for super::Wallet { if !self.is_open() { return Err(Error::WalletNotOpen); } - self.reload(reactivate).await?; + + let guard = self.guard(); + let guard = guard.lock().await; + + self.reload(reactivate, &guard).await?; Ok(WalletReloadResponse {}) } @@ -222,7 +247,10 @@ impl WalletApi for super::Wallet { async fn accounts_rename_call(self: Arc, request: AccountsRenameRequest) -> Result { let AccountsRenameRequest { account_id, name, wallet_secret } = request; - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + let guard = self.guard(); + let guard = guard.lock().await; + + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; account.rename(&wallet_secret, name.as_deref()).await?; Ok(AccountsRenameResponse {}) @@ -231,8 +259,11 @@ impl WalletApi for super::Wallet { async fn accounts_select_call(self: Arc, request: AccountsSelectRequest) -> Result { let AccountsSelectRequest { account_id } = request; + let guard = self.guard(); + let guard = guard.lock().await; + if let Some(account_id) = account_id { - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; self.select(Some(&account)).await?; } else { self.select(None).await?; @@ -243,34 +274,20 @@ impl WalletApi for super::Wallet { } async fn accounts_enumerate_call(self: Arc, _request: AccountsEnumerateRequest) -> Result { - // let iter = self.inner.store.as_account_store().unwrap().iter(None).await.unwrap(); - // let wallet = self.clone(); - - // let stream = iter.then(move |stored| { - // let wallet = wallet.clone(); - - // async move { - // let (stored_account, stored_metadata) = stored.unwrap(); - // if let Some(account) = wallet.legacy_accounts().get(&stored_account.id) { - // account.descriptor() - // } else if let Some(account) = wallet.active_accounts().get(&stored_account.id) { - // account.descriptor() - // } else { - // try_load_account(&wallet, stored_account, stored_metadata).await?.descriptor() - // } - // } - // }); - - // let account_descriptors = stream.try_collect::>().await?; - - let account_descriptors = self.account_descriptors().await?; + let guard = self.guard(); + let guard = guard.lock().await; + + let account_descriptors = self.account_descriptors(&guard).await?; Ok(AccountsEnumerateResponse { account_descriptors }) } async fn accounts_activate_call(self: Arc, request: AccountsActivateRequest) -> Result { let AccountsActivateRequest { account_ids } = request; - self.activate_accounts(account_ids.as_deref()).await?; + let guard = self.guard(); + let guard = guard.lock().await; + + self.activate_accounts(account_ids.as_deref(), &guard).await?; Ok(AccountsActivateResponse {}) } @@ -278,7 +295,10 @@ impl WalletApi for super::Wallet { async fn accounts_deactivate_call(self: Arc, request: AccountsDeactivateRequest) -> Result { let AccountsDeactivateRequest { account_ids } = request; - self.deactivate_accounts(account_ids.as_deref()).await?; + let guard = self.guard(); + let guard = guard.lock().await; + + self.deactivate_accounts(account_ids.as_deref(), &guard).await?; Ok(AccountsDeactivateResponse {}) } @@ -296,7 +316,10 @@ impl WalletApi for super::Wallet { async fn accounts_create_call(self: Arc, request: AccountsCreateRequest) -> Result { let AccountsCreateRequest { wallet_secret, account_create_args } = request; - let account = self.create_account(&wallet_secret, account_create_args, true).await?; + let guard = self.guard(); + let guard = guard.lock().await; + + let account = self.create_account(&wallet_secret, account_create_args, true, &guard).await?; let account_descriptor = account.descriptor()?; Ok(AccountsCreateResponse { account_descriptor }) @@ -308,8 +331,12 @@ impl WalletApi for super::Wallet { ) -> Result { let AccountsEnsureDefaultRequest { wallet_secret, payment_secret, account_kind, mnemonic_phrase } = request; - let account_descriptor = - self.ensure_default_account_impl(&wallet_secret, payment_secret.as_ref(), account_kind, mnemonic_phrase.as_ref()).await?; + let guard = self.guard(); + let guard = guard.lock().await; + + let account_descriptor = self + .ensure_default_account_impl(&wallet_secret, payment_secret.as_ref(), account_kind, mnemonic_phrase.as_ref(), &guard) + .await?; Ok(AccountsEnsureDefaultResponse { account_descriptor }) } @@ -321,7 +348,11 @@ impl WalletApi for super::Wallet { async fn accounts_get_call(self: Arc, request: AccountsGetRequest) -> Result { let AccountsGetRequest { account_id } = request; - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + + let guard = self.guard(); + let guard = guard.lock().await; + + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; let account_descriptor = account.descriptor().unwrap(); Ok(AccountsGetResponse { account_descriptor }) } @@ -332,7 +363,10 @@ impl WalletApi for super::Wallet { ) -> Result { let AccountsCreateNewAddressRequest { account_id, kind } = request; - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + let guard = self.guard(); + let guard = guard.lock().await; + + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; let address = match kind { NewAddressKind::Receive => account.as_derivation_capable()?.new_receive_address().await?, @@ -345,7 +379,9 @@ impl WalletApi for super::Wallet { async fn accounts_send_call(self: Arc, request: AccountsSendRequest) -> Result { let AccountsSendRequest { account_id, wallet_secret, payment_secret, destination, priority_fee_sompi, payload } = request; - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + let guard = self.guard(); + let guard = guard.lock().await; + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; let abortable = Abortable::new(); let (generator_summary, transaction_ids) = @@ -364,7 +400,11 @@ impl WalletApi for super::Wallet { transfer_amount_sompi, } = request; - let source_account = self.get_account_by_id(&source_account_id).await?.ok_or(Error::AccountNotFound(source_account_id))?; + let guard = self.guard(); + let guard = guard.lock().await; + + let source_account = + self.get_account_by_id(&source_account_id, &guard).await?.ok_or(Error::AccountNotFound(source_account_id))?; let abortable = Abortable::new(); let (generator_summary, transaction_ids) = source_account @@ -376,6 +416,7 @@ impl WalletApi for super::Wallet { payment_secret, &abortable, None, + &guard, ) .await?; @@ -385,7 +426,9 @@ impl WalletApi for super::Wallet { async fn accounts_estimate_call(self: Arc, request: AccountsEstimateRequest) -> Result { let AccountsEstimateRequest { account_id, destination, priority_fee_sompi, payload } = request; - let account = self.get_account_by_id(&account_id).await?.ok_or(Error::AccountNotFound(account_id))?; + let guard = self.guard(); + let guard = guard.lock().await; + let account = self.get_account_by_id(&account_id, &guard).await?.ok_or(Error::AccountNotFound(account_id))?; // Abort currently running async estimate for the same account if present. The estimate // call can be invoked continuously by the client/UI. If the estimate call is diff --git a/wallet/core/src/wallet/mod.rs b/wallet/core/src/wallet/mod.rs index 64144e4..82a36a1 100644 --- a/wallet/core/src/wallet/mod.rs +++ b/wallet/core/src/wallet/mod.rs @@ -7,6 +7,7 @@ pub mod maps; pub use args::*; use crate::account::ScanNotifier; +use crate::api::traits::WalletApi; use crate::compat::gen1::decrypt_mnemonic; use crate::error::Error::Custom; use crate::factory::try_load_account; @@ -25,6 +26,8 @@ use spectre_wallet_keys::xpub::NetworkTaggedXpub; use spectre_wrpc_client::{Resolver, SpectreRpcClient, WrpcEncoding}; use workflow_core::task::spawn; +pub type WalletGuard<'l> = AsyncMutexGuard<'l, ()>; + #[derive(Debug)] pub struct EncryptedMnemonic> { pub cipher: T, // raw @@ -91,6 +94,9 @@ pub struct Inner { wallet_bus: Channel, estimation_abortables: Mutex>, retained_contexts: Mutex>>>, + // Mutex used to protect concurrent access to accounts at the wallet api level + guard: Arc>, + account_guard: Arc>, } /// @@ -105,6 +111,13 @@ pub struct Wallet { inner: Arc, } +impl Default for Wallet { + fn default() -> Self { + let storage = Wallet::local_store().expect("Unable to initialize local storage"); + Wallet::try_new(storage, None, None).unwrap() + } +} + impl Wallet { pub fn local_store() -> Result> { Ok(Arc::new(LocalStore::try_new(false)?)) @@ -127,14 +140,6 @@ impl Wallet { None, )?); - // pub fn try_with_wrpc(store: Arc, network_id: Option) -> Result { - // let rpc_client = Arc::new(SpectreRpcClient::new_with_args( - // WrpcEncoding::Borsh, - // NotificationMode::MultiListeners, - // "wrpc://127.0.0.1:19110", - // None, - // )?); - let rpc_ctl = rpc_client.ctl().clone(); let rpc_api: Arc = rpc_client; let rpc = Rpc::new(rpc_api, rpc_ctl); @@ -161,16 +166,52 @@ impl Wallet { wallet_bus, estimation_abortables: Mutex::new(HashMap::new()), retained_contexts: Mutex::new(HashMap::new()), + guard: Arc::new(AsyncMutex::new(())), + account_guard: Arc::new(AsyncMutex::new(())), }), }; Ok(wallet) } + pub fn to_arc(self) -> Arc { + Arc::new(self) + } + + /// Helper fn for creating the wallet using a builder pattern. + pub fn with_network_id(self, network_id: NetworkId) -> Self { + self.set_network_id(&network_id).expect("Unable to set network id"); + self + } + + pub fn with_resolver(self, resolver: Resolver) -> Self { + self.wrpc_client().set_resolver(resolver).expect("Unable to set resolver"); + self + } + + pub fn with_url(self, url: Option<&str>) -> Self { + self.wrpc_client().set_url(url).expect("Unable to set url"); + self + } + pub fn inner(&self) -> &Arc { &self.inner } + // + // Mutex used to protect concurrent access to accounts + // at the wallet api level. This is a global lock that + // is required by various wallet operations. + // + // Due to the fact that Rust Wallet API is async, it is + // possible for clients to concurrently execute API calls + // that can "trip over each-other", causing incorrect + // account states. + // + pub fn guard(&self) -> Arc> { + self.inner.guard.clone() + } + pub fn is_resident(&self) -> Result { Ok(self.store().location()? == StorageDescriptor::Resident) } @@ -210,10 +251,12 @@ impl Wallet { Ok(()) } - pub async fn reload(self: &Arc, reactivate: bool) -> Result<()> { + pub async fn reload(self: &Arc, reactivate: bool, _guard: &WalletGuard<'_>) -> Result<()> { if self.is_open() { // similar to reset(), but effectively reboots the wallet + // let _guard = self.inner.guard.lock().await; + let accounts = self.active_accounts().collect(); let account_descriptors = Some(accounts.iter().map(|account| account.descriptor()).collect::>>()?); let wallet_descriptor = self.store().descriptor(); @@ -299,6 +342,8 @@ impl Wallet { filename: Option, args: WalletOpenArgs, ) -> Result>> { + // let _guard = self.inner.guard.lock().await; + let filename = filename.or_else(|| self.settings().get(WalletSettings::Wallet)); // let name = Some(make_filename(&name, &None)); @@ -335,20 +380,21 @@ impl Wallet { None }; - let account_descriptors = accounts - .as_ref() - .map(|accounts| accounts.iter().map(|account| account.descriptor()).collect::>>()) - .transpose()?; - - if let Some(accounts) = accounts { - for account in accounts.into_iter() { + if let Some(accounts) = &accounts { + for account in accounts.iter() { if let Ok(legacy_account) = account.clone().as_legacy_account() { - self.legacy_accounts().insert(account); legacy_account.create_private_context(wallet_secret, None, None).await?; + log_info!("create_private_context, open_impl: receive_address: {:?}", account.receive_address()); + self.legacy_accounts().insert(account.clone()); } } } + let account_descriptors = accounts + .as_ref() + .map(|accounts| accounts.iter().map(|account| account.descriptor()).collect::>>()) + .transpose()?; + self.notify(Events::WalletOpen { wallet_descriptor: wallet_name, account_descriptors: account_descriptors.clone() }).await?; let hint = self.store().get_user_hint().await?; @@ -363,6 +409,7 @@ impl Wallet { wallet_secret: &Secret, filename: Option, args: WalletOpenArgs, + _guard: &WalletGuard<'_>, ) -> Result>> { // This is a wrapper of open_impl() that catches errors and notifies the UI match self.open_impl(wallet_secret, filename, args).await { @@ -375,6 +422,8 @@ impl Wallet { } async fn activate_accounts_impl(self: &Arc, account_ids: Option<&[AccountId]>) -> Result> { + // let _guard = self.inner.guard.lock().await; + let stored_accounts = if let Some(ids) = account_ids { self.inner.store.as_account_store().unwrap().load_multiple(ids).await? } else { @@ -405,7 +454,7 @@ impl Wallet { } /// Activates accounts (performs account address space counts, initializes balance tracking, etc.) - pub async fn activate_accounts(self: &Arc, account_ids: Option<&[AccountId]>) -> Result<()> { + pub async fn activate_accounts(self: &Arc, account_ids: Option<&[AccountId]>, _guard: &WalletGuard<'_>) -> Result<()> { // This is a wrapper of activate_accounts_impl() that catches errors and notifies the UI if let Err(err) = self.activate_accounts_impl(account_ids).await { self.notify(Events::WalletError { message: err.to_string() }).await?; @@ -415,7 +464,9 @@ impl Wallet { } } - pub async fn deactivate_accounts(self: &Arc, ids: Option<&[AccountId]>) -> Result<()> { + pub async fn deactivate_accounts(self: &Arc, ids: Option<&[AccountId]>, _guard: &WalletGuard<'_>) -> Result<()> { + let _guard = self.inner.guard.lock().await; + let (ids, futures) = if let Some(ids) = ids { let accounts = ids.iter().map(|id| self.active_accounts().get(id).ok_or(Error::AccountNotFound(*id))).collect::>>()?; @@ -430,7 +481,9 @@ impl Wallet { Ok(()) } - pub async fn account_descriptors(self: Arc) -> Result> { + pub async fn account_descriptors(self: Arc, _guard: &WalletGuard<'_>) -> Result> { + // let _guard = self.inner.guard.lock().await; + let iter = self.inner.store.as_account_store().unwrap().iter(None).await.unwrap(); let wallet = self.clone(); @@ -468,6 +521,10 @@ impl Wallet { self.try_rpc_api().and_then(|api| api.clone().downcast_arc::().ok()) } + pub fn wrpc_client(&self) -> Arc { + self.try_rpc_api().and_then(|api| api.clone().downcast_arc::().ok()).unwrap() + } + pub fn rpc_api(&self) -> Arc { self.utxo_processor().rpc_api() } @@ -493,6 +550,14 @@ impl Wallet { Ok(()) } + pub fn as_api(self: &Arc) -> Arc { + self.clone() + } + + pub fn to_api(self) -> Arc { + Arc::new(self) + } + pub fn multiplexer(&self) -> &Multiplexer> { &self.inner.multiplexer } @@ -610,6 +675,7 @@ impl Wallet { wallet_secret: &Secret, account_create_args: AccountCreateArgs, notify: bool, + _guard: &WalletGuard<'_>, ) -> Result> { let account = match account_create_args { AccountCreateArgs::Bip32 { prv_key_data_args, account_args } => { @@ -731,7 +797,13 @@ impl Wallet { let account_index = if let Some(account_index) = account_index { account_index } else { - account_store.clone().len(Some(prv_key_data_id)).await? as u64 + let accounts = account_store.clone().iter(Some(prv_key_data_id)).await?.collect::>().await; + + accounts + .into_iter() + .filter(|a| a.as_ref().ok().and_then(|(a, _)| (a.kind == BIP32_ACCOUNT_KIND).then_some(true)).unwrap_or(false)) + .collect::>() + .len() as u64 }; let xpub_key = prv_key_data.create_xpub(payment_secret, BIP32_ACCOUNT_KIND.into(), account_index).await?; @@ -797,6 +869,12 @@ impl Wallet { .ok_or_else(|| Error::PrivateKeyNotFound(prv_key_data_id))?; let account: Arc = Arc::new(legacy::Legacy::try_new(self, account_name, prv_key_data.id).await?); + if let Ok(legacy_account) = account.clone().as_legacy_account() { + legacy_account.create_private_context(wallet_secret, None, None).await?; + log_info!("create_private_context: create_account_legacy, receive_address: {:?}", account.receive_address()); + self.legacy_accounts().insert(account.clone()); + //legacy_account.clear_private_context().await?; + } if account_store.load_single(account.id()).await?.is_some() { return Err(Error::AccountAlreadyExists(*account.id())); @@ -884,7 +962,13 @@ impl Wallet { Ok((wallet_descriptor, storage_descriptor, mnemonic, account)) } - pub async fn get_account_by_id(self: &Arc, account_id: &AccountId) -> Result>> { + pub async fn get_account_by_id( + self: &Arc, + account_id: &AccountId, + _guard: &WalletGuard<'_>, + ) -> Result>> { + let _guard = self.inner.account_guard.lock().await; + if let Some(account) = self.active_accounts().get(account_id) { Ok(Some(account.clone())) } else { @@ -1073,7 +1157,11 @@ impl Wallet { Ok(matches) } - pub async fn accounts(self: &Arc, filter: Option) -> Result>>> { + pub async fn accounts( + self: &Arc, + filter: Option, + _guard: &WalletGuard<'_>, + ) -> Result>>> { let iter = self.inner.store.as_account_store().unwrap().iter(filter).await.unwrap(); let wallet = self.clone(); @@ -1594,6 +1682,7 @@ impl Wallet { payment_secret: Option<&Secret>, kind: AccountKind, mnemonic_phrase: Option<&Secret>, + guard: &WalletGuard<'_>, ) -> Result { if kind != BIP32_ACCOUNT_KIND { return Err(Error::custom("Account kind is not supported")); @@ -1619,7 +1708,7 @@ impl Wallet { let account_create_args = AccountCreateArgs::new_bip32(prv_key_data_id, payment_secret.cloned(), None, None); - let account = self.clone().create_account(wallet_secret, account_create_args, false).await?; + let account = self.clone().create_account(wallet_secret, account_create_args, false, guard).await?; self.store().flush(wallet_secret).await?; diff --git a/wallet/core/src/wasm/api/message.rs b/wallet/core/src/wasm/api/message.rs index 94654c1..a69d3be 100644 --- a/wallet/core/src/wasm/api/message.rs +++ b/wallet/core/src/wasm/api/message.rs @@ -153,8 +153,16 @@ declare! { * @category Wallet API */ export interface IConnectRequest { - url : string; + // destination wRPC node URL (if omitted, the resolver is used) + url? : string; + // network identifier networkId : NetworkId | string; + // retry on error + retryOnError? : boolean; + // block async connect (method will not return until the connection is established) + block? : boolean; + // require node to be synced (fail otherwise) + requireSync? : boolean; } "#, } @@ -162,7 +170,10 @@ declare! { try_from! ( args: IConnectRequest, ConnectRequest, { let url = args.try_get_string("url")?; let network_id = args.get_network_id("networkId")?; - Ok(ConnectRequest { url, network_id }) + let retry_on_error = args.try_get_bool("retryOnError")?.unwrap_or(true); + let block_async_connect = args.try_get_bool("block")?.unwrap_or(false); + let require_sync = args.try_get_bool("requireSync")?.unwrap_or(true); + Ok(ConnectRequest { url, network_id, retry_on_error, block_async_connect, require_sync }) }); declare! { @@ -971,7 +982,7 @@ try_from! (args: IAccountsDiscoveryRequest, AccountsDiscoveryRequest, { let discovery_kind = if let Some(discovery_kind) = discovery_kind.as_string() { discovery_kind.parse()? } else { - AccountsDiscoveryKind::try_cast_from(&discovery_kind)? + AccountsDiscoveryKind::try_enum_from(&discovery_kind)? }; let account_scan_extent = args.get_u32("accountScanExtent")?; let address_scan_extent = args.get_u32("addressScanExtent")?; @@ -1312,7 +1323,7 @@ try_from!(args: IAccountsCreateNewAddressRequest, AccountsCreateNewAddressReques let value = args.get_value("addressKind")?; let kind: NewAddressKind = if let Some(string) = value.as_string() { string.parse()? - } else if let Ok(kind) = NewAddressKind::try_cast_from(&value) { + } else if let Ok(kind) = NewAddressKind::try_enum_from(&value) { kind } else { NewAddressKind::Receive diff --git a/wallet/core/src/wasm/cryptobox.rs b/wallet/core/src/wasm/cryptobox.rs index 65ca706..2443616 100644 --- a/wallet/core/src/wasm/cryptobox.rs +++ b/wallet/core/src/wasm/cryptobox.rs @@ -35,8 +35,11 @@ impl CryptoBoxPrivateKey { impl TryCastFromJs for CryptoBoxPrivateKey { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let secret_key = value.as_ref().try_as_vec_u8()?; if secret_key.len() != KEY_SIZE { return Err(Error::InvalidPrivateKeyLength); @@ -63,8 +66,11 @@ pub struct CryptoBoxPublicKey { impl TryCastFromJs for CryptoBoxPublicKey { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let public_key = value.as_ref().try_as_vec_u8()?; if public_key.len() != KEY_SIZE { Err(Error::InvalidPublicKeyLength) @@ -114,7 +120,7 @@ pub struct CryptoBox { impl CryptoBox { #[wasm_bindgen(constructor)] #[allow(non_snake_case)] - pub fn ctor(secretKey: CryptoBoxPrivateKeyT, peerPublicKey: CryptoBoxPublicKeyT) -> Result { + pub fn ctor(secretKey: &CryptoBoxPrivateKeyT, peerPublicKey: &CryptoBoxPublicKeyT) -> Result { let secret_key = CryptoBoxPrivateKey::try_cast_from(secretKey)?; let peer_public_key = CryptoBoxPublicKey::try_cast_from(peerPublicKey)?; Ok(Self { inner: Arc::new(NativeCryptoBox::new(&secret_key, &peer_public_key)) }) diff --git a/wallet/core/src/wasm/message.rs b/wallet/core/src/wasm/message.rs index 06b3553..ce65e97 100644 --- a/wallet/core/src/wasm/message.rs +++ b/wallet/core/src/wasm/message.rs @@ -28,10 +28,10 @@ extern "C" { #[wasm_bindgen(js_name = signMessage)] pub fn js_sign_message(value: ISignMessage) -> Result { if let Some(object) = Object::try_from(&value) { - let private_key = object.get_cast::("privateKey")?; + let private_key = object.cast_into::("privateKey")?; let raw_msg = object.get_string("message")?; let mut privkey_bytes = [0u8; 32]; - privkey_bytes.copy_from_slice(&private_key.as_ref().secret_bytes()); + privkey_bytes.copy_from_slice(&private_key.secret_bytes()); let pm = PersonalMessage(&raw_msg); let sig_vec = sign_message(&pm, &privkey_bytes)?; privkey_bytes.zeroize(); @@ -66,7 +66,7 @@ extern "C" { #[wasm_bindgen(js_name = verifyMessage, skip_jsdoc)] pub fn js_verify_message(value: IVerifyMessage) -> Result { if let Some(object) = Object::try_from(&value) { - let public_key = object.get_cast::("publicKey")?; + let public_key = object.cast_into::("publicKey")?; let raw_msg = object.get_string("message")?; let signature = object.get_string("signature")?; @@ -74,7 +74,7 @@ pub fn js_verify_message(value: IVerifyMessage) -> Result { let mut signature_bytes = [0u8; 64]; faster_hex::hex_decode(signature.as_bytes(), &mut signature_bytes)?; - Ok(verify_message(&pm, &signature_bytes.to_vec(), &public_key.as_ref().xonly_public_key).is_ok()) + Ok(verify_message(&pm, &signature_bytes.to_vec(), &public_key.xonly_public_key).is_ok()) } else { Err(Error::custom("Failed to parse input")) } diff --git a/wallet/core/src/wasm/notify.rs b/wallet/core/src/wasm/notify.rs index 3fa9ef9..2c5f72d 100644 --- a/wallet/core/src/wasm/notify.rs +++ b/wallet/core/src/wasm/notify.rs @@ -32,33 +32,12 @@ cfg_if! { } /** - * {@link UtxoProcessor} notification event data. - * @category Wallet SDK - */ - export type UtxoProcessorEventData = IConnectEvent - | IDisconnectEvent - | IUtxoIndexNotEnabledEvent - | ISyncStateEvent - | IServerStatusEvent - | IUtxoProcErrorEvent - | IDaaScoreChangeEvent - | IPendingEvent - | IReorgEvent - | IStasisEvent - | IMaturityEvent - | IDiscoveryEvent - | IBalanceEvent - | IErrorEvent - | undefined - ; - - /** - * UtxoProcessor notification event data map. + * {@link UtxoProcessor} notification event data map. * * @category Wallet API */ export type UtxoProcessorEventMap = { - "connect":IConnectEvent, + "connect": IConnectEvent, "disconnect": IDisconnectEvent, "utxo-index-not-enabled": IUtxoIndexNotEnabledEvent, "sync-state": ISyncStateEvent, @@ -80,10 +59,13 @@ cfg_if! { * * @category Wallet API */ - export type IUtxoProcessorEvent = { - [K in keyof UtxoProcessorEventMap]: { event: K, data: UtxoProcessorEventMap[K] } - }[keyof UtxoProcessorEventMap]; + export type UtxoProcessorEvent = { + [K in T]: { + type: K, + data: UtxoProcessorEventMap[K] + } + }[T]; /** * {@link UtxoProcessor} notification callback type. @@ -95,7 +77,8 @@ cfg_if! { * * @category Wallet SDK */ - export type UtxoProcessorNotificationCallback = (event: IUtxoProcessorEvent) => void; + + export type UtxoProcessorNotificationCallback = (event: UtxoProcessorEvent) => void; "#; #[wasm_bindgen] @@ -150,85 +133,53 @@ cfg_if! { Error = "error", } - - /** - * {@link Wallet} notification event data payload. - * @category Wallet API - */ - export type WalletEventData = IConnectEvent - | IDisconnectEvent - | IUtxoIndexNotEnabledEvent - | ISyncStateEvent - | IWalletHintEvent - | IWalletOpenEvent - | IWalletCreateEvent - | IWalletReloadEvent - | IWalletErrorEvent - // | IWalletCloseEvent - | IPrvKeyDataCreateEvent - | IAccountActivationEvent - | IAccountDeactivationEvent - | IAccountSelectionEvent - | IAccountCreateEvent - | IAccountUpdateEvent - | IServerStatusEvent - // | IUtxoProcStartEvent - // | IUtxoProcStopEvent - | IUtxoProcErrorEvent - | IDaaScoreChangeEvent - | IPendingEvent - | IReorgEvent - | IStasisEvent - | IMaturityEvent - | IDiscoveryEvent - | IBalanceEvent - | IErrorEvent - | undefined - ; - /** * Wallet notification event data map. * @see {@link Wallet.addEventListener} * @category Wallet API */ export type WalletEventMap = { - "connect": IConnectEvent, - "disconnect": IDisconnectEvent, - "utxo-index-not-enabled": IUtxoIndexNotEnabledEvent, - "sync-state": ISyncStateEvent, - "wallet-hint": IWalletHintEvent, - "wallet-open": IWalletOpenEvent, - "wallet-create": IWalletCreateEvent, - "wallet-reload": IWalletReloadEvent, - "wallet-error": IWalletErrorEvent, - "wallet-close": undefined, - "prv-key-data-create": IPrvKeyDataCreateEvent, - "account-activation": IAccountActivationEvent, - "account-deactivation": IAccountDeactivationEvent, - "account-selection": IAccountSelectionEvent, - "account-create": IAccountCreateEvent, - "account-update": IAccountUpdateEvent, - "server-status": IServerStatusEvent, - "utxo-proc-start": undefined, - "utxo-proc-stop": undefined, - "utxo-proc-error": IUtxoProcErrorEvent, - "daa-score-change": IDaaScoreChangeEvent, - "pending": IPendingEvent, - "reorg": IReorgEvent, - "stasis": IStasisEvent, - "maturity": IMaturityEvent, - "discovery": IDiscoveryEvent, - "balance": IBalanceEvent, - "error": IErrorEvent, + "connect": IConnectEvent, + "disconnect": IDisconnectEvent, + "utxo-index-not-enabled": IUtxoIndexNotEnabledEvent, + "sync-state": ISyncStateEvent, + "wallet-hint": IWalletHintEvent, + "wallet-open": IWalletOpenEvent, + "wallet-create": IWalletCreateEvent, + "wallet-reload": IWalletReloadEvent, + "wallet-error": IWalletErrorEvent, + "wallet-close": undefined, + "prv-key-data-create": IPrvKeyDataCreateEvent, + "account-activation": IAccountActivationEvent, + "account-deactivation": IAccountDeactivationEvent, + "account-selection": IAccountSelectionEvent, + "account-create": IAccountCreateEvent, + "account-update": IAccountUpdateEvent, + "server-status": IServerStatusEvent, + "utxo-proc-start": undefined, + "utxo-proc-stop": undefined, + "utxo-proc-error": IUtxoProcErrorEvent, + "daa-score-change": IDaaScoreChangeEvent, + "pending": IPendingEvent, + "reorg": IReorgEvent, + "stasis": IStasisEvent, + "maturity": IMaturityEvent, + "discovery": IDiscoveryEvent, + "balance": IBalanceEvent, + "error": IErrorEvent, } /** * {@link Wallet} notification event interface. * @category Wallet API */ - export type IWalletEvent = { - [K in keyof WalletEventMap]: { type: K, data: WalletEventMap[K] } - }[keyof WalletEventMap]; + export type IWalletEvent = { + [K in T]: { + type: K, + data: WalletEventMap[K] + } + }[T]; + /** * Wallet notification callback type. @@ -240,7 +191,7 @@ cfg_if! { * * @category Wallet API */ - export type WalletNotificationCallback = (event: IWalletEvent) => void; + export type WalletNotificationCallback = (event: IWalletEvent) => void; "#; #[wasm_bindgen] diff --git a/wallet/core/src/wasm/signer.rs b/wallet/core/src/wasm/signer.rs index 1b516a1..f7cdbaa 100644 --- a/wallet/core/src/wasm/signer.rs +++ b/wallet/core/src/wasm/signer.rs @@ -3,10 +3,13 @@ use crate::result::Result; use js_sys::Array; use serde_wasm_bindgen::from_value; use spectre_consensus_client::{sign_with_multiple_v3, Transaction}; +use spectre_consensus_core::hashing::wasm::SighashType; +use spectre_consensus_core::sign::sign_input; use spectre_consensus_core::tx::PopulatedTransaction; use spectre_consensus_core::{hashing::sighash_type::SIG_HASH_ALL, sign::verify}; use spectre_hashes::Hash; use spectre_wallet_keys::privatekey::PrivateKey; +use spectre_wasm_core::types::HexString; #[wasm_bindgen] extern "C" { @@ -31,26 +34,26 @@ impl TryFrom for Vec { /// `signTransaction()` is a helper function to sign a transaction using a private key array or a signer array. /// @category Wallet SDK #[wasm_bindgen(js_name = "signTransaction")] -pub fn js_sign_transaction(tx: Transaction, signer: PrivateKeyArrayT, verify_sig: bool) -> Result { +pub fn js_sign_transaction(tx: &Transaction, signer: &PrivateKeyArrayT, verify_sig: bool) -> Result { if signer.is_array() { let mut private_keys: Vec<[u8; 32]> = vec![]; - for key in Array::from(&signer).iter() { - let key = PrivateKey::try_cast_from(key).map_err(|_| Error::Custom("Unable to cast PrivateKey".to_string()))?; + for key in Array::from(signer).iter() { + let key = PrivateKey::try_cast_from(&key).map_err(|_| Error::Custom("Unable to cast PrivateKey".to_string()))?; private_keys.push(key.as_ref().secret_bytes()); } let tx = sign_transaction(tx, &private_keys, verify_sig).map_err(|err| Error::Custom(format!("Unable to sign: {err:?}")))?; private_keys.zeroize(); - Ok(tx) + Ok(tx.clone()) } else { Err(Error::custom("signTransaction() requires an array of signatures")) } } -pub fn sign_transaction(tx: Transaction, private_keys: &[[u8; 32]], verify_sig: bool) -> Result { +pub fn sign_transaction<'a>(tx: &'a Transaction, private_keys: &[[u8; 32]], verify_sig: bool) -> Result<&'a Transaction> { let tx = sign(tx, private_keys)?; if verify_sig { - let (cctx, utxos) = tx.tx_and_utxos(); + let (cctx, utxos) = tx.tx_and_utxos()?; let populated_transaction = PopulatedTransaction::new(&cctx, utxos); verify(&populated_transaction)?; } @@ -60,10 +63,32 @@ pub fn sign_transaction(tx: Transaction, private_keys: &[[u8; 32]], verify_sig: /// Sign a transaction using schnorr, returns a new transaction with the signatures added. /// The resulting transaction may be partially signed if the supplied keys are not sufficient /// to sign all of its inputs. -pub fn sign(tx: Transaction, privkeys: &[[u8; 32]]) -> Result { +pub fn sign<'a>(tx: &'a Transaction, privkeys: &[[u8; 32]]) -> Result<&'a Transaction> { Ok(sign_with_multiple_v3(tx, privkeys)?.unwrap()) } +/// `createInputSignature()` is a helper function to sign a transaction input with a specific SigHash type using a private key. +/// @category Wallet SDK +#[wasm_bindgen(js_name = "createInputSignature")] +pub fn create_input_signature( + tx: &Transaction, + input_index: u8, + private_key: &PrivateKey, + sighash_type: Option, +) -> Result { + let (cctx, utxos) = tx.tx_and_utxos()?; + let populated_transaction = PopulatedTransaction::new(&cctx, utxos); + + let signature = sign_input( + &populated_transaction, + input_index.into(), + &private_key.secret_bytes(), + sighash_type.unwrap_or(SighashType::All).into(), + ); + + Ok(signature.to_hex().into()) +} + /// @category Wallet SDK #[wasm_bindgen(js_name=signScriptHash)] pub fn sign_script_hash(script_hash: JsValue, privkey: &PrivateKey) -> Result { diff --git a/wallet/core/src/wasm/tx/consensus.rs b/wallet/core/src/wasm/tx/consensus.rs index 489c43f..f5efee3 100644 --- a/wallet/core/src/wasm/tx/consensus.rs +++ b/wallet/core/src/wasm/tx/consensus.rs @@ -3,7 +3,14 @@ use spectre_addresses::Address; use spectre_consensus_core::{config::params::Params, network::NetworkType}; use wasm_bindgen::prelude::*; +/// +/// `ConsensusParams` can be obtained using `getConsensusParametersByNetwork` or `getConsensusParametersByAddress`. +/// +/// @see {@link getConsensusParametersByNetwork} +/// @see {@link getConsensusParametersByAddress} +/// /// @category Wallet SDK +/// #[wasm_bindgen] pub struct ConsensusParams { params: Params, diff --git a/wallet/core/src/wasm/tx/fees.rs b/wallet/core/src/wasm/tx/fees.rs index d9ee278..0fae3a8 100644 --- a/wallet/core/src/wasm/tx/fees.rs +++ b/wallet/core/src/wasm/tx/fees.rs @@ -39,7 +39,7 @@ impl TryFrom for Fees { } else if let Ok(object) = args.dyn_into::() { let amount = object.get_u64("amount")?; if let Some(source) = object.try_get_value("source")? { - let source = FeeSource::try_cast_from(&source)?; + let source = FeeSource::try_enum_from(&source)?; match source { FeeSource::SenderPays => Ok(Fees::SenderPays(amount)), FeeSource::ReceiverPays => Ok(Fees::ReceiverPays(amount)), diff --git a/wallet/core/src/wasm/tx/generator/generator.rs b/wallet/core/src/wasm/tx/generator/generator.rs index 619a0ef..faec27f 100644 --- a/wallet/core/src/wasm/tx/generator/generator.rs +++ b/wallet/core/src/wasm/tx/generator/generator.rs @@ -64,6 +64,15 @@ interface IGeneratorSettingsObject { * interface, or a {@link UtxoContext} instance. */ entries: IUtxoEntry[] | UtxoEntryReference[] | UtxoContext; + /** + * Optional UTXO entries that will be consumed before those available in `entries`. + * You can use this property to apply custom input selection logic. + * Please note that these inputs are consumed first, then `entries` are consumed + * to generate a desirable transaction output amount. If transaction mass + * overflows, these inputs will be consumed into a batch/sweep transaction + * where the destination if the `changeAddress`. + */ + priorityEntries?: IUtxoEntry[] | UtxoEntryReference[], /** * Optional number of signature operations in the transaction. */ @@ -147,6 +156,7 @@ impl Generator { let GeneratorSettings { network_id, source, + priority_utxo_entries, multiplexer, final_transaction_destination, change_address, @@ -167,6 +177,7 @@ impl Generator { native::GeneratorSettings::try_new_with_iterator( network_id, Box::new(utxo_entries.into_iter()), + priority_utxo_entries, change_address, sig_op_count, minimum_signatures, @@ -182,6 +193,7 @@ impl Generator { native::GeneratorSettings::try_new_with_context( utxo_context.into(), + priority_utxo_entries, change_address, sig_op_count, minimum_signatures, @@ -244,6 +256,7 @@ enum GeneratorSource { struct GeneratorSettings { pub network_id: Option, pub source: GeneratorSource, + pub priority_utxo_entries: Option>, pub multiplexer: Option>>, pub final_transaction_destination: PaymentDestination, pub change_address: Option
, @@ -263,18 +276,20 @@ impl TryFrom for GeneratorSettings { let final_transaction_destination: PaymentDestination = if outputs.is_undefined() { PaymentDestination::Change } else { PaymentOutputs::try_owned_from(outputs)?.into() }; - let change_address = args.try_get_cast::
("changeAddress")?.map(Cast::into_owned); + let change_address = args.try_cast_into::
("changeAddress")?; let final_priority_fee = args.get::("priorityFee")?.try_into()?; - let generator_source = if let Ok(Some(context)) = args.try_get_cast::("entries") { - GeneratorSource::UtxoContext(context.into_owned()) + let generator_source = if let Ok(Some(context)) = args.try_cast_into::("entries") { + GeneratorSource::UtxoContext(context) } else if let Some(utxo_entries) = args.try_get_value("entries")? { GeneratorSource::UtxoEntries(utxo_entries.try_into_utxo_entry_references()?) } else { - return Err(Error::custom("'entries', 'context' or 'account' property is required for Generator")); + return Err(Error::custom("'entries' property is required for Generator")); }; + let priority_utxo_entries = args.try_get_value("priorityEntries")?.map(|v| v.try_into_utxo_entry_references()).transpose()?; + let sig_op_count = args.get_value("sigOpCount")?; let sig_op_count = if !sig_op_count.is_undefined() { sig_op_count.as_f64().expect("sigOpCount should be a number") as u8 } else { 1 }; @@ -291,6 +306,7 @@ impl TryFrom for GeneratorSettings { let settings = GeneratorSettings { network_id, source: generator_source, + priority_utxo_entries, multiplexer: None, final_transaction_destination, change_address, diff --git a/wallet/core/src/wasm/tx/generator/pending.rs b/wallet/core/src/wasm/tx/generator/pending.rs index 00b35cb..184261b 100644 --- a/wallet/core/src/wasm/tx/generator/pending.rs +++ b/wallet/core/src/wasm/tx/generator/pending.rs @@ -3,8 +3,10 @@ use crate::result::Result; use crate::tx::generator as native; use crate::wasm::PrivateKeyArrayT; use spectre_consensus_client::{numeric, string}; -use spectre_consensus_client::{ITransaction, Transaction}; +use spectre_consensus_client::{Transaction, TransactionT}; +use spectre_consensus_core::hashing::wasm::SighashType; use spectre_wallet_keys::privatekey::PrivateKey; +use spectre_wasm_core::types::{BinaryT, HexString}; use spectre_wrpc_wasm::RpcClient; /// @category Wallet SDK @@ -70,16 +72,44 @@ impl PendingTransaction { self.inner.utxo_entries().values().map(|utxo_entry| JsValue::from(utxo_entry.clone())).collect() } + #[wasm_bindgen(js_name = createInputSignature)] + pub fn create_input_signature( + &self, + input_index: u8, + private_key: &PrivateKey, + sighash_type: Option, + ) -> Result { + let signature = self.inner.create_input_signature( + input_index.into(), + &private_key.secret_bytes(), + sighash_type.unwrap_or(SighashType::All).into(), + )?; + + Ok(signature.to_hex().into()) + } + + #[wasm_bindgen(js_name = fillInput)] + pub fn fill_input(&self, input_index: u8, signature_script: BinaryT) -> Result<()> { + self.inner.fill_input(input_index.into(), signature_script.try_as_vec_u8()?) + } + + #[wasm_bindgen(js_name = signInput)] + pub fn sign_input(&self, input_index: u8, private_key: &PrivateKey, sighash_type: Option) -> Result<()> { + self.inner.sign_input(input_index.into(), &private_key.secret_bytes(), sighash_type.unwrap_or(SighashType::All).into())?; + + Ok(()) + } + /// Sign transaction with supplied [`Array`] or [`PrivateKey`] or an array of /// raw private key bytes (encoded as `Uint8Array` or as hex strings) - pub fn sign(&self, js_value: PrivateKeyArrayT) -> Result<()> { + pub fn sign(&self, js_value: PrivateKeyArrayT, check_fully_signed: Option) -> Result<()> { if let Ok(keys) = js_value.dyn_into::() { let keys = keys .iter() - .map(PrivateKey::try_cast_from) + .map(PrivateKey::try_owned_from) .collect::, spectre_wallet_keys::error::Error>>()?; - let mut keys = keys.iter().map(|key| key.as_ref().secret_bytes()).collect::>(); - self.inner.try_sign_with_keys(&keys)?; + let mut keys = keys.iter().map(|key| key.secret_bytes()).collect::>(); + self.inner.try_sign_with_keys(&keys, check_fully_signed)?; keys.zeroize(); Ok(()) } else { @@ -92,6 +122,14 @@ impl PendingTransaction { /// {@link UtxoContext} if one was used to create the transaction /// and will return UTXOs back to {@link UtxoContext} in case of /// a failed submission. + /// + /// # Important + /// + /// Make sure to consume the returned `txid` value. Always invoke this method + /// as follows `let txid = await pendingTransaction.submit(rpc);`. If you do not + /// consume the returned value and the rpc object is temporary, the GC will + /// collect the `rpc` object passed to submit() potentially causing a panic. + /// /// @see {@link RpcClient.submitTransaction} pub async fn submit(&self, wasm_rpc_client: &RpcClient) -> Result { let rpc: Arc = wasm_rpc_client.client().clone(); @@ -110,7 +148,7 @@ impl PendingTransaction { /// @see {@link ISerializableTransaction} /// @see {@link Transaction}, {@link ISerializableTransaction} #[wasm_bindgen(js_name = "serializeToObject")] - pub fn serialize_to_object(&self) -> Result { + pub fn serialize_to_object(&self) -> Result { Ok(numeric::SerializableTransaction::from_cctx_transaction(&self.inner.transaction(), self.inner.utxo_entries())? .serialize_to_object()? .into()) diff --git a/wallet/core/src/wasm/tx/mass.rs b/wallet/core/src/wasm/tx/mass.rs index 6eb6f83..c041dba 100644 --- a/wallet/core/src/wasm/tx/mass.rs +++ b/wallet/core/src/wasm/tx/mass.rs @@ -1,151 +1,43 @@ use crate::imports::NetworkParams; use crate::result::Result; use crate::tx::mass; -use crate::wasm::tx::*; use spectre_consensus_client::*; use spectre_consensus_core::config::params::Params; -use spectre_consensus_core::tx as cctx; -use std::sync::Arc; +use spectre_consensus_core::network::{NetworkId, NetworkIdT}; use wasm_bindgen::prelude::*; use workflow_wasm::convert::*; +/// `calculateTransactionMass()` returns the mass of the passed transaction. +/// If the transaction is invalid, the function throws an error. +/// If the mass is larger than the transaction mass allowed by the network, the function +/// returns `undefined` which can be treated as a mass overflow condition. +/// /// @category Wallet SDK -#[wasm_bindgen] -pub struct MassCalculator { - mc: Arc, +/// +#[wasm_bindgen(js_name = calculateTransactionMass)] +pub fn calculate_transaction_mass(network_id: NetworkIdT, tx: &TransactionT) -> Result> { + let tx = Transaction::try_cast_from(tx)?; + let network_id = NetworkId::try_owned_from(network_id)?; + let consensus_params = Params::from(network_id); + let network_params = NetworkParams::from(network_id); + let mc = mass::MassCalculator::new(&consensus_params, network_params); + mc.calc_tx_overall_mass(tx.as_ref()) } -#[wasm_bindgen] -impl MassCalculator { - #[wasm_bindgen(constructor)] - pub fn new(cp: ConsensusParams) -> Self { - let consensus_params = Params::from(cp); - let network_params = NetworkParams::from(consensus_params.net); - Self { mc: Arc::new(mass::MassCalculator::new(&consensus_params, &network_params)) } - } - - #[wasm_bindgen(js_name=isDust)] - pub fn is_dust(&self, amount: u64) -> bool { - self.mc.is_dust(amount) - } - - /// `isTransactionOutputDust()` returns whether or not the passed transaction output - /// amount is considered dust or not based on the configured minimum transaction - /// relay fee. - /// - /// Dust is defined in terms of the minimum transaction relay fee. In particular, - /// if the cost to the network to spend coins is more than 1/3 of the minimum - /// transaction relay fee, it is considered dust. - /// - /// It is exposed by `MiningManager` for use by transaction generators and wallets. - #[wasm_bindgen(js_name=isTransactionOutputDust)] - pub fn is_transaction_output_dust(transaction_output: &JsValue) -> Result { - let transaction_output = TransactionOutput::try_from(transaction_output)?; - let transaction_output = cctx::TransactionOutput::from(&transaction_output); - Ok(mass::is_transaction_output_dust(&transaction_output)) - } - - /// `minimumRelayTransactionFee()` specifies the minimum transaction fee for a transaction to be accepted to - /// the mempool and relayed. It is specified in sompi per 1kg (or 1000 grams) of transaction mass. - /// - /// `pub(crate) const MINIMUM_RELAY_TRANSACTION_FEE: u64 = 1000;` - #[wasm_bindgen(js_name=minimumRelayTransactionFee)] - pub fn minimum_relay_transaction_fee() -> u32 { - mass::MINIMUM_RELAY_TRANSACTION_FEE as u32 - } - - /// `maximumStandardTransactionMass()` is the maximum mass allowed for transactions that - /// are considered standard and will therefore be relayed and considered for mining. - /// - /// `pub const MAXIMUM_STANDARD_TRANSACTION_MASS: u64 = 100_000;` - #[wasm_bindgen(js_name=maximumStandardTransactionMass)] - pub fn maximum_standard_transaction_mass() -> u32 { - mass::MAXIMUM_STANDARD_TRANSACTION_MASS as u32 - } - - /// minimum_required_transaction_relay_fee returns the minimum transaction fee required - /// for a transaction with the passed mass to be accepted into the mempool and relayed. - #[wasm_bindgen(js_name=minimumRequiredTransactionRelayFee)] - pub fn calc_minimum_required_transaction_relay_fee(mass: u32) -> u32 { - mass::calc_minimum_required_transaction_relay_fee(mass as u64) as u32 - } - - #[wasm_bindgen(js_name=calcMassForTransaction)] - pub fn calc_mass_for_transaction(&self, tx: &JsValue) -> Result { - let tx = Transaction::try_cast_from(tx)?; - let tx = cctx::Transaction::from(tx.as_ref()); - Ok(self.mc.calc_mass_for_transaction(&tx) as u32) - } - - #[wasm_bindgen(js_name=blankTransactionSerializedByteSize)] - pub fn blank_transaction_serialized_byte_size() -> u32 { - mass::blank_transaction_serialized_byte_size() as u32 - } - - #[wasm_bindgen(js_name=blankTransactionMass)] - pub fn blank_transaction_mass(&self) -> u32 { - self.mc.blank_transaction_mass() as u32 - } - - #[wasm_bindgen(js_name=calcMassForPayload)] - pub fn calc_mass_for_payload(&self, payload_byte_size: usize) -> u32 { - self.mc.calc_mass_for_payload(payload_byte_size) as u32 - } - - #[wasm_bindgen(js_name=calcMassForOutputs)] - pub fn calc_mass_for_outputs(&self, outputs: JsValue) -> Result { - let outputs = outputs - .dyn_into::()? - .iter() - .map(TransactionOutput::try_from) - .collect::, spectre_consensus_client::error::Error>>()?; - let outputs = outputs.iter().map(|output| self.calc_mass_for_output(output)).collect::>>()?; - Ok(outputs.iter().sum()) - } - - #[wasm_bindgen(js_name=calcMassForInputs)] - pub fn calc_mass_for_inputs(&self, inputs: JsValue) -> Result { - let inputs = inputs - .dyn_into::()? - .iter() - .map(TransactionInput::try_owned_from) - .collect::, spectre_consensus_client::error::Error>>()?; - let inputs = inputs.iter().map(|input| self.calc_mass_for_input(input)).collect::>>()?; - Ok(inputs.iter().sum()) - } - - #[wasm_bindgen(js_name=calcMassForOutput)] - pub fn calc_mass_for_output(&self, output: &TransactionOutput) -> Result { - // let output = TransactionOutput::try_from(output)?; - let output = cctx::TransactionOutput::from(output); - Ok(self.mc.calc_mass_for_output(&output) as u32) - } - - #[wasm_bindgen(js_name=calcMassForInput)] - pub fn calc_mass_for_input(&self, input: &TransactionInput) -> Result { - // let input = TransactionInput::try_from(input)?; - let input = cctx::TransactionInput::from(input); - Ok(self.mc.calc_mass_for_input(&input) as u32) - } - - #[wasm_bindgen(js_name=calcSignatureMass)] - pub fn calc_signature_mass(&self, minimum_signatures: u16) -> u32 { - self.mc.calc_signature_mass(minimum_signatures) as u32 - } - - #[wasm_bindgen(js_name=calcSignatureMassForInputs)] - pub fn calc_signature_mass_for_inputs(&self, number_of_inputs: usize, minimum_signatures: u16) -> u32 { - self.mc.calc_signature_mass_for_inputs(number_of_inputs, minimum_signatures) as u32 - } - - #[wasm_bindgen(js_name=calcMinimumTransactionRelayFeeFromMass)] - pub fn calc_minimum_transaction_relay_fee_from_mass(&self, mass: u64) -> u32 { - self.mc.calc_minimum_transaction_fee_from_mass(mass) as u32 - } - - #[wasm_bindgen(js_name=calcMiniumTxRelayFee)] - pub fn calc_minimum_transaction_relay_fee(&self, transaction: &Transaction, minimum_signatures: u16) -> Result { - let tx = cctx::Transaction::from(transaction); - Ok(self.mc.calc_minium_transaction_relay_fee(&tx, minimum_signatures) as u32) - } +/// `calculateTransactionFee()` returns minimum fees needed for the transaction to be +/// accepted by the network. If the transaction is invalid, the function throws an error. +/// If the mass of the transaction is larger than the maximum allowed by the network, the +/// function returns `undefined` which can be treated as a mass overflow condition. +/// +/// @category Wallet SDK +/// +#[wasm_bindgen(js_name = calculateTransactionFee)] +pub fn calculate_transaction_fee(network_id: NetworkIdT, tx: &TransactionT) -> Result> { + let tx = Transaction::try_cast_from(tx)?; + let network_id = NetworkId::try_owned_from(network_id)?; + let consensus_params = Params::from(network_id); + let network_params = NetworkParams::from(network_id); + let mc = mass::MassCalculator::new(&consensus_params, network_params); + let fee = mc.calc_tx_overall_mass(tx.as_ref())?.map(|mass| mc.calc_fee_for_mass(mass)); + Ok(fee) } diff --git a/wallet/core/src/wasm/tx/mod.rs b/wallet/core/src/wasm/tx/mod.rs index df826a9..742bb89 100644 --- a/wallet/core/src/wasm/tx/mod.rs +++ b/wallet/core/src/wasm/tx/mod.rs @@ -1,10 +1,8 @@ -pub mod consensus; pub mod fees; pub mod generator; pub mod mass; pub mod utils; -pub use self::consensus::*; pub use self::fees::*; pub use self::generator::*; pub use self::mass::*; diff --git a/wallet/core/src/wasm/tx/utils.rs b/wallet/core/src/wasm/tx/utils.rs index a68d9bc..37e0979 100644 --- a/wallet/core/src/wasm/tx/utils.rs +++ b/wallet/core/src/wasm/tx/utils.rs @@ -1,14 +1,11 @@ use crate::imports::*; use crate::result::Result; use crate::tx::{IPaymentOutputArray, PaymentOutputs}; -use crate::wasm::tx::consensus::get_consensus_params_by_address; use crate::wasm::tx::generator::*; -use crate::wasm::tx::mass::MassCalculator; -use spectre_addresses::{Address, AddressT}; use spectre_consensus_client::*; use spectre_consensus_core::subnets::SUBNETWORK_ID_NATIVE; -//use spectre_consensus_wasm::*; use spectre_wallet_macros::declare_typescript_wasm_interface as declare; +use spectre_wasm_core::types::BinaryT; use workflow_core::runtime::is_web; /// Create a basic transaction without any mass limit checks. @@ -17,32 +14,19 @@ use workflow_core::runtime::is_web; pub fn create_transaction_js( utxo_entry_source: IUtxoEntryArray, outputs: IPaymentOutputArray, - change_address: AddressT, priority_fee: BigInt, - payload: JsValue, - sig_op_count: JsValue, - minimum_signatures: JsValue, + payload: Option, + sig_op_count: Option, ) -> crate::result::Result { - let change_address = Address::try_cast_from(change_address)?; - let params = get_consensus_params_by_address(change_address.as_ref()); - let mc = MassCalculator::new(params); - let utxo_entries = if let Some(utxo_entries) = utxo_entry_source.dyn_ref::() { - utxo_entries.to_vec().iter().map(UtxoEntryReference::try_cast_from).collect::, _>>()? + utxo_entries.to_vec().iter().map(UtxoEntryReference::try_owned_from).collect::, _>>()? } else { return Err(Error::custom("utxo_entries must be an array")); }; let priority_fee: u64 = priority_fee.try_into().map_err(|err| Error::custom(format!("invalid fee value: {err}")))?; - let payload = payload.try_as_vec_u8().ok().unwrap_or_default(); + let payload = payload.and_then(|payload| payload.try_as_vec_u8().ok()).unwrap_or_default(); let outputs = PaymentOutputs::try_owned_from(outputs)?; - let sig_op_count = - if !sig_op_count.is_undefined() { sig_op_count.as_f64().expect("sigOpCount should be a number") as u8 } else { 1 }; - - let minimum_signatures = if !minimum_signatures.is_undefined() { - minimum_signatures.as_f64().expect("minimumSignatures should be a number") as u16 - } else { - 1 - }; + let sig_op_count = sig_op_count.unwrap_or(1); // --- @@ -53,10 +37,10 @@ pub fn create_transaction_js( .into_iter() .enumerate() .map(|(sequence, reference)| { - let UtxoEntryReference { utxo } = reference.as_ref(); + let UtxoEntryReference { utxo } = &reference; total_input_amount += utxo.amount(); - entries.push(reference.as_ref().clone()); - TransactionInput::new(utxo.outpoint.clone(), vec![], sequence as u64, sig_op_count, Some(reference.into_owned())) + entries.push(reference.clone()); + TransactionInput::new(utxo.outpoint.clone(), None, sequence as u64, sig_op_count, Some(reference)) }) .collect::>(); @@ -64,12 +48,8 @@ pub fn create_transaction_js( return Err(format!("priority fee({priority_fee}) > amount({total_input_amount})").into()); } - // TODO - Calculate mass and fees - let outputs: Vec = outputs.into(); let transaction = Transaction::new(None, 0, inputs, outputs, 0, SUBNETWORK_ID_NATIVE, 0, payload)?; - let _fee = mc.calc_minimum_transaction_relay_fee(&transaction, minimum_signatures); - //let mtx = SignableTransaction::new(transaction, entries.into()); Ok(transaction) } diff --git a/wallet/core/src/wasm/utxo/context.rs b/wallet/core/src/wasm/utxo/context.rs index 659cc13..26f8fba 100644 --- a/wallet/core/src/wasm/utxo/context.rs +++ b/wallet/core/src/wasm/utxo/context.rs @@ -147,6 +147,7 @@ impl UtxoContext { self.inner().clear().await } + #[wasm_bindgen(getter, js_name = "isActive")] pub fn active(&self) -> bool { let processor = self.inner().processor(); processor.try_rpc_ctl().map(|ctl| ctl.is_connected()).unwrap_or(false) && processor.is_connected() && processor.is_running() @@ -251,7 +252,10 @@ impl From for native::UtxoContext { impl TryCastFromJs for UtxoContext { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { Ok(Self::try_ref_from_js_value_as_cast(value)?) } } @@ -265,15 +269,15 @@ impl TryFrom for UtxoContextCreateArgs { type Error = Error; fn try_from(value: IUtxoContextArgs) -> std::result::Result { if let Some(object) = Object::try_from(&value) { - let processor = object.get_cast::("processor")?; + let processor = object.cast_into::("processor")?; - let binding = if let Some(id) = object.try_get_cast::("id")? { - UtxoContextBinding::Id(UtxoContextId::new(id.into_owned())) + let binding = if let Some(id) = object.try_cast_into::("id")? { + UtxoContextBinding::Id(UtxoContextId::new(id)) } else { UtxoContextBinding::default() }; - Ok(UtxoContextCreateArgs { binding, processor: processor.into_owned() }) + Ok(UtxoContextCreateArgs { binding, processor }) } else { Err(Error::custom("UtxoProcessor: supplied value must be an object")) } diff --git a/wallet/core/src/wasm/utxo/processor.rs b/wallet/core/src/wasm/utxo/processor.rs index aedc8a4..5d56d9f 100644 --- a/wallet/core/src/wasm/utxo/processor.rs +++ b/wallet/core/src/wasm/utxo/processor.rs @@ -63,14 +63,14 @@ cfg_if! { /** * @param {UtxoProcessorNotificationCallback} callback */ - addEventListener(callback:UtxoProcessorNotificationCallback): void; + addEventListener(callback: UtxoProcessorNotificationCallback): void; /** * @param {UtxoProcessorEventType} event * @param {UtxoProcessorNotificationCallback} [callback] */ - addEventListener( - event: M, - callback: (eventData: UtxoProcessorEventMap[M]) => void + addEventListener( + event: E, + callback: UtxoProcessorNotificationCallback ) }"#; } @@ -153,11 +153,54 @@ impl UtxoProcessor { self.inner.processor.set_network_id(network_id.as_ref()); Ok(()) } + + #[wasm_bindgen(getter, js_name = "isActive")] + pub fn is_active(&self) -> bool { + let processor = &self.inner.processor; + processor.try_rpc_ctl().map(|ctl| ctl.is_connected()).unwrap_or(false) && processor.is_connected() && processor.is_running() + } + + /// + /// Set the coinbase transaction maturity period DAA score for a given network. + /// This controls the DAA period after which the user transactions are considered mature + /// and the wallet subsystem emits the transaction maturity event. + /// + /// @see {@link TransactionRecord} + /// @see {@link IUtxoProcessorEvent} + /// + /// @category Wallet SDK + /// + #[wasm_bindgen(js_name = "setCoinbaseTransactionMaturityDAA")] + pub fn set_coinbase_transaction_maturity_period_daa_js(network_id: &NetworkIdT, value: u64) -> Result<()> { + let network_id = NetworkId::try_cast_from(network_id)?.into_owned(); + crate::utxo::set_coinbase_transaction_maturity_period_daa(&network_id, value); + Ok(()) + } + + /// + /// Set the user transaction maturity period DAA score for a given network. + /// This controls the DAA period after which the user transactions are considered mature + /// and the wallet subsystem emits the transaction maturity event. + /// + /// @see {@link TransactionRecord} + /// @see {@link IUtxoProcessorEvent} + /// + /// @category Wallet SDK + /// + #[wasm_bindgen(js_name = "setUserTransactionMaturityDAA")] + pub fn set_user_transaction_maturity_period_daa_js(network_id: &NetworkIdT, value: u64) -> Result<()> { + let network_id = NetworkId::try_cast_from(network_id)?.into_owned(); + crate::utxo::set_user_transaction_maturity_period_daa(&network_id, value); + Ok(()) + } } impl TryCastFromJs for UtxoProcessor { type Error = workflow_wasm::error::Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { Self::try_ref_from_js_value_as_cast(value) } } diff --git a/wallet/core/src/wasm/wallet/account.rs b/wallet/core/src/wasm/wallet/account.rs index 9951895..0d6223c 100644 --- a/wallet/core/src/wasm/wallet/account.rs +++ b/wallet/core/src/wasm/wallet/account.rs @@ -85,11 +85,11 @@ impl Account { self.inner.clone().scan(None, None).await } - pub async fn send(&self, js_value: JsValue) -> Result { - let _args = AccountSendArgs::try_from(js_value)?; + // pub async fn send(&self, js_value: JsValue) -> Result { + // let _args = AccountSendArgs::try_from(js_value)?; - todo!() - } + // todo!() + // } } impl From for Arc { @@ -105,51 +105,51 @@ impl TryFrom<&JsValue> for Account { } } -pub struct AccountSendArgs { - pub outputs: PaymentOutputs, - pub priority_fee_sompi: Option, - pub include_fees_in_amount: bool, - - pub wallet_secret: Secret, - pub payment_secret: Option, - pub abortable: Abortable, -} - -impl TryFrom for AccountSendArgs { - type Error = Error; - fn try_from(js_value: JsValue) -> std::result::Result { - if let Some(object) = Object::try_from(&js_value) { - let outputs = object.get_cast::("outputs")?.into_owned(); - - let priority_fee_sompi = object.get_u64("priorityFee").ok(); - let include_fees_in_amount = object.get_bool("includeFeesInAmount").unwrap_or(false); - let abortable = object.get("abortable").ok().and_then(|v| Abortable::try_from(&v).ok()).unwrap_or_default(); - - let wallet_secret = object.get_string("walletSecret")?.into(); - let payment_secret = object.get_value("paymentSecret")?.as_string().map(|s| s.into()); - - let send_args = - AccountSendArgs { outputs, priority_fee_sompi, include_fees_in_amount, wallet_secret, payment_secret, abortable }; - - Ok(send_args) - } else { - Err("Argument to Account::send() must be an object".into()) - } - } -} - -pub struct AccountCreateArgs {} - -impl TryFrom for AccountCreateArgs { - type Error = Error; - fn try_from(value: JsValue) -> std::result::Result { - if let Some(object) = Object::try_from(&value) { - let _keypair = object.try_get_cast::("keypair")?; - let _public_key = object.try_get_cast::("keypair")?; - - Ok(AccountCreateArgs {}) - } else { - Err(Error::custom("Account: supplied value must be an object")) - } - } -} +// pub struct AccountSendArgs { +// pub outputs: PaymentOutputs, +// pub priority_fee_sompi: Option, +// pub include_fees_in_amount: bool, + +// pub wallet_secret: Secret, +// pub payment_secret: Option, +// pub abortable: Abortable, +// } + +// impl TryFrom for AccountSendArgs { +// type Error = Error; +// fn try_from<'a,R>(js_value: JsValue) -> std::result::Result { +// if let Some(object) = Object::try_from(&js_value) { +// let outputs = object.cast_into::("outputs")?.into_owned(); + +// let priority_fee_sompi = object.get_u64("priorityFee").ok(); +// let include_fees_in_amount = object.get_bool("includeFeesInAmount").unwrap_or(false); +// let abortable = object.get("abortable").ok().and_then(|v| Abortable::try_from(&v).ok()).unwrap_or_default(); + +// let wallet_secret = object.get_string("walletSecret")?.into(); +// let payment_secret = object.get_value("paymentSecret")?.as_string().map(|s| s.into()); + +// let send_args = +// AccountSendArgs { outputs, priority_fee_sompi, include_fees_in_amount, wallet_secret, payment_secret, abortable }; + +// Ok(send_args) +// } else { +// Err("Argument to Account::send() must be an object".into()) +// } +// } +// } + +// pub struct AccountCreateArgs {} + +// impl TryFrom for AccountCreateArgs { +// type Error = Error; +// fn try_from(value: JsValue) -> std::result::Result { +// if let Some(object) = Object::try_from(&value) { +// let _keypair = object.try_cast_into::("keypair")?; +// let _public_key = object.try_cast_into::("keypair")?; + +// Ok(AccountCreateArgs {}) +// } else { +// Err(Error::custom("Account: supplied value must be an object")) +// } +// } +// } diff --git a/wallet/core/src/wasm/wallet/mod.rs b/wallet/core/src/wasm/wallet/mod.rs index cacd736..2ae871c 100644 --- a/wallet/core/src/wasm/wallet/mod.rs +++ b/wallet/core/src/wasm/wallet/mod.rs @@ -1,4 +1,3 @@ -pub mod account; pub mod keydata; #[allow(clippy::module_inception)] pub mod wallet; diff --git a/wallet/keys/Cargo.toml b/wallet/keys/Cargo.toml index 4aadbc4..e7266a2 100644 --- a/wallet/keys/Cargo.toml +++ b/wallet/keys/Cargo.toml @@ -46,5 +46,5 @@ zeroize.workspace = true [target.'cfg(not(target_arch = "wasm32"))'.dev-dependencies] tokio.workspace = true -[lints.clippy] -empty_docs = "allow" +[lints] +workspace = true diff --git a/wallet/keys/src/derivation/gen0/hd.rs b/wallet/keys/src/derivation/gen0/hd.rs index f44f6ce..d87056d 100644 --- a/wallet/keys/src/derivation/gen0/hd.rs +++ b/wallet/keys/src/derivation/gen0/hd.rs @@ -176,7 +176,7 @@ impl PubkeyDerivationManagerV0 { return Ok(*key); } - Err(crate::error::Error::Custom("PubkeyDerivationManagerV0 initialization is pending (Error: 102).".into())) + Err(crate::error::Error::Custom("PubkeyDerivationManagerV0 initialization is pending (Error: 105).".into())) } pub fn create_address(key: &secp256k1::PublicKey, prefix: AddressPrefix, _ecdsa: bool) -> Result
{ diff --git a/wallet/keys/src/derivation_path.rs b/wallet/keys/src/derivation_path.rs index 627284e..07818f0 100644 --- a/wallet/keys/src/derivation_path.rs +++ b/wallet/keys/src/derivation_path.rs @@ -51,8 +51,11 @@ impl DerivationPath { impl TryCastFromJs for DerivationPath { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let value = value.as_ref(); if let Some(path) = value.as_string() { Ok(DerivationPath::new(&path)?) diff --git a/wallet/keys/src/keypair.rs b/wallet/keys/src/keypair.rs index 3bed015..f4b39f3 100644 --- a/wallet/keys/src/keypair.rs +++ b/wallet/keys/src/keypair.rs @@ -100,7 +100,10 @@ impl Keypair { impl TryCastFromJs for Keypair { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { Ok(Self::try_ref_from_js_value_as_cast(value)?) } } diff --git a/wallet/keys/src/privatekey.rs b/wallet/keys/src/privatekey.rs index fca6e1d..f5a5168 100644 --- a/wallet/keys/src/privatekey.rs +++ b/wallet/keys/src/privatekey.rs @@ -93,8 +93,11 @@ impl PrivateKey { impl TryCastFromJs for PrivateKey { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(hex_str) = value.as_ref().as_string() { Self::try_new(hex_str.as_str()) } else if Array::is_array(value.as_ref()) { diff --git a/wallet/keys/src/pubkeygen.rs b/wallet/keys/src/pubkeygen.rs index 3b3a396..c93ac72 100644 --- a/wallet/keys/src/pubkeygen.rs +++ b/wallet/keys/src/pubkeygen.rs @@ -21,7 +21,7 @@ pub struct PublicKeyGenerator { #[wasm_bindgen] impl PublicKeyGenerator { #[wasm_bindgen(js_name=fromXPub)] - pub fn from_xpub(kpub: XPubT, cosigner_index: Option) -> Result { + pub fn from_xpub(kpub: &XPubT, cosigner_index: Option) -> Result { let kpub = XPub::try_cast_from(kpub)?; let xpub = kpub.as_ref().inner(); let hd_wallet = WalletDerivationManager::from_extended_public_key(xpub.clone(), cosigner_index)?; @@ -193,7 +193,7 @@ impl PublicKeyGenerator { #[wasm_bindgen(js_name=changeAddressAsString)] #[allow(non_snake_case)] pub fn change_address_as_string(&self, networkType: &NetworkTypeT, index: u32) -> Result { - Ok(PublicKey::from(self.hd_wallet.receive_pubkey_manager().derive_pubkey(index)?) + Ok(PublicKey::from(self.hd_wallet.change_pubkey_manager().derive_pubkey(index)?) .to_address(networkType.try_into()?)? .to_string()) } diff --git a/wallet/keys/src/publickey.rs b/wallet/keys/src/publickey.rs index 7e9ca8a..91b289c 100644 --- a/wallet/keys/src/publickey.rs +++ b/wallet/keys/src/publickey.rs @@ -138,8 +138,11 @@ extern "C" { impl TryCastFromJs for PublicKey { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { let value = value.as_ref(); if let Some(hex_str) = value.as_string() { Ok(PublicKey::try_new(hex_str.as_str())?) @@ -150,13 +153,13 @@ impl TryCastFromJs for PublicKey { } } -impl TryFrom for Vec { +impl TryFrom<&PublicKeyArrayT> for Vec { type Error = Error; - fn try_from(value: PublicKeyArrayT) -> Result { + fn try_from(value: &PublicKeyArrayT) -> Result { if value.is_array() { - let array = Array::from(&value); - let pubkeys = array.iter().map(PublicKey::try_cast_from).collect::>>()?; - Ok(pubkeys.iter().map(|pk| pk.as_ref().try_into()).collect::>>()?) + let array = Array::from(value); + let pubkeys = array.iter().map(PublicKey::try_owned_from).collect::>>()?; + Ok(pubkeys.iter().map(|pk| pk.try_into()).collect::>>()?) } else { Err(Error::InvalidPublicKeyArray) } diff --git a/wallet/keys/src/xprv.rs b/wallet/keys/src/xprv.rs index bf403a9..4b18021 100644 --- a/wallet/keys/src/xprv.rs +++ b/wallet/keys/src/xprv.rs @@ -1,3 +1,5 @@ +use spectre_bip32::{ChainCode, KeyFingerprint}; + use crate::imports::*; /// @@ -13,7 +15,7 @@ use crate::imports::*; /// #[derive(Clone, CastFromJs)] -#[wasm_bindgen] +#[wasm_bindgen(inspectable)] pub struct XPrv { inner: ExtendedPrivateKey, } @@ -41,9 +43,9 @@ impl XPrv { } #[wasm_bindgen(js_name=deriveChild)] - pub fn derive_child(&self, chile_number: u32, hardened: Option) -> Result { - let chile_number = ChildNumber::new(chile_number, hardened.unwrap_or(false))?; - let inner = self.inner.derive_child(chile_number)?; + pub fn derive_child(&self, child_number: u32, hardened: Option) -> Result { + let child_number = ChildNumber::new(child_number, hardened.unwrap_or(false))?; + let inner = self.inner.derive_child(child_number)?; Ok(Self { inner }) } @@ -70,6 +72,60 @@ impl XPrv { let public_key = self.inner.public_key(); Ok(public_key.into()) } + + #[wasm_bindgen(js_name = toPrivateKey)] + pub fn to_private_key(&self) -> Result { + let private_key = self.inner.private_key(); + Ok(private_key.into()) + } + + // ~~~~ Getters ~~~~ + + #[wasm_bindgen(getter)] + pub fn xprv(&self) -> Result { + let str = self.inner.to_extended_key("kprv".try_into()?).to_string(); + Ok(str) + } + + #[wasm_bindgen(getter, js_name = "privateKey")] + pub fn private_key_as_hex_string(&self) -> String { + use spectre_bip32::PrivateKey; + self.inner.private_key().to_bytes().to_vec().to_hex() + } + + #[wasm_bindgen(getter)] + pub fn depth(&self) -> u8 { + self.inner.attrs().depth + } + + #[wasm_bindgen(getter, js_name = parentFingerprint)] + pub fn parent_fingerprint_as_hex_string(&self) -> String { + self.inner.attrs().parent_fingerprint.to_vec().to_hex() + } + + #[wasm_bindgen(getter, js_name = childNumber)] + pub fn child_number(&self) -> u32 { + self.inner.attrs().child_number.into() + } + + #[wasm_bindgen(getter, js_name = chainCode)] + pub fn chain_code_as_hex_string(&self) -> String { + self.inner.attrs().chain_code.to_vec().to_hex() + } +} + +impl XPrv { + pub fn private_key(&self) -> &SecretKey { + self.inner.private_key() + } + + pub fn parent_fingerprint(&self) -> KeyFingerprint { + self.inner.attrs().parent_fingerprint + } + + pub fn chain_code(&self) -> ChainCode { + self.inner.attrs().chain_code + } } impl<'a> From<&'a XPrv> for &'a ExtendedPrivateKey { @@ -86,8 +142,11 @@ extern "C" { impl TryCastFromJs for XPrv { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(xprv) = value.as_ref().as_string() { Ok(XPrv::from_xprv_str(xprv)?) } else { diff --git a/wallet/keys/src/xpub.rs b/wallet/keys/src/xpub.rs index 6b2d46d..a00f2f7 100644 --- a/wallet/keys/src/xpub.rs +++ b/wallet/keys/src/xpub.rs @@ -1,4 +1,4 @@ -use spectre_bip32::Prefix; +use spectre_bip32::{ChainCode, KeyFingerprint, Prefix}; use std::{fmt, str::FromStr}; use crate::imports::*; @@ -15,7 +15,7 @@ use crate::imports::*; /// @category Wallet SDK /// #[derive(Clone, CastFromJs)] -#[wasm_bindgen] +#[wasm_bindgen(inspectable)] pub struct XPub { inner: ExtendedPublicKey, } @@ -35,9 +35,9 @@ impl XPub { } #[wasm_bindgen(js_name=deriveChild)] - pub fn derive_child(&self, chile_number: u32, hardened: Option) -> Result { - let chile_number = ChildNumber::new(chile_number, hardened.unwrap_or(false))?; - let inner = self.inner.derive_child(chile_number)?; + pub fn derive_child(&self, child_number: u32, hardened: Option) -> Result { + let child_number = ChildNumber::new(child_number, hardened.unwrap_or(false))?; + let inner = self.inner.derive_child(child_number)?; Ok(Self { inner }) } @@ -58,6 +58,44 @@ impl XPub { pub fn public_key(&self) -> PublicKey { self.inner.public_key().into() } + + // ~~~~ Getters ~~~~ + + #[wasm_bindgen(getter)] + pub fn xpub(&self) -> Result { + let str = self.inner.to_extended_key("kpub".try_into()?).to_string(); + Ok(str) + } + + #[wasm_bindgen(getter)] + pub fn depth(&self) -> u8 { + self.inner.attrs().depth + } + + #[wasm_bindgen(getter, js_name = parentFingerprint)] + pub fn parent_fingerprint_as_hex_string(&self) -> String { + self.inner.attrs().parent_fingerprint.to_vec().to_hex() + } + + #[wasm_bindgen(getter, js_name = childNumber)] + pub fn child_number(&self) -> u32 { + self.inner.attrs().child_number.into() + } + + #[wasm_bindgen(getter, js_name = chainCode)] + pub fn chain_code_as_hex_string(&self) -> String { + self.inner.attrs().chain_code.to_vec().to_hex() + } +} + +impl XPub { + pub fn parent_fingerprint(&self) -> KeyFingerprint { + self.inner.attrs().parent_fingerprint + } + + pub fn chain_code(&self) -> ChainCode { + self.inner.attrs().chain_code + } } impl From> for XPub { @@ -74,8 +112,11 @@ extern "C" { impl TryCastFromJs for XPub { type Error = Error; - fn try_cast_from(value: impl AsRef) -> Result, Self::Error> { - Self::resolve(&value, || { + fn try_cast_from<'a, R>(value: &'a R) -> Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { if let Some(xpub) = value.as_ref().as_string() { Ok(XPub::try_new(xpub.as_str())?) } else { diff --git a/wallet/macros/src/wallet/client.rs b/wallet/macros/src/wallet/client.rs index 8bbc51b..c942af0 100644 --- a/wallet/macros/src/wallet/client.rs +++ b/wallet/macros/src/wallet/client.rs @@ -61,7 +61,7 @@ impl ToTokens for RpcTable { { match __self.codec { Codec::Borsh(ref codec) => { - Ok(#response_type::try_from_slice(&codec.call(op, request.try_to_vec()?).await?)?) + Ok(#response_type::try_from_slice(&codec.call(op, borsh::to_vec(&request)?).await?)?) }, Codec::Serde(ref codec) => { let request = serde_json::to_string(&request)?; diff --git a/wallet/macros/src/wallet/server.rs b/wallet/macros/src/wallet/server.rs index a1e7bf1..073d330 100644 --- a/wallet/macros/src/wallet/server.rs +++ b/wallet/macros/src/wallet/server.rs @@ -38,7 +38,7 @@ impl ToTokens for RpcTable { targets_borsh.push(quote! { #hash_64 => { - Ok(self.wallet_api().#fn_call(#request_type::try_from_slice(&request)?).await?.try_to_vec()?) + Ok(borsh::to_vec(&self.wallet_api().#fn_call(#request_type::try_from_slice(&request)?).await?)?) } }); diff --git a/wallet/psst/Cargo.toml b/wallet/psst/Cargo.toml index 9518635..f935567 100644 --- a/wallet/psst/Cargo.toml +++ b/wallet/psst/Cargo.toml @@ -19,6 +19,7 @@ wasm32-sdk = ["spectre-consensus-client/wasm32-sdk"] wasm32-types = ["spectre-consensus-client/wasm32-types"] [dependencies] +spectre-addresses.workspace = true spectre-bip32.workspace = true spectre-consensus-client.workspace = true spectre-consensus-core.workspace = true @@ -26,12 +27,20 @@ spectre-txscript-errors.workspace = true spectre-txscript.workspace = true spectre-utils.workspace = true +bincode.workspace = true derive_builder.workspace = true +js-sys.workspace = true +futures.workspace = true +hex.workspace = true secp256k1.workspace = true +serde_repr.workspace = true serde-value.workspace = true serde.workspace = true -serde_repr.workspace = true thiserror.workspace = true +wasm-bindgen.workspace = true +serde_json.workspace = true +serde-wasm-bindgen.workspace = true +workflow-wasm.workspace = true [dev-dependencies] -serde_json.workspace = true \ No newline at end of file +serde_json.workspace = true diff --git a/wallet/psst/examples/multisig.rs b/wallet/psst/examples/multisig.rs index 3b2f907..3c6abf4 100644 --- a/wallet/psst/examples/multisig.rs +++ b/wallet/psst/examples/multisig.rs @@ -4,7 +4,7 @@ use spectre_consensus_core::{ tx::{TransactionId, TransactionOutpoint, UtxoEntry}, }; use spectre_txscript::{multisig_redeem_script, opcodes::codes::OpData65, pay_to_script_hash_script, script_builder::ScriptBuilder}; -use spectre_wallet_psst::{ +use spectre_wallet_psst::prelude::{ Combiner, Creator, Extractor, Finalizer, Inner, InputBuilder, SignInputOk, Signature, Signer, Updater, PSST, }; use std::{iter, str::FromStr}; diff --git a/wallet/psst/src/bundle.rs b/wallet/psst/src/bundle.rs new file mode 100644 index 0000000..bbebbeb --- /dev/null +++ b/wallet/psst/src/bundle.rs @@ -0,0 +1,353 @@ +use crate::error::Error; +use crate::prelude::*; +use crate::psst::{Inner as PSSTInner, PSST}; +// use crate::wasm::result; + +use spectre_addresses::{Address, Prefix}; +// use spectre_bip32::Prefix; +use spectre_consensus_core::network::{NetworkId, NetworkType}; +use spectre_consensus_core::tx::{ScriptPublicKey, TransactionOutpoint, UtxoEntry}; + +use hex; +use serde::{Deserialize, Serialize}; +use spectre_txscript::{extract_script_pub_key_address, pay_to_address_script, pay_to_script_hash_script}; +use std::ops::Deref; + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct Bundle(pub Vec); + +impl From> for Bundle { + fn from(psst: PSST) -> Self { + Bundle(vec![psst.deref().clone()]) + } +} + +impl From>> for Bundle { + fn from(pssts: Vec>) -> Self { + let inner_list = pssts.into_iter().map(|psst| psst.deref().clone()).collect(); + Bundle(inner_list) + } +} + +impl Bundle { + pub fn new() -> Self { + Self(Vec::new()) + } + + /// Adds an Inner instance to the bundle + pub fn add_inner(&mut self, inner: PSSTInner) { + self.0.push(inner); + } + + /// Adds a PSST instance to the bundle + pub fn add_psst(&mut self, psst: PSST) { + self.0.push(psst.deref().clone()); + } + + /// Merges another bundle into the current bundle + pub fn merge(&mut self, other: Bundle) { + for inner in other.0 { + self.0.push(inner); + } + } + + /// Iterator over the inner PSST instances + pub fn iter(&self) -> std::slice::Iter { + self.0.iter() + } + + pub fn serialize(&self) -> Result { + Ok(format!("PSSB{}", hex::encode(serde_json::to_string(self)?))) + } + + pub fn deserialize(hex_data: &str) -> Result { + if let Some(hex_data) = hex_data.strip_prefix("PSSB") { + Ok(serde_json::from_slice(hex::decode(hex_data)?.as_slice())?) + } else { + Err(Error::PssbPrefixError) + } + } + + pub fn display_format(&self, network_id: NetworkId, sompi_formatter: F) -> String + where + F: Fn(u64, &NetworkType) -> String, + { + let mut result = "".to_string(); + + for (psst_index, bundle_inner) in self.0.iter().enumerate() { + let psst: PSST = PSST::::from(bundle_inner.to_owned()); + + result.push_str(&format!("\r\nPSST #{:02}\r\n", psst_index + 1)); + + for (key_inner, input) in psst.clone().inputs.iter().enumerate() { + result.push_str(&format!("Input #{:02}\r\n", key_inner + 1)); + + if let Some(utxo_entry) = &input.utxo_entry { + result.push_str(&format!(" amount: {}\r\n", sompi_formatter(utxo_entry.amount, &NetworkType::from(network_id)))); + result.push_str(&format!( + " address: {}\r\n", + extract_script_pub_key_address(&utxo_entry.script_public_key, Prefix::from(network_id)) + .expect("Input address") + )); + } + } + + result.push_str("---\r\n"); + + for (key_inner, output) in psst.clone().outputs.iter().enumerate() { + result.push_str(&format!("Output #{:02}\r\n", key_inner + 1)); + result.push_str(&format!(" amount: {}\r\n", sompi_formatter(output.amount, &NetworkType::from(network_id)))); + result.push_str(&format!( + " address: {}\r\n", + extract_script_pub_key_address(&output.script_public_key, Prefix::from(network_id)).expect("Input address") + )); + } + } + result + } +} + +impl AsRef<[PSSTInner]> for Bundle { + fn as_ref(&self) -> &[PSSTInner] { + self.0.as_slice() + } +} + +impl TryFrom for Bundle { + type Error = Error; + fn try_from(value: String) -> Result { + Bundle::deserialize(&value) + } +} + +impl TryFrom<&str> for Bundle { + type Error = Error; + fn try_from(value: &str) -> Result { + Bundle::deserialize(value) + } +} +impl TryFrom for String { + type Error = Error; + fn try_from(value: Bundle) -> Result { + match Bundle::serialize(&value) { + Ok(output) => Ok(output.to_owned()), + Err(e) => Err(Error::PssbSerializeError(e.to_string())), + } + } +} + +impl Default for Bundle { + fn default() -> Self { + Self::new() + } +} + +pub fn lock_script_sig_templating(payload: String, pubkey_bytes: Option<&[u8]>) -> Result, Error> { + let mut payload_bytes: Vec = hex::decode(payload)?; + + if let Some(pubkey) = pubkey_bytes { + let placeholder = b"{{pubkey}}"; + + // Search for the placeholder in payload bytes to be replaced by public key. + if let Some(pos) = payload_bytes.windows(placeholder.len()).position(|window| window == placeholder) { + payload_bytes.splice(pos..pos + placeholder.len(), pubkey.iter().cloned()); + } + } + Ok(payload_bytes) +} + +pub fn script_sig_to_address(script_sig: &[u8], prefix: spectre_addresses::Prefix) -> Result { + extract_script_pub_key_address(&pay_to_script_hash_script(script_sig), prefix).map_err(Error::P2SHExtractError) +} + +pub fn unlock_utxos_as_pssb( + utxo_references: Vec<(UtxoEntry, TransactionOutpoint)>, + recipient: &Address, + script_sig: Vec, + priority_fee_sompi_per_transaction: u64, +) -> Result { + // Fee per transaction. + // Check if each UTXO's amounts can cover priority fee. + utxo_references + .iter() + .map(|(entry, _)| { + if entry.amount <= priority_fee_sompi_per_transaction { + return Err(Error::ExcessUnlockFeeError); + } + Ok(()) + }) + .collect::, _>>()?; + + let recipient_spk = pay_to_address_script(recipient); + let (successes, errors): (Vec<_>, Vec<_>) = utxo_references + .into_iter() + .map(|(utxo_entry, outpoint)| { + unlock_utxo(&utxo_entry, &outpoint, &recipient_spk, &script_sig, priority_fee_sompi_per_transaction) + }) + .partition(Result::is_ok); + + let successful_bundles: Vec<_> = successes.into_iter().filter_map(Result::ok).collect(); + let error_list: Vec<_> = errors.into_iter().filter_map(Result::err).collect(); + + if !error_list.is_empty() { + return Err(Error::MultipleUnlockUtxoError(error_list)); + } + + let merged_bundle = successful_bundles.into_iter().fold(None, |acc: Option, bundle| match acc { + Some(mut merged_bundle) => { + merged_bundle.merge(bundle); + Some(merged_bundle) + } + None => Some(bundle), + }); + + match merged_bundle { + None => Err("Generating an empty PSSB".into()), + Some(bundle) => Ok(bundle), + } +} + +pub fn unlock_utxo( + utxo_entry: &UtxoEntry, + outpoint: &TransactionOutpoint, + script_public_key: &ScriptPublicKey, + script_sig: &[u8], + priority_fee_sompi: u64, +) -> Result { + let input = InputBuilder::default() + .utxo_entry(utxo_entry.to_owned()) + .previous_outpoint(outpoint.to_owned()) + .sig_op_count(1) + .redeem_script(script_sig.to_vec()) + .build()?; + + let output = OutputBuilder::default() + .amount(utxo_entry.amount - priority_fee_sompi) + .script_public_key(script_public_key.clone()) + .build()?; + + let psst: PSST = PSST::::default().constructor().input(input).output(output); + Ok(psst.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::prelude::*; + use crate::role::Creator; + use crate::role::*; + use secp256k1::Secp256k1; + use secp256k1::{rand::thread_rng, Keypair}; + use spectre_consensus_core::tx::{TransactionId, TransactionOutpoint, UtxoEntry}; + use spectre_txscript::{multisig_redeem_script, pay_to_script_hash_script}; + use std::str::FromStr; + use std::sync::Once; + + static INIT: Once = Once::new(); + static mut CONTEXT: Option)>> = None; + + fn mock_context() -> &'static ([Keypair; 2], Vec) { + unsafe { + INIT.call_once(|| { + let kps = [Keypair::new(&Secp256k1::new(), &mut thread_rng()), Keypair::new(&Secp256k1::new(), &mut thread_rng())]; + let redeem_script: Vec = multisig_redeem_script(kps.iter().map(|pk| pk.x_only_public_key().0.serialize()), 2) + .expect("Test multisig redeem script"); + + CONTEXT = Some(Box::new((kps, redeem_script))); + }); + + CONTEXT.as_ref().unwrap() + } + } + + // Mock multisig PSST from example + fn mock_psst_constructor() -> PSST { + let (_, redeem_script) = mock_context(); + let psst = PSST::::default().inputs_modifiable().outputs_modifiable(); + let input_0 = InputBuilder::default() + .utxo_entry(UtxoEntry { + amount: 12793000000000, + script_public_key: pay_to_script_hash_script(redeem_script), + block_daa_score: 36151168, + is_coinbase: false, + }) + .previous_outpoint(TransactionOutpoint { + transaction_id: TransactionId::from_str("63020db736215f8b1105a9281f7bcbb6473d965ecc45bb2fb5da59bd35e6ff84").unwrap(), + index: 0, + }) + .sig_op_count(2) + .redeem_script(redeem_script.to_owned()) + .build() + .expect("Mock PSST constructor"); + + psst.constructor().input(input_0) + } + + #[test] + fn test_pssb_serialization() { + let constructor = mock_psst_constructor(); + let bundle = Bundle::from(constructor.clone()); + + println!("Bundle: {}", serde_json::to_string(&bundle).unwrap()); + + // Serialize Bundle + let serialized = bundle.serialize().map_err(|err| format!("Unable to serialize bundle: {err}")).unwrap(); + println!("Serialized: {}", serialized); + + assert!(!bundle.0.is_empty()); + + match Bundle::deserialize(&serialized) { + Ok(bundle_constructor_deser) => { + println!("Deserialized: {:?}", bundle_constructor_deser); + let psst_constructor_deser: Option> = + bundle_constructor_deser.0.first().map(|inner| PSST::from(inner.clone())); + match psst_constructor_deser { + Some(_) => println!("PSST deserialized successfully"), + None => println!("No elements in the inner list to deserialize"), + } + } + Err(e) => { + eprintln!("Failed to deserialize: {}", e); + panic!() + } + } + } + + #[test] + fn test_pssb_bundle_creation() { + let bundle = Bundle::new(); + assert!(bundle.0.is_empty()); + } + + #[test] + fn test_pssb_new_with_psst() { + let psst = PSST::::default(); + let bundle = Bundle::from(psst); + assert_eq!(bundle.0.len(), 1); + } + + #[test] + fn test_pssb_add_psst() { + let mut bundle = Bundle::new(); + let psst = PSST::::default(); + bundle.add_psst(psst); + assert_eq!(bundle.0.len(), 1); + } + + #[test] + fn test_pssb_merge_bundles() { + let mut bundle1 = Bundle::new(); + let mut bundle2 = Bundle::new(); + + let inner1 = PSSTInner::default(); + let inner2 = PSSTInner::default(); + + bundle1.add_inner(inner1.clone()); + bundle2.add_inner(inner2.clone()); + + bundle1.merge(bundle2); + + assert_eq!(bundle1.0.len(), 2); + } +} diff --git a/wallet/psst/src/convert.rs b/wallet/psst/src/convert.rs new file mode 100644 index 0000000..0d3f118 --- /dev/null +++ b/wallet/psst/src/convert.rs @@ -0,0 +1,109 @@ +use crate::error::Error; +use crate::input::{Input, InputBuilder}; +use crate::output::{Output, OutputBuilder}; +use crate::psst::{Global, Inner}; +use spectre_consensus_client::{Transaction, TransactionInput, TransactionInputInner, TransactionOutput, TransactionOutputInner}; +use spectre_consensus_core::tx as cctx; + +impl TryFrom for Inner { + type Error = Error; + fn try_from(_transaction: Transaction) -> Result { + Inner::try_from(cctx::Transaction::from(&_transaction)) + } +} + +impl TryFrom for Input { + type Error = Error; + fn try_from(input: TransactionInput) -> std::result::Result { + let TransactionInputInner { previous_outpoint, signature_script: _, sequence: _, sig_op_count, utxo } = &*input.inner(); + + let input = InputBuilder::default() + .utxo_entry(utxo.as_ref().ok_or(Error::MissingUtxoEntry)?.into()) + .previous_outpoint(previous_outpoint.into()) + // .sequence(*sequence) + // min_time + // partial_sigs + // sighash_type + // redeem_script + .sig_op_count(*sig_op_count) + // bip32_derivations + // final_script_sig + .build()?; + + Ok(input) + } +} + +impl TryFrom for Output { + type Error = Error; + fn try_from(output: TransactionOutput) -> std::result::Result { + // Self::Transaction(transaction) + + let TransactionOutputInner { value, script_public_key } = &*output.inner(); + + let output = OutputBuilder::default() + .amount(*value) + .script_public_key(script_public_key.clone()) + // .redeem_script + // .bip32_derivations + // .proprietaries + // .unknowns + .build()?; + + Ok(output) + } +} + +impl TryFrom<(cctx::Transaction, Vec<(&cctx::TransactionInput, &cctx::UtxoEntry)>)> for Inner { + type Error = Error; // Define your error type + + fn try_from( + (transaction, populated_inputs): (cctx::Transaction, Vec<(&cctx::TransactionInput, &cctx::UtxoEntry)>), + ) -> Result { + let inputs: Result, Self::Error> = populated_inputs + .into_iter() + .map(|(input, utxo)| { + InputBuilder::default() + .utxo_entry(utxo.to_owned().clone()) + .previous_outpoint(input.previous_outpoint) + .sig_op_count(input.sig_op_count) + .build() + .map_err(Error::TxToInnerConversionInputBuildingError) + // Handle the error + }) + .collect::>(); + + let outputs: Result, Self::Error> = transaction + .outputs + .iter() + .map(|output| { + Output::try_from(TransactionOutput::from(output.to_owned())).map_err(|e| Error::TxToInnerConversionError(Box::new(e))) + }) + .collect::>(); + + Ok(Inner { global: Global::default(), inputs: inputs?, outputs: outputs? }) + } +} + +impl TryFrom for Inner { + type Error = Error; + fn try_from(transaction: cctx::Transaction) -> Result { + let inputs = transaction + .inputs + .iter() + .map(|input| { + Input::try_from(TransactionInput::from(input.to_owned())).map_err(|e| Error::TxToInnerConversionError(Box::new(e))) + }) + .collect::>()?; + + let outputs = transaction + .outputs + .iter() + .map(|output| { + Output::try_from(TransactionOutput::from(output.to_owned())).map_err(|e| Error::TxToInnerConversionError(Box::new(e))) + }) + .collect::>()?; + + Ok(Inner { global: Global::default(), inputs, outputs }) + } +} diff --git a/wallet/psst/src/error.rs b/wallet/psst/src/error.rs index 5041190..0ce433d 100644 --- a/wallet/psst/src/error.rs +++ b/wallet/psst/src/error.rs @@ -1,11 +1,46 @@ +use spectre_txscript_errors::TxScriptError; + +use crate::input::InputBuilderError; + #[derive(thiserror::Error, Debug)] pub enum Error { + #[error("{0}")] + Custom(String), #[error(transparent)] ConstructorError(#[from] ConstructorError), #[error("OutputNotModifiable")] OutOfBounds, + #[error("Missing UTXO entry")] + MissingUtxoEntry, + #[error("Missing redeem script")] + MissingRedeemScript, + #[error(transparent)] + InputBuilder(#[from] crate::input::InputBuilderError), + #[error(transparent)] + OutputBuilder(#[from] crate::output::OutputBuilderError), + #[error("Serialization error: {0}")] + HexDecodeError(#[from] hex::FromHexError), + #[error("Json deserialize error: {0}")] + JsonDeserializeError(#[from] serde_json::Error), + #[error("Serialize error")] + PssbSerializeError(String), + #[error("Unlock utxo error")] + MultipleUnlockUtxoError(Vec), + #[error("Unlock fees exceed available amount")] + ExcessUnlockFeeError, + #[error("Transaction output to output conversion error")] + TxToInnerConversionError(#[source] Box), + #[error("Transaction input building error in conversion")] + TxToInnerConversionInputBuildingError(#[source] InputBuilderError), + #[error("P2SH extraction error")] + P2SHExtractError(#[source] TxScriptError), + #[error("PSSB hex serialization error: {0}")] + PssbSerializeToHexError(String), + #[error("PSSB serialization requires 'PSSB' prefix")] + PssbPrefixError, + #[error("PSST serialization requires 'PSST' prefix")] + PsstPrefixError, } - #[derive(thiserror::Error, Debug)] pub enum ConstructorError { #[error("InputNotModifiable")] @@ -13,3 +48,21 @@ pub enum ConstructorError { #[error("OutputNotModifiable")] OutputNotModifiable, } + +impl From for Error { + fn from(err: String) -> Self { + Self::Custom(err) + } +} + +impl From<&str> for Error { + fn from(err: &str) -> Self { + Self::Custom(err.to_string()) + } +} + +#[derive(Debug, thiserror::Error)] +pub enum ConversionError { + #[error("Invalid output conversion")] + InvalidOutput, +} diff --git a/wallet/psst/src/global.rs b/wallet/psst/src/global.rs index b04bb4d..85e15d9 100644 --- a/wallet/psst/src/global.rs +++ b/wallet/psst/src/global.rs @@ -1,4 +1,5 @@ -use crate::{utils::combine_if_no_conflicts, KeySource, Version}; +use crate::psst::{KeySource, Version}; +use crate::utils::combine_if_no_conflicts; use derive_builder::Builder; use serde::{Deserialize, Serialize}; use spectre_consensus_core::tx::TransactionId; @@ -10,6 +11,7 @@ use std::{ type Xpub = spectre_bip32::ExtendedPublicKey; #[derive(Debug, Clone, Builder, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] #[builder(default)] pub struct Global { /// The version number of this PSST. @@ -33,6 +35,7 @@ pub struct Global { /// Proprietary key-value pairs for this output. pub proprietaries: BTreeMap, /// Unknown key-value pairs for this output. + #[serde(flatten)] pub unknowns: BTreeMap, } diff --git a/wallet/psst/src/input.rs b/wallet/psst/src/input.rs index 33bb61a..192c789 100644 --- a/wallet/psst/src/input.rs +++ b/wallet/psst/src/input.rs @@ -1,7 +1,5 @@ -use crate::{ - utils::{combine_if_no_conflicts, Error as CombineMapErr}, - KeySource, PartialSigs, -}; +use crate::psst::{KeySource, PartialSigs}; +use crate::utils::{combine_if_no_conflicts, Error as CombineMapErr}; use derive_builder::Builder; use serde::{Deserialize, Serialize}; use spectre_consensus_core::{ @@ -12,6 +10,7 @@ use std::{collections::BTreeMap, marker::PhantomData, ops::Add}; // todo add unknown field? combine them by deduplicating, if there are different values - return error? #[derive(Builder, Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] #[builder(default)] #[builder(setter(skip))] pub struct Input { @@ -47,7 +46,7 @@ pub struct Input { /// scripts necessary for this input to pass validation. pub final_script_sig: Option>, #[serde(skip_serializing, default)] - hidden: PhantomData<()>, // prevents manual filling of fields + pub(crate) hidden: PhantomData<()>, // prevents manual filling of fields #[builder(setter)] /// Proprietary key-value pairs for this output. pub proprietaries: BTreeMap, diff --git a/wallet/psst/src/lib.rs b/wallet/psst/src/lib.rs index 4211a40..32229bb 100644 --- a/wallet/psst/src/lib.rs +++ b/wallet/psst/src/lib.rs @@ -1,458 +1,32 @@ -use serde::{Deserialize, Serialize}; -use serde_repr::{Deserialize_repr, Serialize_repr}; -use spectre_bip32::{secp256k1, DerivationPath, KeyFingerprint}; -use std::{collections::BTreeMap, fmt::Display, fmt::Formatter, future::Future, marker::PhantomData, ops::Deref}; - -mod error; -mod global; -mod input; - -mod output; - -mod role; +//! +//! PSST is a crate for working with Partially Signed Spectre Transactions (PSSTs). +//! This crate provides following primitives: `PSST`, `PSSTBuilder` and `Bundle`. +//! The `Bundle` struct is used for PSST exchange payload serialization and carries +//! multiple `PSST` instances allowing for exchange of Spectre sweep transactions. +//! + +pub mod bundle; +pub mod error; +pub mod global; +pub mod input; +pub mod output; +pub mod psst; +pub mod role; +pub mod wasm; + +mod convert; mod utils; -pub use error::Error; -pub use global::{Global, GlobalBuilder}; -pub use input::{Input, InputBuilder}; -pub use output::{Output, OutputBuilder}; -pub use role::{Combiner, Constructor, Creator, Extractor, Finalizer, Signer, Updater}; -use spectre_consensus_core::tx::UtxoEntry; -use spectre_consensus_core::{ - hashing::{sighash::SigHashReusedValues, sighash_type::SigHashType}, - subnets::SUBNETWORK_ID_NATIVE, - tx::{MutableTransaction, SignableTransaction, Transaction, TransactionId, TransactionInput, TransactionOutput}, -}; -use spectre_txscript::{caches::Cache, TxScriptEngine}; - -#[derive(Debug, Default, Serialize, Deserialize, Clone)] -pub struct Inner { - /// The global map. - pub global: Global, - /// The corresponding key-value map for each input in the unsigned transaction. - pub inputs: Vec, - /// The corresponding key-value map for each output in the unsigned transaction. - pub outputs: Vec, -} - -#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize_repr, Deserialize_repr)] -#[repr(u8)] -pub enum Version { - #[default] - Zero = 0, -} - -impl Display for Version { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match self { - Version::Zero => write!(f, "{}", Version::Zero as u8), - } - } -} - -/// Full information on the used extended public key: fingerprint of the -/// master extended public key and a derivation path from it. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct KeySource { - #[serde(with = "spectre_utils::serde_bytes_fixed")] - pub key_fingerprint: KeyFingerprint, - pub derivation_path: DerivationPath, -} - -impl KeySource { - pub fn new(key_fingerprint: KeyFingerprint, derivation_path: DerivationPath) -> Self { - Self { key_fingerprint, derivation_path } - } -} - -pub type PartialSigs = BTreeMap; - -#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)] -pub enum Signature { - ECDSA(secp256k1::ecdsa::Signature), - Schnorr(secp256k1::schnorr::Signature), -} - -impl Signature { - pub fn into_bytes(self) -> [u8; 64] { - match self { - Signature::ECDSA(s) => s.serialize_compact(), - Signature::Schnorr(s) => s.serialize(), - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -pub struct PSST { - #[serde(flatten)] - inner_psst: Inner, - #[serde(skip_serializing, default)] - role: PhantomData, -} - -impl Clone for PSST { - fn clone(&self) -> Self { - PSST { inner_psst: self.inner_psst.clone(), role: Default::default() } - } -} - -impl Deref for PSST { - type Target = Inner; - - fn deref(&self) -> &Self::Target { - &self.inner_psst - } -} - -impl PSST { - fn unsigned_tx(&self) -> SignableTransaction { - let tx = Transaction::new( - self.global.tx_version, - self.inputs - .iter() - .map(|Input { previous_outpoint, sequence, sig_op_count, .. }| TransactionInput { - previous_outpoint: *previous_outpoint, - signature_script: vec![], - sequence: sequence.unwrap_or(u64::MAX), - sig_op_count: sig_op_count.unwrap_or(0), - }) - .collect(), - self.outputs - .iter() - .map(|Output { amount, script_public_key, .. }: &Output| TransactionOutput { - value: *amount, - script_public_key: script_public_key.clone(), - }) - .collect(), - self.determine_lock_time(), - SUBNETWORK_ID_NATIVE, - 0, - vec![], - ); - let entries = self.inputs.iter().filter_map(|Input { utxo_entry, .. }| utxo_entry.clone()).collect(); - SignableTransaction::with_entries(tx, entries) - } - - fn calculate_id_internal(&self) -> TransactionId { - self.unsigned_tx().tx.id() - } - - fn determine_lock_time(&self) -> u64 { - self.inputs.iter().map(|input: &Input| input.min_time).max().unwrap_or(self.global.fallback_lock_time).unwrap_or(0) - } -} - -impl Default for PSST { - fn default() -> Self { - PSST { inner_psst: Default::default(), role: Default::default() } - } -} - -impl PSST { - /// Sets the fallback lock time. - pub fn fallback_lock_time(mut self, fallback: u64) -> Self { - self.inner_psst.global.fallback_lock_time = Some(fallback); - self - } - - // todo generic const - /// Sets the inputs modifiable bit in the transaction modifiable flags. - pub fn inputs_modifiable(mut self) -> Self { - self.inner_psst.global.inputs_modifiable = true; - self - } - // todo generic const - /// Sets the outputs modifiable bit in the transaction modifiable flags. - pub fn outputs_modifiable(mut self) -> Self { - self.inner_psst.global.outputs_modifiable = true; - self - } - - pub fn constructor(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } -} - -impl PSST { - // todo generic const - /// Marks that the `PSST` can not have any more inputs added to it. - pub fn no_more_inputs(mut self) -> Self { - self.inner_psst.global.inputs_modifiable = false; - self - } - // todo generic const - /// Marks that the `PSST` can not have any more outputs added to it. - pub fn no_more_outputs(mut self) -> Self { - self.inner_psst.global.outputs_modifiable = false; - self - } - - /// Adds an input to the PSST. - pub fn input(mut self, input: Input) -> Self { - self.inner_psst.inputs.push(input); - self.inner_psst.global.input_count += 1; - self - } - - /// Adds an output to the PSST. - pub fn output(mut self, output: Output) -> Self { - self.inner_psst.outputs.push(output); - self.inner_psst.global.output_count += 1; - self - } - - /// Returns a PSST [`Updater`] once construction is completed. - pub fn updater(self) -> PSST { - let psst = self.no_more_inputs().no_more_outputs(); - PSST { inner_psst: psst.inner_psst, role: Default::default() } - } - - pub fn signer(self) -> PSST { - self.updater().signer() - } - - pub fn combiner(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } -} - -impl PSST { - pub fn set_sequence(mut self, n: u64, input_index: usize) -> Result { - self.inner_psst.inputs.get_mut(input_index).ok_or(Error::OutOfBounds)?.sequence = Some(n); - Ok(self) - } - - pub fn signer(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } - - pub fn combiner(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } -} - -impl PSST { - // todo use iterator instead of vector - pub fn pass_signature_sync(mut self, sign_fn: SignFn) -> Result - where - E: Display, - SignFn: FnOnce(SignableTransaction, Vec) -> Result, E>, - { - let unsigned_tx = self.unsigned_tx(); - let sighashes = self.inputs.iter().map(|input| input.sighash_type).collect(); - self.inner_psst.inputs.iter_mut().zip(sign_fn(unsigned_tx, sighashes)?).for_each( - |(input, SignInputOk { signature, pub_key, key_source })| { - input.bip32_derivations.insert(pub_key, key_source); - input.partial_sigs.insert(pub_key, signature); - }, - ); - - Ok(self) - } - // todo use iterator instead of vector - pub async fn pass_signature(mut self, sign_fn: SignFn) -> Result - where - E: Display, - Fut: Future, E>>, - SignFn: FnOnce(SignableTransaction, Vec) -> Fut, - { - let unsigned_tx = self.unsigned_tx(); - let sighashes = self.inputs.iter().map(|input| input.sighash_type).collect(); - self.inner_psst.inputs.iter_mut().zip(sign_fn(unsigned_tx, sighashes).await?).for_each( - |(input, SignInputOk { signature, pub_key, key_source })| { - input.bip32_derivations.insert(pub_key, key_source); - input.partial_sigs.insert(pub_key, signature); - }, - ); - Ok(self) - } - - pub fn calculate_id(&self) -> TransactionId { - self.calculate_id_internal() - } - - pub fn finalizer(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } - - pub fn combiner(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SignInputOk { - pub signature: Signature, - pub pub_key: secp256k1::PublicKey, - pub key_source: Option, -} - -impl std::ops::Add> for PSST { - type Output = Result; - - fn add(mut self, mut rhs: PSST) -> Self::Output { - self.inner_psst.global = (self.inner_psst.global + rhs.inner_psst.global)?; - macro_rules! combine { - ($left:expr, $right:expr, $err: ty) => { - if $left.len() > $right.len() { - $left.iter_mut().zip($right.iter_mut()).try_for_each(|(left, right)| -> Result<(), $err> { - *left = (std::mem::take(left) + std::mem::take(right))?; - Ok(()) - })?; - $left - } else { - $right.iter_mut().zip($left.iter_mut()).try_for_each(|(left, right)| -> Result<(), $err> { - *left = (std::mem::take(left) + std::mem::take(right))?; - Ok(()) - })?; - $right - } - }; - } - // todo add sort to build deterministic combination - self.inner_psst.inputs = combine!(self.inner_psst.inputs, rhs.inner_psst.inputs, input::CombineError); - self.inner_psst.outputs = combine!(self.inner_psst.outputs, rhs.inner_psst.outputs, output::CombineError); - Ok(self) - } -} - -impl PSST { - pub fn signer(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } - pub fn finalizer(self) -> PSST { - PSST { inner_psst: self.inner_psst, role: Default::default() } - } -} - -impl PSST { - pub fn finalize_sync( - self, - final_sig_fn: impl FnOnce(&Inner) -> Result>, E>, - ) -> Result> { - let sigs = final_sig_fn(&self); - self.finalize_internal(sigs) - } - - pub async fn finalize(self, final_sig_fn: F) -> Result> - where - E: Display, - F: FnOnce(&Inner) -> Fut, - Fut: Future>, E>>, - { - let sigs = final_sig_fn(&self).await; - self.finalize_internal(sigs) - } - - pub fn id(&self) -> Option { - self.global.id - } - - pub fn extractor(self) -> Result, TxNotFinalized> { - if self.global.id.is_none() { - Err(TxNotFinalized {}) - } else { - Ok(PSST { inner_psst: self.inner_psst, role: Default::default() }) - } - } - - fn finalize_internal(mut self, sigs: Result>, E>) -> Result> { - let sigs = sigs?; - if sigs.len() != self.inputs.len() { - return Err(FinalizeError::WrongFinalizedSigsCount { expected: self.inputs.len(), actual: sigs.len() }); - } - self.inner_psst.inputs.iter_mut().enumerate().zip(sigs).try_for_each(|((idx, input), sig)| { - if sig.is_empty() { - return Err(FinalizeError::EmptySignature(idx)); - } - input.sequence = Some(input.sequence.unwrap_or(u64::MAX)); // todo discussable - input.final_script_sig = Some(sig); - Ok(()) - })?; - self.inner_psst.global.id = Some(self.calculate_id_internal()); - Ok(self) - } -} - -impl PSST { - pub fn extract_tx_unchecked(self) -> Result (Transaction, Vec>), TxNotFinalized> { - let tx = self.unsigned_tx(); - let entries = tx.entries; - let mut tx = tx.tx; - tx.inputs.iter_mut().zip(self.inner_psst.inputs).try_for_each(|(dest, src)| { - dest.signature_script = src.final_script_sig.ok_or(TxNotFinalized {})?; - Ok(()) - })?; - Ok(move |mass| { - tx.set_mass(mass); - (tx, entries) - }) - } - - pub fn extract_tx(self) -> Result (Transaction, Vec>), ExtractError> { - let (tx, entries) = self.extract_tx_unchecked()?(0); - - let tx = MutableTransaction::with_entries(tx, entries.into_iter().flatten().collect()); - use spectre_consensus_core::tx::VerifiableTransaction; - { - let tx = tx.as_verifiable(); - let cache = Cache::new(10_000); - let mut reused_values = SigHashReusedValues::new(); - - tx.populated_inputs().enumerate().try_for_each(|(idx, (input, entry))| { - TxScriptEngine::from_transaction_input(&tx, input, idx, entry, &mut reused_values, &cache)?.execute()?; - >::Ok(()) - })?; - } - let entries = tx.entries; - let tx = tx.tx; - let closure = move |mass| { - tx.set_mass(mass); - (tx, entries) - }; - Ok(closure) - } -} - -/// Error combining psst. -#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -pub enum CombineError { - #[error(transparent)] - Global(#[from] global::CombineError), - #[error(transparent)] - Inputs(#[from] input::CombineError), - #[error(transparent)] - Outputs(#[from] output::CombineError), -} - -#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -pub enum FinalizeError { - #[error("Signatures count mismatch")] - WrongFinalizedSigsCount { expected: usize, actual: usize }, - #[error("Signatures at index: {0} is empty")] - EmptySignature(usize), - #[error(transparent)] - FinalaziCb(#[from] E), -} - -#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -pub enum ExtractError { - #[error(transparent)] - TxScriptError(#[from] spectre_txscript_errors::TxScriptError), - #[error(transparent)] - TxNotFinalized(#[from] TxNotFinalized), -} - -#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] -#[error("Transaction is not finalized")] -pub struct TxNotFinalized {} - -#[cfg(test)] -mod tests { - - // #[test] - // fn it_works() { - // let result = add(2, 2); - // assert_eq!(result, 4); - // } +pub mod prelude { + pub use crate::bundle::Bundle; + pub use crate::bundle::*; + pub use crate::global::Global; + pub use crate::input::Input; + pub use crate::output::Output; + pub use crate::psst::*; + + // not quite sure why it warns of unused imports, + // perhaps due to the fact that enums have no variants? + #[allow(unused_imports)] + pub use crate::role::*; } diff --git a/wallet/psst/src/output.rs b/wallet/psst/src/output.rs index 3aafdb6..f33c953 100644 --- a/wallet/psst/src/output.rs +++ b/wallet/psst/src/output.rs @@ -1,11 +1,12 @@ +use crate::psst::KeySource; use crate::utils::combine_if_no_conflicts; -use crate::KeySource; use derive_builder::Builder; use serde::{Deserialize, Serialize}; use spectre_consensus_core::tx::ScriptPublicKey; use std::{collections::BTreeMap, ops::Add}; #[derive(Builder, Default, Serialize, Deserialize, Clone, Debug)] +#[serde(rename_all = "camelCase")] #[builder(default)] pub struct Output { /// The output's amount (serialized as sompi). diff --git a/wallet/psst/src/psst.rs b/wallet/psst/src/psst.rs new file mode 100644 index 0000000..9eb1c3c --- /dev/null +++ b/wallet/psst/src/psst.rs @@ -0,0 +1,472 @@ +use serde::{Deserialize, Serialize}; +use serde_repr::{Deserialize_repr, Serialize_repr}; +use spectre_bip32::{secp256k1, DerivationPath, KeyFingerprint}; +use std::{collections::BTreeMap, fmt::Display, fmt::Formatter, future::Future, marker::PhantomData, ops::Deref}; + +pub use crate::error::Error; +pub use crate::global::{Global, GlobalBuilder}; +pub use crate::input::{Input, InputBuilder}; +pub use crate::output::{Output, OutputBuilder}; +pub use crate::role::{Combiner, Constructor, Creator, Extractor, Finalizer, Signer, Updater}; +use spectre_consensus_core::tx::UtxoEntry; +use spectre_consensus_core::{ + hashing::{sighash::SigHashReusedValues, sighash_type::SigHashType}, + subnets::SUBNETWORK_ID_NATIVE, + tx::{MutableTransaction, SignableTransaction, Transaction, TransactionId, TransactionInput, TransactionOutput}, +}; +use spectre_txscript::{caches::Cache, TxScriptEngine}; + +#[derive(Debug, Default, Serialize, Deserialize, Clone)] +#[serde(rename_all = "camelCase")] +pub struct Inner { + /// The global map. + pub global: Global, + /// The corresponding key-value map for each input in the unsigned transaction. + pub inputs: Vec, + /// The corresponding key-value map for each output in the unsigned transaction. + pub outputs: Vec, +} + +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, Hash, Serialize_repr, Deserialize_repr)] +#[repr(u8)] +pub enum Version { + #[default] + Zero = 0, +} + +impl Display for Version { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Version::Zero => write!(f, "{}", Version::Zero as u8), + } + } +} + +/// Full information on the used extended public key: fingerprint of the +/// master extended public key and a derivation path from it. +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] +#[serde(rename_all = "camelCase")] +pub struct KeySource { + #[serde(with = "spectre_utils::serde_bytes_fixed")] + pub key_fingerprint: KeyFingerprint, + pub derivation_path: DerivationPath, +} + +impl KeySource { + pub fn new(key_fingerprint: KeyFingerprint, derivation_path: DerivationPath) -> Self { + Self { key_fingerprint, derivation_path } + } +} + +pub type PartialSigs = BTreeMap; + +#[derive(Debug, Serialize, Deserialize, Eq, PartialEq, Copy, Clone)] +#[serde(rename_all = "camelCase")] +pub enum Signature { + ECDSA(secp256k1::ecdsa::Signature), + Schnorr(secp256k1::schnorr::Signature), +} + +impl Signature { + pub fn into_bytes(self) -> [u8; 64] { + match self { + Signature::ECDSA(s) => s.serialize_compact(), + Signature::Schnorr(s) => s.serialize(), + } + } +} + +#[derive(Debug, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct PSST { + #[serde(flatten)] + inner_psst: Inner, + #[serde(skip_serializing, default)] + role: PhantomData, +} + +impl From for PSST { + fn from(inner_psst: Inner) -> Self { + PSST { inner_psst, role: Default::default() } + } +} + +impl Clone for PSST { + fn clone(&self) -> Self { + PSST { inner_psst: self.inner_psst.clone(), role: Default::default() } + } +} + +impl Deref for PSST { + type Target = Inner; + + fn deref(&self) -> &Self::Target { + &self.inner_psst + } +} + +impl PSST { + fn unsigned_tx(&self) -> SignableTransaction { + let tx = Transaction::new( + self.global.tx_version, + self.inputs + .iter() + .map(|Input { previous_outpoint, sequence, sig_op_count, .. }| TransactionInput { + previous_outpoint: *previous_outpoint, + signature_script: vec![], + sequence: sequence.unwrap_or(u64::MAX), + sig_op_count: sig_op_count.unwrap_or(0), + }) + .collect(), + self.outputs + .iter() + .map(|Output { amount, script_public_key, .. }: &Output| TransactionOutput { + value: *amount, + script_public_key: script_public_key.clone(), + }) + .collect(), + self.determine_lock_time(), + SUBNETWORK_ID_NATIVE, + 0, + vec![], + ); + let entries = self.inputs.iter().filter_map(|Input { utxo_entry, .. }| utxo_entry.clone()).collect(); + SignableTransaction::with_entries(tx, entries) + } + + fn calculate_id_internal(&self) -> TransactionId { + self.unsigned_tx().tx.id() + } + + fn determine_lock_time(&self) -> u64 { + self.inputs.iter().map(|input: &Input| input.min_time).max().unwrap_or(self.global.fallback_lock_time).unwrap_or(0) + } + + pub fn to_hex(&self) -> Result { + Ok(format!("PSST{}", hex::encode(serde_json::to_string(self)?))) + } + + pub fn from_hex(hex_data: &str) -> Result { + if let Some(hex_data) = hex_data.strip_prefix("PSST") { + Ok(serde_json::from_slice(hex::decode(hex_data)?.as_slice())?) + } else { + Err(Error::PsstPrefixError) + } + } +} + +impl Default for PSST { + fn default() -> Self { + PSST { inner_psst: Default::default(), role: Default::default() } + } +} + +impl PSST { + /// Sets the fallback lock time. + pub fn fallback_lock_time(mut self, fallback: u64) -> Self { + self.inner_psst.global.fallback_lock_time = Some(fallback); + self + } + + // todo generic const + /// Sets the inputs modifiable bit in the transaction modifiable flags. + pub fn inputs_modifiable(mut self) -> Self { + self.inner_psst.global.inputs_modifiable = true; + self + } + // todo generic const + /// Sets the outputs modifiable bit in the transaction modifiable flags. + pub fn outputs_modifiable(mut self) -> Self { + self.inner_psst.global.outputs_modifiable = true; + self + } + + pub fn constructor(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } +} + +impl PSST { + // todo generic const + /// Marks that the `PSST` can not have any more inputs added to it. + pub fn no_more_inputs(mut self) -> Self { + self.inner_psst.global.inputs_modifiable = false; + self + } + // todo generic const + /// Marks that the `PSST` can not have any more outputs added to it. + pub fn no_more_outputs(mut self) -> Self { + self.inner_psst.global.outputs_modifiable = false; + self + } + + /// Adds an input to the PSST. + pub fn input(mut self, input: Input) -> Self { + self.inner_psst.inputs.push(input); + self.inner_psst.global.input_count += 1; + self + } + + /// Adds an output to the PSST. + pub fn output(mut self, output: Output) -> Self { + self.inner_psst.outputs.push(output); + self.inner_psst.global.output_count += 1; + self + } + + /// Returns a PSST [`Updater`] once construction is completed. + pub fn updater(self) -> PSST { + let psst = self.no_more_inputs().no_more_outputs(); + PSST { inner_psst: psst.inner_psst, role: Default::default() } + } + + pub fn signer(self) -> PSST { + self.updater().signer() + } + + pub fn combiner(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } +} + +impl PSST { + pub fn set_sequence(mut self, n: u64, input_index: usize) -> Result { + self.inner_psst.inputs.get_mut(input_index).ok_or(Error::OutOfBounds)?.sequence = Some(n); + Ok(self) + } + + pub fn signer(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } + + pub fn combiner(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } +} + +impl PSST { + // todo use iterator instead of vector + pub fn pass_signature_sync(mut self, sign_fn: SignFn) -> Result + where + E: Display, + SignFn: FnOnce(SignableTransaction, Vec) -> Result, E>, + { + let unsigned_tx = self.unsigned_tx(); + let sighashes = self.inputs.iter().map(|input| input.sighash_type).collect(); + self.inner_psst.inputs.iter_mut().zip(sign_fn(unsigned_tx, sighashes)?).for_each( + |(input, SignInputOk { signature, pub_key, key_source })| { + input.bip32_derivations.insert(pub_key, key_source); + input.partial_sigs.insert(pub_key, signature); + }, + ); + + Ok(self) + } + // todo use iterator instead of vector + pub async fn pass_signature(mut self, sign_fn: SignFn) -> Result + where + E: Display, + Fut: Future, E>>, + SignFn: FnOnce(SignableTransaction, Vec) -> Fut, + { + let unsigned_tx = self.unsigned_tx(); + let sighashes = self.inputs.iter().map(|input| input.sighash_type).collect(); + self.inner_psst.inputs.iter_mut().zip(sign_fn(unsigned_tx, sighashes).await?).for_each( + |(input, SignInputOk { signature, pub_key, key_source })| { + input.bip32_derivations.insert(pub_key, key_source); + input.partial_sigs.insert(pub_key, signature); + }, + ); + Ok(self) + } + + pub fn calculate_id(&self) -> TransactionId { + self.calculate_id_internal() + } + + pub fn finalizer(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } + + pub fn combiner(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct SignInputOk { + pub signature: Signature, + pub pub_key: secp256k1::PublicKey, + pub key_source: Option, +} + +impl std::ops::Add> for PSST { + type Output = Result; + + fn add(mut self, mut rhs: PSST) -> Self::Output { + self.inner_psst.global = (self.inner_psst.global + rhs.inner_psst.global)?; + macro_rules! combine { + ($left:expr, $right:expr, $err: ty) => { + if $left.len() > $right.len() { + $left.iter_mut().zip($right.iter_mut()).try_for_each(|(left, right)| -> Result<(), $err> { + *left = (std::mem::take(left) + std::mem::take(right))?; + Ok(()) + })?; + $left + } else { + $right.iter_mut().zip($left.iter_mut()).try_for_each(|(left, right)| -> Result<(), $err> { + *left = (std::mem::take(left) + std::mem::take(right))?; + Ok(()) + })?; + $right + } + }; + } + // todo add sort to build deterministic combination + self.inner_psst.inputs = combine!(self.inner_psst.inputs, rhs.inner_psst.inputs, crate::input::CombineError); + self.inner_psst.outputs = combine!(self.inner_psst.outputs, rhs.inner_psst.outputs, crate::output::CombineError); + Ok(self) + } +} + +impl PSST { + pub fn signer(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } + pub fn finalizer(self) -> PSST { + PSST { inner_psst: self.inner_psst, role: Default::default() } + } +} + +impl PSST { + pub fn finalize_sync( + self, + final_sig_fn: impl FnOnce(&Inner) -> Result>, E>, + ) -> Result> { + let sigs = final_sig_fn(&self); + self.finalize_internal(sigs) + } + + pub async fn finalize(self, final_sig_fn: F) -> Result> + where + E: Display, + F: FnOnce(&Inner) -> Fut, + Fut: Future>, E>>, + { + let sigs = final_sig_fn(&self).await; + self.finalize_internal(sigs) + } + + pub fn id(&self) -> Option { + self.global.id + } + + pub fn extractor(self) -> Result, TxNotFinalized> { + if self.global.id.is_none() { + Err(TxNotFinalized {}) + } else { + Ok(PSST { inner_psst: self.inner_psst, role: Default::default() }) + } + } + + fn finalize_internal(mut self, sigs: Result>, E>) -> Result> { + let sigs = sigs?; + if sigs.len() != self.inputs.len() { + return Err(FinalizeError::WrongFinalizedSigsCount { expected: self.inputs.len(), actual: sigs.len() }); + } + self.inner_psst.inputs.iter_mut().enumerate().zip(sigs).try_for_each(|((idx, input), sig)| { + if sig.is_empty() { + return Err(FinalizeError::EmptySignature(idx)); + } + input.sequence = Some(input.sequence.unwrap_or(u64::MAX)); // todo discussable + input.final_script_sig = Some(sig); + Ok(()) + })?; + self.inner_psst.global.id = Some(self.calculate_id_internal()); + Ok(self) + } +} + +impl PSST { + pub fn extract_tx_unchecked(self) -> Result (Transaction, Vec>), TxNotFinalized> { + let tx = self.unsigned_tx(); + let entries = tx.entries; + let mut tx = tx.tx; + tx.inputs.iter_mut().zip(self.inner_psst.inputs).try_for_each(|(dest, src)| { + dest.signature_script = src.final_script_sig.ok_or(TxNotFinalized {})?; + Ok(()) + })?; + Ok(move |mass| { + tx.set_mass(mass); + (tx, entries) + }) + } + + pub fn extract_tx(self) -> Result (Transaction, Vec>), ExtractError> { + let (tx, entries) = self.extract_tx_unchecked()?(0); + + let tx = MutableTransaction::with_entries(tx, entries.into_iter().flatten().collect()); + use spectre_consensus_core::tx::VerifiableTransaction; + { + let tx = tx.as_verifiable(); + let cache = Cache::new(10_000); + let mut reused_values = SigHashReusedValues::new(); + + tx.populated_inputs().enumerate().try_for_each(|(idx, (input, entry))| { + TxScriptEngine::from_transaction_input(&tx, input, idx, entry, &mut reused_values, &cache)?.execute()?; + >::Ok(()) + })?; + } + let entries = tx.entries; + let tx = tx.tx; + let closure = move |mass| { + tx.set_mass(mass); + (tx, entries) + }; + Ok(closure) + } +} + +/// Error combining psst. +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum CombineError { + #[error(transparent)] + Global(#[from] crate::global::CombineError), + #[error(transparent)] + Inputs(#[from] crate::input::CombineError), + #[error(transparent)] + Outputs(#[from] crate::output::CombineError), +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum FinalizeError { + #[error("Signatures count mismatch")] + WrongFinalizedSigsCount { expected: usize, actual: usize }, + #[error("Signatures at index: {0} is empty")] + EmptySignature(usize), + #[error(transparent)] + FinalaziCb(#[from] E), +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +pub enum ExtractError { + #[error(transparent)] + TxScriptError(#[from] spectre_txscript_errors::TxScriptError), + #[error(transparent)] + TxNotFinalized(#[from] TxNotFinalized), +} + +#[derive(thiserror::Error, Debug, Clone, PartialEq, Eq)] +#[error("Transaction is not finalized")] +pub struct TxNotFinalized {} + +#[cfg(test)] +mod tests { + + // #[test] + // fn it_works() { + // let result = add(2, 2); + // assert_eq!(result, 4); + // } +} diff --git a/wallet/psst/src/wasm/bundle.rs b/wallet/psst/src/wasm/bundle.rs new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/wallet/psst/src/wasm/bundle.rs @@ -0,0 +1 @@ + diff --git a/wallet/psst/src/wasm/error.rs b/wallet/psst/src/wasm/error.rs new file mode 100644 index 0000000..519e1df --- /dev/null +++ b/wallet/psst/src/wasm/error.rs @@ -0,0 +1,64 @@ +use super::psst::State; +use thiserror::Error; +use wasm_bindgen::prelude::*; + +#[derive(Error, Debug)] +pub enum Error { + #[error("{0}")] + Custom(String), + + #[error("Unexpected state: {0}")] + State(String), + + #[error("Constructor argument must be a valid payload, another PSST instance, Transaction or undefined")] + Ctor(String), + + #[error("Invalid payload")] + InvalidPayload, + + #[error("Transaction not finalized")] + TxNotFinalized(#[from] crate::psst::TxNotFinalized), + + #[error(transparent)] + Wasm(#[from] workflow_wasm::error::Error), + + #[error("Create state is not allowed for PSST initialized from transaction or a payload")] + CreateNotAllowed, + + #[error("PSST must be initialized with a payload or CREATE role")] + NotInitialized, + + #[error(transparent)] + ConsensusClient(#[from] spectre_consensus_client::error::Error), + + #[error(transparent)] + Psst(#[from] crate::error::Error), +} + +impl Error { + pub fn custom(msg: T) -> Self { + Error::Custom(msg.to_string()) + } + + pub fn state(state: impl AsRef) -> Self { + Error::State(state.as_ref().display().to_string()) + } +} + +impl From<&str> for Error { + fn from(msg: &str) -> Self { + Error::Custom(msg.to_string()) + } +} + +impl From for Error { + fn from(msg: String) -> Self { + Error::Custom(msg) + } +} + +impl From for JsValue { + fn from(err: Error) -> Self { + JsValue::from_str(&err.to_string()) + } +} diff --git a/wallet/psst/src/wasm/input.rs b/wallet/psst/src/wasm/input.rs new file mode 100644 index 0000000..b6a827d --- /dev/null +++ b/wallet/psst/src/wasm/input.rs @@ -0,0 +1 @@ +// TODO - InputBuilder & Input diff --git a/wallet/psst/src/wasm/mod.rs b/wallet/psst/src/wasm/mod.rs new file mode 100644 index 0000000..29e5e0c --- /dev/null +++ b/wallet/psst/src/wasm/mod.rs @@ -0,0 +1,6 @@ +pub mod bundle; +pub mod error; +pub mod input; +pub mod output; +pub mod psst; +pub mod result; diff --git a/wallet/psst/src/wasm/output.rs b/wallet/psst/src/wasm/output.rs new file mode 100644 index 0000000..eb91824 --- /dev/null +++ b/wallet/psst/src/wasm/output.rs @@ -0,0 +1 @@ +// TODO - OutputBuilder & Output diff --git a/wallet/psst/src/wasm/psst.rs b/wallet/psst/src/wasm/psst.rs new file mode 100644 index 0000000..139dcae --- /dev/null +++ b/wallet/psst/src/wasm/psst.rs @@ -0,0 +1,320 @@ +use crate::psst::PSST as Native; +use crate::role::*; +use spectre_consensus_core::tx::TransactionId; +use wasm_bindgen::prelude::*; +// use js_sys::Object; +use crate::psst::Inner; +use serde::{Deserialize, Serialize}; +use spectre_consensus_client::{Transaction, TransactionInput, TransactionInputT, TransactionOutput, TransactionOutputT}; +use std::sync::MutexGuard; +use std::sync::{Arc, Mutex}; +use workflow_wasm::{ + convert::{Cast, CastFromJs, TryCastFromJs}, + // extensions::object::*, + // error::Error as CastError, +}; + +use super::error::*; +use super::result::*; + +#[derive(Clone, Serialize, Deserialize)] +#[serde(tag = "state", content = "payload")] +pub enum State { + NoOp(Option), + Creator(Native), + Constructor(Native), + Updater(Native), + Signer(Native), + Combiner(Native), + Finalizer(Native), + Extractor(Native), +} + +impl AsRef for State { + fn as_ref(&self) -> &State { + self + } +} + +impl State { + // this is not a Display trait intentionally + pub fn display(&self) -> &'static str { + match self { + State::NoOp(_) => "Init", + State::Creator(_) => "Creator", + State::Constructor(_) => "Constructor", + State::Updater(_) => "Updater", + State::Signer(_) => "Signer", + State::Combiner(_) => "Combiner", + State::Finalizer(_) => "Finalizer", + State::Extractor(_) => "Extractor", + } + } +} + +impl From for PSST { + fn from(state: State) -> Self { + PSST { state: Arc::new(Mutex::new(Some(state))) } + } +} + +#[wasm_bindgen] +extern "C" { + #[wasm_bindgen(typescript_type = "PSST | Transaction | string | undefined")] + pub type CtorT; +} + +#[derive(Clone, Serialize, Deserialize)] +pub struct Payload { + data: String, +} + +impl TryFrom for Native { + type Error = Error; + + fn try_from(value: Payload) -> Result { + let Payload { data } = value; + if data.starts_with("PSST") { + unimplemented!("PSST binary serialization") + } else { + Ok(serde_json::from_str(&data).map_err(|err| format!("Invalid JSON: {err}"))?) + } + } +} + +#[wasm_bindgen(inspectable)] +#[derive(Clone, CastFromJs)] +pub struct PSST { + state: Arc>>, +} + +impl TryCastFromJs for PSST { + type Error = Error; + fn try_cast_from<'a, R>(value: &'a R) -> std::result::Result, Self::Error> + where + R: AsRef + 'a, + { + Self::resolve(value, || { + if let Some(data) = value.as_ref().as_string() { + let psst_inner: Inner = serde_json::from_str(&data).map_err(|_| Error::InvalidPayload)?; + Ok(PSST::from(State::NoOp(Some(psst_inner)))) + } else if let Ok(transaction) = Transaction::try_owned_from(value) { + let psst_inner: Inner = transaction.try_into()?; + Ok(PSST::from(State::NoOp(Some(psst_inner)))) + } else { + Err(Error::InvalidPayload) + } + }) + } +} + +#[wasm_bindgen] +impl PSST { + #[wasm_bindgen(constructor)] + pub fn new(payload: CtorT) -> Result { + PSST::try_owned_from(payload.unchecked_into::().as_ref()).map_err(|err| Error::Ctor(err.to_string())) + } + + #[wasm_bindgen(getter, js_name = "role")] + pub fn role_getter(&self) -> String { + self.state().as_ref().unwrap().display().to_string() + } + + #[wasm_bindgen(getter, js_name = "payload")] + pub fn payload_getter(&self) -> JsValue { + let state = self.state(); + serde_wasm_bindgen::to_value(state.as_ref().unwrap()).unwrap() + } + + fn state(&self) -> MutexGuard> { + self.state.lock().unwrap() + } + + fn take(&self) -> State { + self.state.lock().unwrap().take().unwrap() + } + + fn replace(&self, state: State) -> Result { + self.state.lock().unwrap().replace(state); + Ok(self.clone()) + } + + /// Change role to `CREATOR` + /// #[wasm_bindgen(js_name = toCreator)] + pub fn creator(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => match inner { + None => State::Creator(Native::default()), + Some(_) => Err(Error::CreateNotAllowed)?, + }, + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `CONSTRUCTOR` + #[wasm_bindgen(js_name = toConstructor)] + pub fn constructor(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Constructor(inner.ok_or(Error::NotInitialized)?.into()), + State::Creator(psst) => State::Constructor(psst.constructor()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `UPDATER` + #[wasm_bindgen(js_name = toUpdater)] + pub fn updater(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Updater(inner.ok_or(Error::NotInitialized)?.into()), + State::Constructor(constructor) => State::Updater(constructor.updater()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `SIGNER` + #[wasm_bindgen(js_name = toSigner)] + pub fn signer(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Signer(inner.ok_or(Error::NotInitialized)?.into()), + State::Constructor(psst) => State::Signer(psst.signer()), + State::Updater(psst) => State::Signer(psst.signer()), + State::Combiner(psst) => State::Signer(psst.signer()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `COMBINER` + #[wasm_bindgen(js_name = toCombiner)] + pub fn combiner(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Combiner(inner.ok_or(Error::NotInitialized)?.into()), + State::Constructor(psst) => State::Combiner(psst.combiner()), + State::Updater(psst) => State::Combiner(psst.combiner()), + State::Signer(psst) => State::Combiner(psst.combiner()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `FINALIZER` + #[wasm_bindgen(js_name = toFinalizer)] + pub fn finalizer(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Finalizer(inner.ok_or(Error::NotInitialized)?.into()), + State::Combiner(psst) => State::Finalizer(psst.finalizer()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + /// Change role to `EXTRACTOR` + #[wasm_bindgen(js_name = toExtractor)] + pub fn extractor(&self) -> Result { + let state = match self.take() { + State::NoOp(inner) => State::Extractor(inner.ok_or(Error::NotInitialized)?.into()), + State::Finalizer(psst) => State::Extractor(psst.extractor()?), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = fallbackLockTime)] + pub fn fallback_lock_time(&self, lock_time: u64) -> Result { + let state = match self.take() { + State::Creator(psst) => State::Creator(psst.fallback_lock_time(lock_time)), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = inputsModifiable)] + pub fn inputs_modifiable(&self) -> Result { + let state = match self.take() { + State::Creator(psst) => State::Creator(psst.inputs_modifiable()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = outputsModifiable)] + pub fn outputs_modifiable(&self) -> Result { + let state = match self.take() { + State::Creator(psst) => State::Creator(psst.outputs_modifiable()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = noMoreInputs)] + pub fn no_more_inputs(&self) -> Result { + let state = match self.take() { + State::Constructor(psst) => State::Constructor(psst.no_more_inputs()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = noMoreOutputs)] + pub fn no_more_outputs(&self) -> Result { + let state = match self.take() { + State::Constructor(psst) => State::Constructor(psst.no_more_outputs()), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + pub fn input(&self, input: &TransactionInputT) -> Result { + let input = TransactionInput::try_owned_from(input)?; + let state = match self.take() { + State::Constructor(psst) => State::Constructor(psst.input(input.try_into()?)), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + pub fn output(&self, output: &TransactionOutputT) -> Result { + let output = TransactionOutput::try_owned_from(output)?; + let state = match self.take() { + State::Constructor(psst) => State::Constructor(psst.output(output.try_into()?)), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = setSequence)] + pub fn set_sequence(&self, n: u64, input_index: usize) -> Result { + let state = match self.take() { + State::Updater(psst) => State::Updater(psst.set_sequence(n, input_index)?), + state => Err(Error::state(state))?, + }; + + self.replace(state) + } + + #[wasm_bindgen(js_name = calculateId)] + pub fn calculate_id(&self) -> Result { + let state = self.state(); + match state.as_ref().unwrap() { + State::Signer(psst) => Ok(psst.calculate_id()), + state => Err(Error::state(state))?, + } + } +} diff --git a/wallet/psst/src/wasm/result.rs b/wallet/psst/src/wasm/result.rs new file mode 100644 index 0000000..32f6633 --- /dev/null +++ b/wallet/psst/src/wasm/result.rs @@ -0,0 +1 @@ +pub type Result = std::result::Result; diff --git a/wasm/CHANGELOG.md b/wasm/CHANGELOG.md index 6626ef1..7a4a834 100644 --- a/wasm/CHANGELOG.md +++ b/wasm/CHANGELOG.md @@ -1,3 +1,22 @@ +### Latest Release + +- Support numeric interface (IP) argument without port in `--rpclisten-borsh` or `--rpclisten-json` +- Replace `MassCalculator` with `calculateTransactionMass` and `calculateTransactionFee` functions. +- Change `createTransaction` function signature (remove requirement for change address). +- Make `ITransactionInput.signatureScript` optional (if not supplied, the signatureScript is assigned an empty vector). + +- Fix issues with deserializing manually-created objects matching `IUtxoEntry` interface. +- Allow arguments expecting ScriptPublicKey to receive `{ version, script }` object or a hex string. +- Fix `Transaction::serializeToObject()` return type (now returning `ISerializeTransaction` interface). +- Adding `setUserTransactionMaturityDAA()` and `setCoinbaseTransactionMaturityDAA()` that allow customizing + the maturity DAA periods for user and coinbase transactions. + +- Fix `PublicKeyGenerator::change_address_as_string()` that was returning the receive address. +- WASM SDK now builds as a GitHub artifact during the CI process. +- `State` renamed to `PoW` +- Docs now have a PoW section that unifies all PoW-related classes and functions. +- `TransactionRecord.data` (`TransactionData`) now has correct TypeScript bindings. + ### Release 2024-05-24 - First version with Spectre Network support diff --git a/wasm/Cargo.toml b/wasm/Cargo.toml index c3cad96..a793092 100644 --- a/wasm/Cargo.toml +++ b/wasm/Cargo.toml @@ -18,11 +18,13 @@ crate-type = ["cdylib"] cfg-if.workspace = true js-sys.workspace = true spectre-addresses.workspace = true +spectre-bip32.workspace = true spectre-consensus-core.workspace = true spectre-consensus-wasm.workspace = true spectre-core.workspace = true spectre-math.workspace = true spectre-pow.workspace = true +spectre-txscript.workspace = true spectre-rpc-core.workspace = true spectre-utils.workspace = true spectre-wasm-core.workspace = true @@ -40,10 +42,12 @@ workflow-wasm.workspace = true wasm32-sdk = [ "spectre-wallet-core/wasm32-sdk", "spectre-pow/wasm32-sdk", + "spectre-txscript/wasm32-sdk", ] wasm32-core = [ "spectre-wallet-core/wasm32-core", "spectre-pow/wasm32-sdk", + "spectre-txscript/wasm32-sdk", ] wasm32-rpc = [ "spectre-consensus-core/wasm32-sdk", diff --git a/wasm/README.md b/wasm/README.md index 88bf8a5..6dd8b5b 100644 --- a/wasm/README.md +++ b/wasm/README.md @@ -12,7 +12,7 @@ codebase within JavaScript and TypeScript environments such as Node.js and Web B As of now the code is compatible with Kaspa and its documentation can be used from the official links. Please note that while WASM directly binds JavaScript and Rust resources, their names on JavaScript side -are different from their name in Rust as they conform to the 'camelCase' convention in JavaScript and +are different from their name in Rust as they conform to the 'camelCase' convention in JavaScript and to the 'snake_case' convention in Rust. The WASM32 bindings can be used in both TypeScript and JavaScript environments, where in JavaScript @@ -29,11 +29,12 @@ The SDK is currently separated into the following top-level categories: ## WASM32 SDK release packages The SDK is built as 4 packages for Web Browsers as follows: + - KeyGen - Key & Address Generation only - RPC - RPC only - Core - RPC + Key & Address Generation + Wallet SDK - Full - Full SDK + Integrated Wallet -For NodeJS, the SDK is built as a single package containing all features. + For NodeJS, the SDK is built as a single package containing all features. ## SDK folder structure @@ -51,11 +52,11 @@ The following is a brief overview of the SDK folder structure (as available in t - `examples/javascript/wallet` - Interfacing with the Rusty Spectre Wallet framework. - `examples/typescript` - TypeScript examples. -If you are using JavaScript and Visual Studio Code, it is highly recommended you replicate -the `jsconfig.json` configuration file as is done in the SDK examples. This file allows +If you are using JavaScript and Visual Studio Code, it is highly recommended you replicate +the `jsconfig.json` configuration file as is done in the SDK examples. This file allows Visual Studio to provide TypeScript-like code completion, type checking and documentation. -Included documentation in the release can be accessed by loading the `docs/spectre/index.html` +Included documentation in the release can be accessed by loading the `docs/spectre/index.html` file in a web browser. ## Building from Source @@ -80,12 +81,14 @@ serve them from the root of the SDK folder (`spectre-wasm32-sdk` if using a redi WASM32 currently can not be loaded using the `file://` protocol. You can use any web server of your choice. If you don't have one, you can run one as follows: + ```bash cargo install http-server http-server ``` -Access the examples at [http://localhost:7878/examples/web/index.html](http://localhost:7878/examples/web/index.html). -(Make sure to change the port if you are using a different server. Many servers will serve on + +Access the examples at [http://localhost:7878/examples/web/index.html](http://localhost:7878/examples/web/index.html). +(Make sure to change the port if you are using a different server. Many servers will serve on [http://localhost:8000/examples/web/index.html](http://localhost:8000/examples/web/index.html) by default) If building from source, you must run `build-release` or `build-web` scripts before running the examples. @@ -95,14 +98,16 @@ If building from source, you must run `build-release` or `build-web` scripts bef This applies to running examples while building the project from source as some dependencies are instantiated as a part of the build process. You just need to run `node init` to initialize a local config. NOTES: + - `npm install` will install NodeJs types for TypeScript and W3C websocket modules - `npm install -g typedoc` is needed for the release build to generate documentation - `node init` creates a local `examples/data/config.json` that contains a private key (mnemonic) use across NodeJS examples. You can override address used in some examples by specifying the address as a command line argument. - Majority of examples will accept following arguments: `node