From beb28c7a6c158847620c29456875a936b27be6b2 Mon Sep 17 00:00:00 2001 From: arya2 Date: Fri, 11 Nov 2022 17:23:21 -0500 Subject: [PATCH 01/11] adds new zebra_state::ReadRequest variant --- zebra-state/src/request.rs | 8 ++++++++ zebra-state/src/response.rs | 6 +++++- zebra-state/src/service.rs | 37 ++++++++++++++++++++++++++++++------- 3 files changed, 43 insertions(+), 8 deletions(-) diff --git a/zebra-state/src/request.rs b/zebra-state/src/request.rs index 151bb769c39..76fc18c2180 100644 --- a/zebra-state/src/request.rs +++ b/zebra-state/src/request.rs @@ -744,6 +744,12 @@ pub enum ReadRequest { /// * [`ReadResponse::BlockHash(Some(hash))`](ReadResponse::BlockHash) if the block is in the best chain; /// * [`ReadResponse::BlockHash(None)`](ReadResponse::BlockHash) otherwise. BestChainBlockHash(block::Height), + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Performs contextual validation of the given block + /// + /// Returns [`ReadResponse::Validated`] when successful or a validation error + CheckContextualValidity(PreparedBlock), } impl ReadRequest { @@ -766,6 +772,8 @@ impl ReadRequest { ReadRequest::UtxosByAddresses(_) => "utxos_by_addesses", #[cfg(feature = "getblocktemplate-rpcs")] ReadRequest::BestChainBlockHash(_) => "best_chain_block_hash", + #[cfg(feature = "getblocktemplate-rpcs")] + ReadRequest::CheckContextualValidity(_) => "check_contextual_validity", } } diff --git a/zebra-state/src/response.rs b/zebra-state/src/response.rs index 71d9ff10576..27476afb754 100644 --- a/zebra-state/src/response.rs +++ b/zebra-state/src/response.rs @@ -115,6 +115,10 @@ pub enum ReadResponse { /// Response to [`ReadRequest::BestChainBlockHash`](crate::ReadRequest::BestChainBlockHash) with the /// specified block hash. BlockHash(Option), + + #[cfg(feature = "getblocktemplate-rpcs")] + /// Response to [`ReadRequest::CheckContextualValidity`] + Validated, } /// Conversion from read-only [`ReadResponse`]s to read-write [`Response`]s. @@ -152,7 +156,7 @@ impl TryFrom for Response { } #[cfg(feature = "getblocktemplate-rpcs")] - ReadResponse::BlockHash(_) => { + ReadResponse::BlockHash(_) | ReadResponse::Validated => { Err("there is no corresponding Response for this ReadResponse") } } diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 0f45aa0f448..90de33fe55a 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1522,13 +1522,6 @@ impl Service for ReadStateService { // Used by get_block_hash RPC. #[cfg(feature = "getblocktemplate-rpcs")] ReadRequest::BestChainBlockHash(height) => { - metrics::counter!( - "state.requests", - 1, - "service" => "read_state", - "type" => "best_chain_block_hash", - ); - let timer = CodeTimer::start(); let state = self.clone(); @@ -1554,6 +1547,36 @@ impl Service for ReadStateService { .map(|join_result| join_result.expect("panic in ReadRequest::BestChainBlockHash")) .boxed() } + + // Used by get_block_template RPC. + #[cfg(feature = "getblocktemplate-rpcs")] + ReadRequest::CheckContextualValidity(_block) => { + let timer = CodeTimer::start(); + + let state = self.clone(); + + let span = Span::current(); + tokio::task::spawn_blocking(move || { + span.in_scope(move || { + let _best_chain = state.non_finalized_state_receiver.with_watch_data( + |non_finalized_state| non_finalized_state.best_chain().map(Arc::clone), + ); + + // The work is done in the future. + timer.finish( + module_path!(), + line!(), + "ReadRequest::CheckContextualValidity", + ); + + Ok(ReadResponse::Validated) + }) + }) + .map(|join_result| { + join_result.expect("panic in ReadRequest::CheckContextualValidity") + }) + .boxed() + } } } } From 20669907a3c967c2f0b3ed167dc6d08174e26beb Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 14 Nov 2022 14:24:20 -0500 Subject: [PATCH 02/11] adds call to check::initial_contextual_validity --- zebra-state/src/service.rs | 16 ++++++++++++---- zebra-state/src/service/check.rs | 18 ++++++++---------- zebra-state/src/service/write.rs | 7 ++++++- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 90de33fe55a..676758c7161 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1550,7 +1550,7 @@ impl Service for ReadStateService { // Used by get_block_template RPC. #[cfg(feature = "getblocktemplate-rpcs")] - ReadRequest::CheckContextualValidity(_block) => { + ReadRequest::CheckContextualValidity(block) => { let timer = CodeTimer::start(); let state = self.clone(); @@ -1558,9 +1558,17 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { - let _best_chain = state.non_finalized_state_receiver.with_watch_data( - |non_finalized_state| non_finalized_state.best_chain().map(Arc::clone), - ); + let latest_non_finalized_state = state.latest_non_finalized_state(); + if let Some(best_chain) = + latest_non_finalized_state.best_chain().map(Arc::clone) + { + check::initial_contextual_validity( + state.network, + &state.db, + &latest_non_finalized_state, + &block, + )?; + }; // The work is done in the future. timer.finish( diff --git a/zebra-state/src/service/check.rs b/zebra-state/src/service/check.rs index e540941f33d..078d196efc7 100644 --- a/zebra-state/src/service/check.rs +++ b/zebra-state/src/service/check.rs @@ -13,15 +13,12 @@ use zebra_chain::{ }; use crate::{ - service::{ - block_iter::any_ancestor_blocks, finalized_state::FinalizedState, - non_finalized_state::NonFinalizedState, - }, + service::{block_iter::any_ancestor_blocks, non_finalized_state::NonFinalizedState}, BoxError, PreparedBlock, ValidateContextError, }; // use self as check -use super::check; +use super::{check, finalized_state::ZebraDb}; // These types are used in doc links #[allow(unused_imports)] @@ -369,25 +366,26 @@ where /// /// Additional contextual validity checks are performed by the non-finalized [`Chain`]. pub(crate) fn initial_contextual_validity( - finalized_state: &FinalizedState, + network: Network, + db: &ZebraDb, non_finalized_state: &NonFinalizedState, prepared: &PreparedBlock, ) -> Result<(), ValidateContextError> { let relevant_chain = any_ancestor_blocks( non_finalized_state, - &finalized_state.db, + db, prepared.block.header.previous_block_hash, ); // Security: check proof of work before any other checks check::block_is_valid_for_recent_chain( prepared, - finalized_state.network(), - finalized_state.db.finalized_tip_height(), + network, + db.finalized_tip_height(), relevant_chain, )?; - check::nullifier::no_duplicates_in_finalized_chain(prepared, &finalized_state.db)?; + check::nullifier::no_duplicates_in_finalized_chain(prepared, db)?; Ok(()) } diff --git a/zebra-state/src/service/write.rs b/zebra-state/src/service/write.rs index 5ba92218431..a3226081c71 100644 --- a/zebra-state/src/service/write.rs +++ b/zebra-state/src/service/write.rs @@ -40,7 +40,12 @@ pub(crate) fn validate_and_commit_non_finalized( non_finalized_state: &mut NonFinalizedState, prepared: PreparedBlock, ) -> Result<(), CommitBlockError> { - check::initial_contextual_validity(finalized_state, non_finalized_state, &prepared)?; + check::initial_contextual_validity( + finalized_state.network(), + &finalized_state.db, + non_finalized_state, + &prepared, + )?; let parent_hash = prepared.block.header.previous_block_hash; if finalized_state.db.finalized_tip_hash() == parent_hash { From 775d6e78a64a0a8e3c4311a77208fb2b369c07bb Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 14 Nov 2022 15:14:37 -0500 Subject: [PATCH 03/11] adds new_contextually_valid_block method to chain --- zebra-state/src/service.rs | 30 +++++++++++++ .../src/service/non_finalized_state.rs | 31 +------------ .../src/service/non_finalized_state/chain.rs | 45 ++++++++++++++++++- 3 files changed, 74 insertions(+), 32 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 676758c7161..3cea347a32c 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -1568,6 +1568,36 @@ impl Service for ReadStateService { &latest_non_finalized_state, &block, )?; + + // Reads from disk + let sprout_final_treestates = + check::anchors::fetch_sprout_final_treestates( + &state.db, + &best_chain, + &block, + ); + + let contextual = + best_chain.new_contextually_valid_block(block, &state.db)?; + + check::anchors::sprout_anchors_refer_to_treestates( + sprout_final_treestates, + Arc::clone(&contextual.block), + contextual.height, + contextual.transaction_hashes, + )?; + + check::block_commitment_is_valid_for_chain_history( + contextual.block, + state.network, + &best_chain.history_tree, + )?; + } else { + check::block_commitment_is_valid_for_chain_history( + block.block, + state.network, + &state.db.history_tree(), + )?; }; // The work is done in the future. diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index 0c9b23a5170..dbb1a58b112 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -215,41 +215,12 @@ impl NonFinalizedState { prepared: PreparedBlock, finalized_state: &ZebraDb, ) -> Result, ValidateContextError> { - // Reads from disk - // - // TODO: if these disk reads show up in profiles, run them in parallel, using std::thread::spawn() - let spent_utxos = check::utxo::transparent_spend( - &prepared, - &new_chain.unspent_utxos(), - &new_chain.spent_utxos, - finalized_state, - )?; - - // Reads from disk - check::anchors::sapling_orchard_anchors_refer_to_final_treestates( - finalized_state, - &new_chain, - &prepared, - )?; - // Reads from disk let sprout_final_treestates = check::anchors::fetch_sprout_final_treestates(finalized_state, &new_chain, &prepared); // Quick check that doesn't read from disk - let contextual = ContextuallyValidBlock::with_block_and_spent_utxos( - prepared.clone(), - spent_utxos.clone(), - ) - .map_err(|value_balance_error| { - ValidateContextError::CalculateBlockChainValueChange { - value_balance_error, - height: prepared.height, - block_hash: prepared.hash, - transaction_count: prepared.block.transactions.len(), - spent_utxo_count: spent_utxos.len(), - } - })?; + let contextual = new_chain.new_contextually_valid_block(prepared, finalized_state)?; Self::validate_and_update_parallel(new_chain, contextual, sprout_final_treestates) } diff --git a/zebra-state/src/service/non_finalized_state/chain.rs b/zebra-state/src/service/non_finalized_state/chain.rs index 1c91f9c42e3..20d4e304b5a 100644 --- a/zebra-state/src/service/non_finalized_state/chain.rs +++ b/zebra-state/src/service/non_finalized_state/chain.rs @@ -30,8 +30,10 @@ use zebra_chain::{ }; use crate::{ - request::Treestate, service::check, ContextuallyValidBlock, HashOrHeight, OutputLocation, - TransactionLocation, ValidateContextError, + request::Treestate, + service::{check, finalized_state::ZebraDb}, + ContextuallyValidBlock, HashOrHeight, OutputLocation, PreparedBlock, TransactionLocation, + ValidateContextError, }; use self::index::TransparentTransfers; @@ -435,6 +437,45 @@ impl Chain { self.blocks.get(&height) } + /// Contextually validate `prepared` using `finalized_state`. + /// + /// Returns new contextually valid block + #[tracing::instrument(level = "debug", skip(self, finalized_state))] + pub fn new_contextually_valid_block( + &self, + prepared: PreparedBlock, + finalized_state: &ZebraDb, + ) -> Result { + // Reads from disk + // + // TODO: if these disk reads show up in profiles, run them in parallel, using std::thread::spawn() + let spent_utxos = check::utxo::transparent_spend( + &prepared, + &self.unspent_utxos(), + &self.spent_utxos, + finalized_state, + )?; + + // Reads from disk + check::anchors::sapling_orchard_anchors_refer_to_final_treestates( + finalized_state, + self, + &prepared, + )?; + + // Quick check that doesn't read from disk + ContextuallyValidBlock::with_block_and_spent_utxos(prepared.clone(), spent_utxos.clone()) + .map_err( + |value_balance_error| ValidateContextError::CalculateBlockChainValueChange { + value_balance_error, + height: prepared.height, + block_hash: prepared.hash, + transaction_count: prepared.block.transactions.len(), + spent_utxo_count: spent_utxos.len(), + }, + ) + } + /// Returns the [`Transaction`] with [`transaction::Hash`], if it exists in this chain. pub fn transaction( &self, From 2f5a7dddc8c800933f5a3f57816947db06a0be83 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 14 Nov 2022 18:49:50 -0500 Subject: [PATCH 04/11] adds make_test_prepared_block fn --- zebra-chain/src/work/equihash.rs | 7 +++ .../src/methods/get_block_template_rpcs.rs | 57 ++++++++++++++++++- .../types/get_block_template.rs | 37 +++++++++++- 3 files changed, 98 insertions(+), 3 deletions(-) diff --git a/zebra-chain/src/work/equihash.rs b/zebra-chain/src/work/equihash.rs index 230ba6d5a94..956b75b66b2 100644 --- a/zebra-chain/src/work/equihash.rs +++ b/zebra-chain/src/work/equihash.rs @@ -32,6 +32,13 @@ pub(crate) const SOLUTION_SIZE: usize = 1344; #[derive(Deserialize, Serialize)] pub struct Solution(#[serde(with = "BigArray")] pub [u8; SOLUTION_SIZE]); +#[cfg(feature = "getblocktemplate-rpcs")] +impl Default for Solution { + fn default() -> Self { + Self([0; SOLUTION_SIZE]) + } +} + impl Solution { /// The length of the portion of the header used as input when verifying /// equihash solutions, in bytes. diff --git a/zebra-rpc/src/methods/get_block_template_rpcs.rs b/zebra-rpc/src/methods/get_block_template_rpcs.rs index eda2cbc3026..afc6a6e32c1 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs.rs @@ -268,6 +268,7 @@ where let network = self.network; let miner_address = self.miner_address; + let mut state = self.state.clone(); let mempool = self.mempool.clone(); let latest_chain_tip = self.latest_chain_tip.clone(); @@ -294,11 +295,18 @@ where let (merkle_root, auth_data_root) = calculate_transaction_roots(&coinbase_tx, &mempool_txs); + let prepared_block_txs: Vec<_> = mempool_txs + .iter() + .map(|tx| tx.transaction.transaction.clone()) + .chain(iter::once(coinbase_tx.transaction.clone())) + .collect(); + // Convert into TransactionTemplates let mempool_txs = mempool_txs.iter().map(Into::into).collect(); let empty_string = String::from(""); - Ok(GetBlockTemplate { + + let get_block_template = GetBlockTemplate { capabilities: vec![], version: ZCASH_BLOCK_VERSION, @@ -338,7 +346,28 @@ where bits: empty_string, height: 0, - }) + }; + + let request = zebra_state::ReadRequest::CheckContextualValidity( + make_test_prepared_block(&get_block_template, prepared_block_txs), + ); + + let response = state + .ready() + .and_then(|service| service.call(request)) + .await + .map_err(|error| Error { + code: ErrorCode::ServerError(0), + message: error.to_string(), + data: None, + })?; + + match response { + zebra_state::ReadResponse::Validated => Ok(()), + _ => unreachable!("unmatched response to a CheckContextualValidity request"), + }?; + + Ok(get_block_template) } .boxed() } @@ -558,3 +587,27 @@ fn get_height_from_int(index: i32, tip_height: Height) -> Result { Ok(Height(sanitized_height)) } } + +/// Make a test PreparedBlock the getblocktemplate data to contextually validate +fn make_test_prepared_block( + get_block_template: &GetBlockTemplate, + transactions: Vec>, +) -> zebra_state::PreparedBlock { + let block = Arc::new(Block { + header: Arc::new(get_block_template.into()), + transactions, + }); + + let hash = block.hash(); + let height = Height(get_block_template.height); + let transaction_hashes: Arc<[_]> = block.transactions.iter().map(|t| t.hash()).collect(); + let new_outputs = transparent::new_ordered_outputs(&block, &transaction_hashes); + + zebra_state::PreparedBlock { + block, + hash, + height, + new_outputs, + transaction_hashes, + } +} diff --git a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs index 63840d15a47..333c324e8dc 100644 --- a/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs +++ b/zebra-rpc/src/methods/get_block_template_rpcs/types/get_block_template.rs @@ -1,6 +1,12 @@ //! The `GetBlockTempate` type is the output of the `getblocktemplate` RPC method. -use zebra_chain::{amount, block::ChainHistoryBlockTxAuthCommitmentHash}; +use chrono::Utc; +use zebra_chain::{ + amount, + block::{ChainHistoryBlockTxAuthCommitmentHash, Header}, + parameters::Network, + work::{difficulty::ExpandedDifficulty, equihash::Solution}, +}; use crate::methods::{ get_block_template_rpcs::types::{ @@ -102,3 +108,32 @@ pub struct GetBlockTemplate { // TODO: use Height type? pub height: u32, } + +impl From<&GetBlockTemplate> for Header { + fn from(get_block_template: &GetBlockTemplate) -> Self { + let &GetBlockTemplate { + version, + previous_block_hash: GetBlockHash(previous_block_hash), + default_roots: + DefaultRoots { + merkle_root, + block_commitments_hash, + .. + }, + .. + } = get_block_template; + + Self { + version, + previous_block_hash, + merkle_root, + commitment_bytes: block_commitments_hash.into(), + time: Utc::now(), + // TODO: get difficulty from target once it uses ExpandedDifficulty type + difficulty_threshold: ExpandedDifficulty::target_difficulty_limit(Network::Mainnet) + .to_compact(), + nonce: [0; 32], + solution: Solution::default(), + } + } +} From bfd4c0d34102d0d35ebf125d61c39660fe257019 Mon Sep 17 00:00:00 2001 From: arya2 Date: Mon, 14 Nov 2022 19:11:01 -0500 Subject: [PATCH 05/11] moves CheckContextualValidity logic into a method --- zebra-state/src/service.rs | 87 ++++++++++++++++++++------------------ 1 file changed, 45 insertions(+), 42 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 3cea347a32c..ac5adb6e915 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -797,6 +797,50 @@ impl ReadStateService { fn latest_non_finalized_state(&self) -> NonFinalizedState { self.non_finalized_state_receiver.cloned_watch_data() } + + /// Check the contextual validity of a block in the best chain + #[cfg(feature = "getblocktemplate-rpcs")] + fn check_best_chain_contextual_validity( + &self, + block: PreparedBlock, + ) -> Result<(), crate::ValidateContextError> { + let latest_non_finalized_state = self.latest_non_finalized_state(); + if let Some(best_chain) = latest_non_finalized_state.best_chain() { + check::initial_contextual_validity( + self.network, + &self.db, + &latest_non_finalized_state, + &block, + )?; + + // Reads from disk + let sprout_final_treestates = + check::anchors::fetch_sprout_final_treestates(&self.db, best_chain, &block); + + let contextual = best_chain.new_contextually_valid_block(block, &self.db)?; + + check::anchors::sprout_anchors_refer_to_treestates( + sprout_final_treestates, + Arc::clone(&contextual.block), + contextual.height, + contextual.transaction_hashes, + )?; + + check::block_commitment_is_valid_for_chain_history( + contextual.block, + self.network, + &best_chain.history_tree, + )?; + } else { + check::block_commitment_is_valid_for_chain_history( + block.block, + self.network, + &self.db.history_tree(), + )?; + }; + + Ok(()) + } } impl Service for StateService { @@ -1558,48 +1602,7 @@ impl Service for ReadStateService { let span = Span::current(); tokio::task::spawn_blocking(move || { span.in_scope(move || { - let latest_non_finalized_state = state.latest_non_finalized_state(); - if let Some(best_chain) = - latest_non_finalized_state.best_chain().map(Arc::clone) - { - check::initial_contextual_validity( - state.network, - &state.db, - &latest_non_finalized_state, - &block, - )?; - - // Reads from disk - let sprout_final_treestates = - check::anchors::fetch_sprout_final_treestates( - &state.db, - &best_chain, - &block, - ); - - let contextual = - best_chain.new_contextually_valid_block(block, &state.db)?; - - check::anchors::sprout_anchors_refer_to_treestates( - sprout_final_treestates, - Arc::clone(&contextual.block), - contextual.height, - contextual.transaction_hashes, - )?; - - check::block_commitment_is_valid_for_chain_history( - contextual.block, - state.network, - &best_chain.history_tree, - )?; - } else { - check::block_commitment_is_valid_for_chain_history( - block.block, - state.network, - &state.db.history_tree(), - )?; - }; - + state.check_best_chain_contextual_validity(block)?; // The work is done in the future. timer.finish( module_path!(), From 34724079e0d116ed41a2a57030e558eb8f3c69bf Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 16:43:43 -0500 Subject: [PATCH 06/11] adds NonFinalizedStateBuilder uses it to build a new NonFinalizedState with only the best chain updates initial_contextual_validity call to use only the best chain --- zebra-state/src/service.rs | 19 +++++++-- .../src/service/non_finalized_state.rs | 39 +++++++++++++++++++ 2 files changed, 55 insertions(+), 3 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index ac5adb6e915..3e03abdda83 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -804,12 +804,15 @@ impl ReadStateService { &self, block: PreparedBlock, ) -> Result<(), crate::ValidateContextError> { - let latest_non_finalized_state = self.latest_non_finalized_state(); - if let Some(best_chain) = latest_non_finalized_state.best_chain() { + if let Some(best_chain) = self.latest_non_finalized_state().best_chain() { + let best_chain_latest_non_finalized_state = NonFinalizedState::build(self.network) + .insert_chain(Arc::clone(best_chain)) + .finish(); + check::initial_contextual_validity( self.network, &self.db, - &latest_non_finalized_state, + &best_chain_latest_non_finalized_state, &block, )?; @@ -832,6 +835,16 @@ impl ReadStateService { &best_chain.history_tree, )?; } else { + let next_valid_height = self + .db + .finalized_tip_height() + .map(|height| (height + 1).expect("committed heights are valid")) + .unwrap_or(block::Height(0)); + + if block.height != next_valid_height { + return Err(crate::ValidateContextError::NotReadyToBeCommitted); + } + check::block_commitment_is_valid_for_chain_history( block.block, self.network, diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index dbb1a58b112..f9ed93da94b 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -49,6 +49,36 @@ pub struct NonFinalizedState { pub network: Network, } +/// Builder for NonFinalizedState +#[derive(Clone, Debug)] +#[cfg(feature = "getblocktemplate-rpcs")] +pub struct NonFinalizedStateBuilder { + /// Verified, non-finalized chains, in ascending order + pub chain_set: BTreeSet>, + + /// The configured Zcash network. + pub network: Network, +} + +#[cfg(feature = "getblocktemplate-rpcs")] +impl NonFinalizedStateBuilder { + /// Add a chain to be included in the built NonFinalizedState + /// Returns the builder + pub fn insert_chain(mut self, chain: Arc) -> Self { + self.chain_set.insert(chain); + self + } + + /// Consumes the builder and returns the NonFinalizedState + pub fn finish(self) -> NonFinalizedState { + NonFinalizedState { + chain_set: self.chain_set, + network: self.network, + } + } +} + +#[cfg(feature = "getblocktemplate-rpcs")] impl NonFinalizedState { /// Returns a new non-finalized state for `network`. pub fn new(network: Network) -> NonFinalizedState { @@ -58,6 +88,15 @@ impl NonFinalizedState { } } + /// Returns a new NonFinalizedStateBuilder for creating a non-finalized state for `network`. + #[cfg(feature = "getblocktemplate-rpcs")] + pub fn build(network: Network) -> NonFinalizedStateBuilder { + NonFinalizedStateBuilder { + chain_set: Default::default(), + network, + } + } + /// Is the internal state of `self` the same as `other`? /// /// [`Chain`] has a custom [`Eq`] implementation based on proof of work, From a6eb045050c5f54568a48dfd7cbfca37b9a17705 Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 16:52:06 -0500 Subject: [PATCH 07/11] adds parallel contextual validity checks --- zebra-state/src/service.rs | 40 +++++++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 11 deletions(-) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 3e03abdda83..822b2a2f152 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -800,6 +800,8 @@ impl ReadStateService { /// Check the contextual validity of a block in the best chain #[cfg(feature = "getblocktemplate-rpcs")] + #[allow(clippy::unwrap_in_result)] + #[tracing::instrument(level = "debug", skip_all)] fn check_best_chain_contextual_validity( &self, block: PreparedBlock, @@ -821,19 +823,35 @@ impl ReadStateService { check::anchors::fetch_sprout_final_treestates(&self.db, best_chain, &block); let contextual = best_chain.new_contextually_valid_block(block, &self.db)?; + let mut block_commitment_result = None; + let mut sprout_anchor_result = None; + + rayon::in_place_scope_fifo(|scope| { + // Clone function arguments for different threads + let block = Arc::clone(&contextual.block); + + scope.spawn_fifo(|_scope| { + block_commitment_result = + Some(check::anchors::sprout_anchors_refer_to_treestates( + sprout_final_treestates, + block, + contextual.height, + contextual.transaction_hashes, + )); + }); - check::anchors::sprout_anchors_refer_to_treestates( - sprout_final_treestates, - Arc::clone(&contextual.block), - contextual.height, - contextual.transaction_hashes, - )?; + scope.spawn_fifo(|_scope| { + sprout_anchor_result = + Some(check::block_commitment_is_valid_for_chain_history( + contextual.block, + self.network, + &best_chain.history_tree, + )); + }); + }); - check::block_commitment_is_valid_for_chain_history( - contextual.block, - self.network, - &best_chain.history_tree, - )?; + block_commitment_result.expect("scope has finished")?; + sprout_anchor_result.expect("scope has finished")?; } else { let next_valid_height = self .db From 9e63b63048c47908e719e5ee49459450093ead67 Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 18:59:13 -0500 Subject: [PATCH 08/11] removes misplaced feature flag --- zebra-state/src/service/non_finalized_state.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/zebra-state/src/service/non_finalized_state.rs b/zebra-state/src/service/non_finalized_state.rs index f9ed93da94b..0abdd71dbd5 100644 --- a/zebra-state/src/service/non_finalized_state.rs +++ b/zebra-state/src/service/non_finalized_state.rs @@ -78,7 +78,6 @@ impl NonFinalizedStateBuilder { } } -#[cfg(feature = "getblocktemplate-rpcs")] impl NonFinalizedState { /// Returns a new non-finalized state for `network`. pub fn new(network: Network) -> NonFinalizedState { From 61ccf119375a9a74da08f440ac70ec9946d6280b Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 20:16:06 -0500 Subject: [PATCH 09/11] uses MockService in getblocktemplate vector tests --- zebra-rpc/src/methods/tests/vectors.rs | 35 +++++++++----------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/zebra-rpc/src/methods/tests/vectors.rs b/zebra-rpc/src/methods/tests/vectors.rs index c8967141d8c..4eaa4fea1ba 100644 --- a/zebra-rpc/src/methods/tests/vectors.rs +++ b/zebra-rpc/src/methods/tests/vectors.rs @@ -797,29 +797,9 @@ async fn rpc_getblocktemplate() { let _init_guard = zebra_test::init(); - // Create a continuous chain of mainnet blocks from genesis - let blocks: Vec> = zebra_test::vectors::CONTINUOUS_MAINNET_BLOCKS - .iter() - .map(|(_height, block_bytes)| block_bytes.zcash_deserialize_into().unwrap()) - .collect(); - let mut mempool: MockService<_, _, _, BoxError> = MockService::build().for_unit_tests(); - // Create a populated state service - let (state, read_state, _latest_chain_tip, _chain_tip_change) = - zebra_state::populated_state(blocks.clone(), Mainnet).await; - - let ( - chain_verifier, - _transaction_verifier, - _parameter_download_task_handle, - _max_checkpoint_height, - ) = zebra_consensus::chain::init( - zebra_consensus::Config::default(), - Mainnet, - state.clone(), - true, - ) - .await; + let mut read_state = MockService::build().for_unit_tests(); + let chain_verifier = MockService::build().for_unit_tests(); let mining_config = get_block_template_rpcs::config::Config { miner_address: Some(transparent::Address::from_script_hash(Mainnet, [0x7e; 20])), @@ -833,11 +813,20 @@ async fn rpc_getblocktemplate() { Mainnet, mining_config, Buffer::new(mempool.clone(), 1), - read_state, + read_state.clone(), mock_chain_tip, tower::ServiceBuilder::new().service(chain_verifier), ); + tokio::spawn(async move { + read_state + .expect_request_that(|req| { + matches!(req, zebra_state::ReadRequest::CheckContextualValidity(_)) + }) + .await + .respond(zebra_state::ReadResponse::Validated); + }); + let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template()); mempool From eec81b16f974cb6b778e963489fd1a05cfb1beb6 Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 21:28:21 -0500 Subject: [PATCH 10/11] Adds comment about the db-only check being unused --- zebra-state/src/service.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zebra-state/src/service.rs b/zebra-state/src/service.rs index 822b2a2f152..4f006dc2b31 100644 --- a/zebra-state/src/service.rs +++ b/zebra-state/src/service.rs @@ -853,6 +853,9 @@ impl ReadStateService { block_commitment_result.expect("scope has finished")?; sprout_anchor_result.expect("scope has finished")?; } else { + // Not currently used by the getblocktemplate rpc method because it requires + // a low estimated distance to the network chain tip that means it would return + // an error before reaching this when the non-finalized state is empty anyways let next_valid_height = self .db .finalized_tip_height() From b15bc4157c21db6aaf2334798f716bc3c6ae773f Mon Sep 17 00:00:00 2001 From: arya2 Date: Wed, 16 Nov 2022 22:52:31 -0500 Subject: [PATCH 11/11] update snapshot test to use mock read state --- .../tests/snapshot/get_block_template_rpcs.rs | 42 ++++++++++++++----- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs index a97557d60be..53a1cc971a1 100644 --- a/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs +++ b/zebra-rpc/src/methods/tests/snapshot/get_block_template_rpcs.rs @@ -82,11 +82,11 @@ pub async fn test_responses( let get_block_template_rpc = GetBlockTemplateRpcImpl::new( network, - mining_config, + mining_config.clone(), Buffer::new(mempool.clone(), 1), read_state, - mock_chain_tip, - chain_verifier, + mock_chain_tip.clone(), + chain_verifier.clone(), ); // `getblockcount` @@ -104,6 +104,34 @@ pub async fn test_responses( snapshot_rpc_getblockhash(get_block_hash, &settings); + // `submitblock` + let submit_block = get_block_template_rpc + .submit_block(HexData("".into()), None) + .await + .expect("unexpected error in submitblock RPC call"); + + snapshot_rpc_submit_block_invalid(submit_block, &settings); + + // Init RPC for getblocktemplate rpc method with the mock read_state service. + let mut read_state = MockService::build().for_unit_tests(); + let get_block_template_rpc = GetBlockTemplateRpcImpl::new( + network, + mining_config, + Buffer::new(mempool.clone(), 1), + read_state.clone(), + mock_chain_tip, + chain_verifier, + ); + + tokio::spawn(async move { + read_state + .expect_request_that(|req| { + matches!(req, zebra_state::ReadRequest::CheckContextualValidity(_)) + }) + .await + .respond(zebra_state::ReadResponse::Validated); + }); + // `getblocktemplate` let get_block_template = tokio::spawn(get_block_template_rpc.get_block_template()); @@ -125,14 +153,6 @@ pub async fn test_responses( .expect("coinbase bytes are valid"); snapshot_rpc_getblocktemplate(get_block_template, coinbase_tx, &settings); - - // `submitblock` - let submit_block = get_block_template_rpc - .submit_block(HexData("".into()), None) - .await - .expect("unexpected error in submitblock RPC call"); - - snapshot_rpc_submit_block_invalid(submit_block, &settings); } /// Snapshot `getblockcount` response, using `cargo insta` and JSON serialization.