diff --git a/Cargo.lock b/Cargo.lock index 3e75de385..c12025d9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4558,6 +4558,7 @@ dependencies = [ "zcash_primitives", "zingo-memo", "zingo-netutils", + "zip32", ] [[package]] diff --git a/libtonode-tests/tests/sync.rs b/libtonode-tests/tests/sync.rs index 0344c6beb..3a8d1f474 100644 --- a/libtonode-tests/tests/sync.rs +++ b/libtonode-tests/tests/sync.rs @@ -55,16 +55,29 @@ async fn sync_test() { let (regtest_manager, _cph, faucet, mut recipient, _txid) = scenarios::orchard_funded_recipient(5_000_000).await; from_inputs::quick_send( - &recipient, + &faucet, vec![( - &get_base_address_macro!(&faucet, "unified"), + &get_base_address_macro!(&recipient, "transparent"), 100_000, - Some("Outgoing decrypt test"), + None, )], ) .await .unwrap(); + // from_inputs::quick_send( + // &recipient, + // vec![( + // &get_base_address_macro!(&faucet, "unified"), + // 100_000, + // Some("Outgoing decrypt test"), + // )], + // ) + // .await + // .unwrap(); + increase_server_height(®test_manager, 1).await; + recipient.do_sync(false).await.unwrap(); + recipient.quick_shield().await.unwrap(); increase_server_height(®test_manager, 1).await; let uri = recipient.config().lightwalletd_uri.read().unwrap().clone(); diff --git a/zingo-sync/Cargo.toml b/zingo-sync/Cargo.toml index 3df25e5ec..a93e90105 100644 --- a/zingo-sync/Cargo.toml +++ b/zingo-sync/Cargo.toml @@ -17,6 +17,7 @@ sapling-crypto.workspace = true orchard.workspace = true incrementalmerkletree.workspace = true shardtree.workspace = true +zip32.workspace = true # Async futures.workspace = true diff --git a/zingo-sync/src/client.rs b/zingo-sync/src/client.rs index fa556a493..c9762677f 100644 --- a/zingo-sync/src/client.rs +++ b/zingo-sync/src/client.rs @@ -8,7 +8,7 @@ use zcash_client_backend::{ data_api::chain::ChainState, proto::{ compact_formats::CompactBlock, - service::{BlockId, TreeState}, + service::{BlockId, GetAddressUtxosReply, TreeState}, }, }; use zcash_primitives::{ @@ -27,10 +27,20 @@ pub enum FetchRequest { ChainTip(oneshot::Sender), /// Gets the specified range of compact blocks from the server (end exclusive). CompactBlockRange(oneshot::Sender>, Range), - /// Gets the tree states for a specified block height.. + /// Gets the tree states for a specified block height. TreeState(oneshot::Sender, BlockHeight), /// Get a full transaction by txid. Transaction(oneshot::Sender<(Transaction, BlockHeight)>, TxId), + /// Get a list of unspent transparent output metadata for a given list of transparent addresses and start height. + UtxoMetadata( + oneshot::Sender>, + (Vec, BlockHeight), + ), + /// Get a list of transactions for a given transparent address and block range. + TransparentAddressTxs( + oneshot::Sender>, + (String, Range), + ), } /// Gets the height of the blockchain from the server. @@ -47,6 +57,7 @@ pub async fn get_chain_height( Ok(BlockHeight::from_u32(chain_tip.height as u32)) } + /// Gets the specified range of compact blocks from the server (end exclusive). /// /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. @@ -62,6 +73,7 @@ pub async fn get_compact_block_range( Ok(compact_blocks) } + /// Gets the frontiers for a specified block height. /// /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. @@ -78,6 +90,7 @@ pub async fn get_frontiers( Ok(frontiers) } + /// Gets a full transaction for a specified txid. /// /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. @@ -93,3 +106,47 @@ pub async fn get_transaction_and_block_height( Ok(transaction_and_block_height) } + +/// Gets unspent transparent output metadata for a list of `transparent addresses` from the specified `start_height`. +/// +/// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. +pub async fn get_utxo_metadata( + fetch_request_sender: UnboundedSender, + transparent_addresses: Vec, + start_height: BlockHeight, +) -> Result, ()> { + if transparent_addresses.is_empty() { + panic!("addresses must be non-empty!"); + } + + let (sender, receiver) = oneshot::channel(); + fetch_request_sender + .send(FetchRequest::UtxoMetadata( + sender, + (transparent_addresses, start_height), + )) + .unwrap(); + let transparent_output_metadata = receiver.await.unwrap(); + + Ok(transparent_output_metadata) +} + +/// Gets transactions relevant to a given `transparent address` in the specified `block_range`. +/// +/// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel. +pub async fn get_transparent_address_transactions( + fetch_request_sender: UnboundedSender, + transparent_address: String, + block_range: Range, +) -> Result, ()> { + let (sender, receiver) = oneshot::channel(); + fetch_request_sender + .send(FetchRequest::TransparentAddressTxs( + sender, + (transparent_address, block_range), + )) + .unwrap(); + let transactions = receiver.await.unwrap(); + + Ok(transactions) +} diff --git a/zingo-sync/src/client/fetch.rs b/zingo-sync/src/client/fetch.rs index 979b1491f..f6c341a88 100644 --- a/zingo-sync/src/client/fetch.rs +++ b/zingo-sync/src/client/fetch.rs @@ -8,6 +8,7 @@ use zcash_client_backend::proto::{ compact_formats::CompactBlock, service::{ compact_tx_streamer_client::CompactTxStreamerClient, BlockId, BlockRange, ChainSpec, + GetAddressUtxosArg, GetAddressUtxosReply, RawTransaction, TransparentAddressBlockFilter, TreeState, TxFilter, }, }; @@ -121,6 +122,28 @@ async fn fetch_from_server( let transaction = get_transaction(client, parameters, txid).await.unwrap(); sender.send(transaction).unwrap(); } + FetchRequest::UtxoMetadata(sender, (addresses, start_height)) => { + tracing::info!( + "Fetching unspent transparent output metadata from {:?} for addresses:\n{:?}", + &start_height, + &addresses + ); + let utxo_metadata = get_address_utxos(client, addresses, start_height, 0) + .await + .unwrap(); + sender.send(utxo_metadata).unwrap(); + } + FetchRequest::TransparentAddressTxs(sender, (address, block_range)) => { + tracing::info!( + "Fetching raw transactions in block range {:?} for address {:?}", + &block_range, + &address + ); + let transactions = get_taddress_txs(client, parameters, address, block_range) + .await + .unwrap(); + sender.send(transactions).unwrap(); + } } Ok(()) @@ -182,7 +205,7 @@ async fn get_transaction( }); let raw_transaction = client.get_transaction(request).await.unwrap().into_inner(); - let block_height = BlockHeight::from_u32(raw_transaction.height as u32); + let block_height = BlockHeight::from_u32(u32::try_from(raw_transaction.height).unwrap()); let transaction = Transaction::read( &raw_transaction.data[..], @@ -192,3 +215,74 @@ async fn get_transaction( Ok((transaction, block_height)) } + +async fn get_address_utxos( + client: &mut CompactTxStreamerClient, + addresses: Vec, + start_height: BlockHeight, + max_entries: u32, +) -> Result, ()> { + let start_height: u64 = start_height.into(); + let request = tonic::Request::new(GetAddressUtxosArg { + addresses, + start_height, + max_entries, + }); + + Ok(client + .get_address_utxos(request) + .await + .unwrap() + .into_inner() + .address_utxos) +} + +async fn get_taddress_txs( + client: &mut CompactTxStreamerClient, + parameters: &impl consensus::Parameters, + address: String, + block_range: Range, +) -> Result, ()> { + let mut raw_transactions: Vec = Vec::new(); + + let range = Some(BlockRange { + start: Some(BlockId { + height: block_range.start.into(), + hash: vec![], + }), + end: Some(BlockId { + height: u64::from(block_range.end) - 1, + hash: vec![], + }), + }); + + let request = tonic::Request::new(TransparentAddressBlockFilter { address, range }); + + let mut raw_tx_stream = client + .get_taddress_txids(request) + .await + .unwrap() + .into_inner(); + + while let Some(raw_tx) = raw_tx_stream.message().await.unwrap() { + raw_transactions.push(raw_tx); + } + + let transactions: Vec<(BlockHeight, Transaction)> = raw_transactions + .into_iter() + .map(|raw_transaction| { + let block_height = + BlockHeight::from_u32(u32::try_from(raw_transaction.height).unwrap()); + + let transaction = Transaction::read( + &raw_transaction.data[..], + BranchId::for_height(parameters, block_height), + ) + .unwrap(); + + (block_height, transaction) + }) + .collect(); + + Ok(transactions) +} diff --git a/zingo-sync/src/keys.rs b/zingo-sync/src/keys.rs index 0e2538aa0..dc39a033d 100644 --- a/zingo-sync/src/keys.rs +++ b/zingo-sync/src/keys.rs @@ -5,7 +5,7 @@ use std::collections::HashMap; use getset::Getters; use incrementalmerkletree::Position; use orchard::{ - keys::{FullViewingKey, IncomingViewingKey, Scope}, + keys::{FullViewingKey, IncomingViewingKey}, note_encryption::OrchardDomain, }; use sapling_crypto::{ @@ -13,7 +13,13 @@ use sapling_crypto::{ }; use zcash_keys::keys::UnifiedFullViewingKey; use zcash_note_encryption::Domain; +use zcash_primitives::zip32::AccountId; +use zip32::Scope; +/// Child index for the `address_index` path level in the BIP44 hierarchy. +pub type AddressIndex = u32; + +/// Unique ID for shielded keys. #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)] pub struct KeyId { account_id: zcash_primitives::zip32::AccountId, @@ -21,7 +27,7 @@ pub struct KeyId { } impl KeyId { - pub fn from_parts(account_id: zcash_primitives::zip32::AccountId, scope: Scope) -> Self { + pub(crate) fn from_parts(account_id: zcash_primitives::zip32::AccountId, scope: Scope) -> Self { Self { account_id, scope } } } @@ -36,8 +42,56 @@ impl memuse::DynamicUsage for KeyId { } } +/// Unique ID for transparent addresses. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct TransparentAddressId { + account_id: AccountId, + scope: TransparentScope, + address_index: AddressIndex, +} + +impl TransparentAddressId { + pub(crate) fn from_parts( + account_id: zcash_primitives::zip32::AccountId, + scope: TransparentScope, + address_index: AddressIndex, + ) -> Self { + Self { + account_id, + scope, + address_index, + } + } + + /// Gets address account ID + pub fn account_id(&self) -> AccountId { + self.account_id + } + + /// Gets address scope + pub fn scope(&self) -> TransparentScope { + self.scope + } + + /// Gets address index + pub fn address_index(&self) -> AddressIndex { + self.address_index + } +} + +/// Child index for the `change` path level in the BIP44 hierarchy (a.k.a. scope/chain). +#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)] +pub enum TransparentScope { + /// External scope + External, + /// Internal scope (a.k.a. change) + Internal, + /// Refund scope (a.k.a. ephemeral) + Refund, +} + /// A key that can be used to perform trial decryption and nullifier -/// computation for a [`CompactSaplingOutput`] or [`CompactOrchardAction`]. +/// computation for a CompactSaplingOutput or CompactOrchardAction. pub trait ScanningKeyOps { /// Prepare the key for use in batch trial decryption. fn prepare(&self) -> D::IncomingViewingKey; diff --git a/zingo-sync/src/lib.rs b/zingo-sync/src/lib.rs index d49635aae..10f7410c7 100644 --- a/zingo-sync/src/lib.rs +++ b/zingo-sync/src/lib.rs @@ -2,10 +2,16 @@ //! Zingo sync engine prototype //! //! Entrypoint: [`crate::sync::sync`] +//! +//! Terminology: +//! Chain height - highest block height of best chain from the server +//! Wallet height - highest block height of blockchain known to the wallet. Commonly used, to determine the chain height +//! of the previous sync, before the server is contacted to update the wallet height to the new chain height. +//! Fully scanned height - block height in which the wallet has completed scanning all blocks equal to and below this height. pub mod client; pub mod error; -pub(crate) mod keys; +pub mod keys; #[allow(missing_docs)] pub mod primitives; pub(crate) mod scan; diff --git a/zingo-sync/src/primitives.rs b/zingo-sync/src/primitives.rs index 4e6c30fe0..c15e28435 100644 --- a/zingo-sync/src/primitives.rs +++ b/zingo-sync/src/primitives.rs @@ -1,20 +1,24 @@ //! Module for primitive structs associated with the sync engine -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use getset::{CopyGetters, Getters, MutGetters, Setters}; use incrementalmerkletree::Position; -use zcash_client_backend::data_api::scanning::ScanRange; +use zcash_client_backend::data_api::scanning::{ScanPriority, ScanRange}; use zcash_keys::{address::UnifiedAddress, encoding::encode_payment_address}; use zcash_primitives::{ block::BlockHash, consensus::{BlockHeight, NetworkConstants, Parameters}, + legacy::Script, memo::Memo, transaction::TxId, }; -use crate::{keys::KeyId, utils}; +use crate::{ + keys::{KeyId, TransparentAddressId}, + utils, +}; /// Encapsulates the current state of sync #[derive(Debug, Getters, MutGetters)] @@ -23,8 +27,9 @@ pub struct SyncState { /// A vec of block ranges with scan priorities from wallet birthday to chain tip. /// In block height order with no overlaps or gaps. scan_ranges: Vec, - /// Block height and txid of known spends which are awaiting the scanning of the range it belongs to for transaction decryption. - spend_locations: Vec<(BlockHeight, TxId)>, + /// Block height and txid of relevant transactions that have yet to be scanned. These may be added due to spend + /// detections or transparent output discovery + locators: BTreeSet<(BlockHeight, TxId)>, } impl SyncState { @@ -32,17 +37,32 @@ impl SyncState { pub fn new() -> Self { SyncState { scan_ranges: Vec::new(), - spend_locations: Vec::new(), + locators: BTreeSet::new(), } } - pub fn fully_scanned(&self) -> bool { - self.scan_ranges().iter().all(|scan_range| { - matches!( - scan_range.priority(), - zcash_client_backend::data_api::scanning::ScanPriority::Scanned - ) - }) + /// Returns true if all scan ranges are scanned. + pub(crate) fn scan_complete(&self) -> bool { + self.scan_ranges() + .iter() + .all(|scan_range| scan_range.priority() == ScanPriority::Scanned) + } + + /// Returns the block height at which all blocks equal to and below this height are scanned. + pub fn fully_scanned_height(&self) -> BlockHeight { + if let Some(scan_range) = self + .scan_ranges() + .iter() + .find(|scan_range| scan_range.priority() != ScanPriority::Scanned) + { + scan_range.block_range().start - 1 + } else { + self.scan_ranges() + .last() + .expect("scan ranges always non-empty") + .block_range() + .end + } } } @@ -147,6 +167,8 @@ pub struct WalletTransaction { outgoing_sapling_notes: Vec, #[getset(skip)] outgoing_orchard_notes: Vec, + #[getset(skip)] + transparent_coins: Vec, } impl WalletTransaction { @@ -157,6 +179,7 @@ impl WalletTransaction { orchard_notes: Vec, outgoing_sapling_notes: Vec, outgoing_orchard_notes: Vec, + transparent_coins: Vec, ) -> Self { Self { transaction, @@ -165,6 +188,7 @@ impl WalletTransaction { orchard_notes, outgoing_sapling_notes, outgoing_orchard_notes, + transparent_coins, } } @@ -191,6 +215,14 @@ impl WalletTransaction { pub fn outgoing_orchard_notes(&self) -> &[OutgoingOrchardNote] { &self.outgoing_orchard_notes } + + pub fn transparent_coins(&self) -> &[TransparentCoin] { + &self.transparent_coins + } + + pub fn transparent_coins_mut(&mut self) -> Vec<&mut TransparentCoin> { + self.transparent_coins.iter_mut().collect() + } } pub type SaplingNote = WalletNote; @@ -311,3 +343,46 @@ pub(crate) trait SyncOutgoingNotes { where P: Parameters + NetworkConstants; } + +/// Transparent coin (output) with metadata relevant to the wallet +#[derive(Debug, Clone, Getters, CopyGetters, Setters)] +pub struct TransparentCoin { + /// Output ID + #[getset(get_copy = "pub")] + output_id: OutputId, + /// Identifier for key used to derive address + #[getset(get_copy = "pub")] + key_id: TransparentAddressId, + /// Encoded transparent address + #[getset(get = "pub")] + address: String, + /// Script + #[getset(get = "pub")] + script: Script, + /// Coin value + #[getset(get_copy = "pub")] + value: u64, + /// Spend status + #[getset(get = "pub", set = "pub")] + spent: bool, +} + +impl TransparentCoin { + pub fn from_parts( + output_id: OutputId, + key_id: TransparentAddressId, + address: String, + script: Script, + value: u64, + spent: bool, + ) -> Self { + Self { + output_id, + key_id, + address, + script, + value, + spent, + } + } +} diff --git a/zingo-sync/src/scan/compact_blocks.rs b/zingo-sync/src/scan/compact_blocks.rs index 2a1daf980..9285c257f 100644 --- a/zingo-sync/src/scan/compact_blocks.rs +++ b/zingo-sync/src/scan/compact_blocks.rs @@ -149,8 +149,6 @@ fn trial_decrypt

( where P: Parameters + Send + 'static, { - // TODO: add outgoing decryption - let mut runners = BatchRunners::<(), ()>::for_keys(100, scanning_keys); for block in compact_blocks { runners.add_block(parameters, block.clone()).unwrap(); diff --git a/zingo-sync/src/scan/task.rs b/zingo-sync/src/scan/task.rs index 59adc632b..ad09c1a7e 100644 --- a/zingo-sync/src/scan/task.rs +++ b/zingo-sync/src/scan/task.rs @@ -198,7 +198,7 @@ where } } - if !wallet.get_sync_state().unwrap().fully_scanned() && self.worker_poolsize() == 0 { + if !wallet.get_sync_state().unwrap().scan_complete() && self.worker_poolsize() == 0 { panic!("worker pool should not be empty with unscanned ranges!") } } diff --git a/zingo-sync/src/scan/transactions.rs b/zingo-sync/src/scan/transactions.rs index 9d613834f..3ed897325 100644 --- a/zingo-sync/src/scan/transactions.rs +++ b/zingo-sync/src/scan/transactions.rs @@ -247,6 +247,7 @@ fn scan_transaction( orchard_notes, outgoing_sapling_notes, outgoing_orchard_notes, + vec![], // TODO: add coins )) } diff --git a/zingo-sync/src/sync.rs b/zingo-sync/src/sync.rs index 0187666ca..4a6f1705f 100644 --- a/zingo-sync/src/sync.rs +++ b/zingo-sync/src/sync.rs @@ -26,11 +26,14 @@ use tokio::sync::mpsc; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::AccountId; -// TODO: create sub modules for sync module to organise code +mod transparent; + +// TODO: create sub modules for sync module to organise code, the "brain" which organises all the scan ranges should be separated out // TODO; replace fixed batches with orchard shard ranges (block ranges containing all note commitments to an orchard shard or fragment of a shard) const BATCH_SIZE: u32 = 1_000; const VERIFY_BLOCK_RANGE_SIZE: u32 = 10; +const MAX_VERIFICATION_WINDOW: u32 = 100; // TODO: fail if re-org goes beyond this window /// Syncs a wallet to the latest state of the blockchain pub async fn sync( @@ -52,10 +55,43 @@ where consensus_parameters.clone(), )); - update_scan_ranges( + let wallet_height = + if let Some(highest_range) = wallet.get_sync_state().unwrap().scan_ranges().last() { + highest_range.block_range().end - 1 + } else { + let wallet_birthday = wallet.get_birthday().unwrap(); + let sapling_activation_height = consensus_parameters + .activation_height(NetworkUpgrade::Sapling) + .expect("sapling activation height should always return Some"); + + let highest = match wallet_birthday.cmp(&sapling_activation_height) { + cmp::Ordering::Greater | cmp::Ordering::Equal => wallet_birthday, + cmp::Ordering::Less => sapling_activation_height, + }; + highest - 1 + }; + let chain_height = client::get_chain_height(fetch_request_sender.clone()) + .await + .unwrap(); + if wallet_height > chain_height { + // TODO: truncate wallet to server height in case of reorg + panic!("wallet is ahead of server!") + } + let ufvks = wallet.get_unified_full_viewing_keys().unwrap(); + + transparent::update_addresses_and_locators( + wallet, fetch_request_sender.clone(), consensus_parameters, - wallet.get_birthday().unwrap(), + &ufvks, + wallet_height, + chain_height, + ) + .await; + + update_scan_ranges( + wallet_height, + chain_height, wallet.get_sync_state_mut().unwrap(), ) .await @@ -63,7 +99,6 @@ where // create channel for receiving scan results and launch scanner let (scan_results_sender, mut scan_results_receiver) = mpsc::unbounded_channel(); - let ufvks = wallet.get_unified_full_viewing_keys().unwrap(); let mut scanner = Scanner::new( scan_results_sender, fetch_request_sender.clone(), @@ -124,29 +159,16 @@ where { scanner.worker_poolsize() == 0 && scan_results_receiver.is_empty() - && wallet.get_sync_state().unwrap().fully_scanned() + && wallet.get_sync_state().unwrap().scan_complete() } /// Update scan ranges for scanning -async fn update_scan_ranges

( - fetch_request_sender: mpsc::UnboundedSender, - consensus_parameters: &P, - wallet_birthday: BlockHeight, +async fn update_scan_ranges( + wallet_height: BlockHeight, + chain_height: BlockHeight, sync_state: &mut SyncState, -) -> Result<(), ()> -where - P: consensus::Parameters, -{ - let chain_height = client::get_chain_height(fetch_request_sender) - .await - .unwrap(); - create_scan_range( - chain_height, - consensus_parameters, - wallet_birthday, - sync_state, - ) - .await?; +) -> Result<(), ()> { + create_scan_range(wallet_height, chain_height, sync_state).await?; reset_scan_ranges(sync_state)?; set_verification_scan_range(sync_state)?; @@ -157,43 +179,17 @@ where Ok(()) } -/// Create scan range between the last known chain height (wallet height) and the chain height from the server -async fn create_scan_range

( +/// Create scan range between the wallet height and the chain height from the server +async fn create_scan_range( + wallet_height: BlockHeight, chain_height: BlockHeight, - consensus_parameters: &P, - wallet_birthday: BlockHeight, sync_state: &mut SyncState, -) -> Result<(), ()> -where - P: consensus::Parameters, -{ +) -> Result<(), ()> { let scan_ranges = sync_state.scan_ranges_mut(); - let wallet_height = if scan_ranges.is_empty() { - let sapling_activation_height = consensus_parameters - .activation_height(NetworkUpgrade::Sapling) - .expect("sapling activation height should always return Some"); - - match wallet_birthday.cmp(&sapling_activation_height) { - cmp::Ordering::Greater | cmp::Ordering::Equal => wallet_birthday, - cmp::Ordering::Less => sapling_activation_height, - } - } else { - scan_ranges - .last() - .expect("Vec should not be empty") - .block_range() - .end - }; - - if wallet_height > chain_height { - // TODO: truncate wallet to server height in case of reorg - panic!("wallet is ahead of server!") - } - let new_scan_range = ScanRange::from_parts( Range { - start: wallet_height, + start: wallet_height + 1, end: chain_height + 1, }, ScanPriority::Historic, diff --git a/zingo-sync/src/sync/transparent.rs b/zingo-sync/src/sync/transparent.rs new file mode 100644 index 000000000..0bee22ae7 --- /dev/null +++ b/zingo-sync/src/sync/transparent.rs @@ -0,0 +1,209 @@ +use std::collections::{BTreeSet, HashMap}; +use std::ops::Range; + +use tokio::sync::mpsc; + +use zcash_address::{ToAddress, ZcashAddress}; +use zcash_keys::keys::UnifiedFullViewingKey; +use zcash_primitives::consensus::{self, BlockHeight}; +use zcash_primitives::legacy::keys::{AccountPubKey, IncomingViewingKey, NonHardenedChildIndex}; +use zcash_primitives::legacy::TransparentAddress; +use zcash_primitives::transaction::TxId; +use zcash_primitives::zip32::AccountId; + +use crate::client::{get_transparent_address_transactions, FetchRequest}; +use crate::keys::{AddressIndex, TransparentAddressId, TransparentScope}; +use crate::traits::SyncWallet; + +use super::MAX_VERIFICATION_WINDOW; + +const ADDRESS_GAP_LIMIT: usize = 20; + +/// Discovers all addresses in use by the wallet and returns locators for any new relevant transactions to scan transparent +/// bundles. +/// `wallet_height` should be the value before updating scan ranges. i.e. the wallet height as of previous sync. +pub(crate) async fn update_addresses_and_locators( + wallet: &mut W, + fetch_request_sender: mpsc::UnboundedSender, + consensus_parameters: &P, + ufvks: &HashMap, + wallet_height: BlockHeight, + chain_height: BlockHeight, +) where + P: consensus::Parameters, + W: SyncWallet, +{ + let wallet_addresses = wallet.get_transparent_addresses_mut().unwrap(); + let mut locators: BTreeSet<(BlockHeight, TxId)> = BTreeSet::new(); + let block_range = Range { + start: wallet_height + 1 - MAX_VERIFICATION_WINDOW, + end: chain_height + 1, + }; + + // find locators for any new transactions relevant to known addresses + for address in wallet_addresses.values() { + let transactions = get_transparent_address_transactions( + fetch_request_sender.clone(), + address.clone(), + block_range.clone(), + ) + .await + .unwrap(); + + transactions.iter().for_each(|(height, tx)| { + locators.insert((*height, tx.txid())); + }); + } + + // discover new addresses and find locators for relevant transactions + for (account_id, ufvk) in ufvks { + if let Some(account_pubkey) = ufvk.transparent() { + for scope in [ + TransparentScope::External, + TransparentScope::Internal, + TransparentScope::Refund, + ] { + // start with the first address index previously unused by the wallet + let mut address_index = if let Some(id) = wallet_addresses + .iter() + .map(|(id, _)| id) + .filter(|id| id.account_id() == *account_id && id.scope() == scope) + .next_back() + { + id.address_index() + 1 + } else { + 0 + }; + let mut unused_address_count: usize = 0; + let mut addresses: Vec<(TransparentAddressId, String)> = Vec::new(); + + while unused_address_count < ADDRESS_GAP_LIMIT { + let address_id = + TransparentAddressId::from_parts(*account_id, scope, address_index); + let address = derive_address(consensus_parameters, account_pubkey, address_id); + addresses.push((address_id, address.clone())); + + let transactions = get_transparent_address_transactions( + fetch_request_sender.clone(), + address, + block_range.clone(), + ) + .await + .unwrap(); + + if transactions.is_empty() { + unused_address_count += 1; + } else { + transactions.iter().for_each(|(height, tx)| { + locators.insert((*height, tx.txid())); + }); + unused_address_count = 0; + } + + address_index += 1; + } + + addresses.truncate(addresses.len() - ADDRESS_GAP_LIMIT); + addresses.into_iter().for_each(|(id, address)| { + wallet_addresses.insert(id, address); + }); + } + } + } + + wallet + .get_sync_state_mut() + .unwrap() + .locators_mut() + .append(&mut locators); +} + +fn derive_address

( + consensus_parameters: &P, + account_pubkey: &AccountPubKey, + address_id: TransparentAddressId, +) -> String +where + P: consensus::Parameters, +{ + let address = match address_id.scope() { + TransparentScope::External => { + derive_external_address(account_pubkey, address_id.address_index()) + } + TransparentScope::Internal => { + derive_internal_address(account_pubkey, address_id.address_index()) + } + TransparentScope::Refund => { + derive_refund_address(account_pubkey, address_id.address_index()) + } + }; + + encode_address(consensus_parameters, address) +} + +fn derive_external_address( + account_pubkey: &AccountPubKey, + address_index: AddressIndex, +) -> TransparentAddress { + account_pubkey + .derive_external_ivk() + .unwrap() + .derive_address( + NonHardenedChildIndex::from_index(address_index) + .expect("all non-hardened address indexes in use!"), + ) + .unwrap() +} + +fn derive_internal_address( + account_pubkey: &AccountPubKey, + address_index: AddressIndex, +) -> TransparentAddress { + account_pubkey + .derive_internal_ivk() + .unwrap() + .derive_address( + NonHardenedChildIndex::from_index(address_index) + .expect("all non-hardened address indexes in use!"), + ) + .unwrap() +} + +fn derive_refund_address( + account_pubkey: &AccountPubKey, + address_index: AddressIndex, +) -> TransparentAddress { + account_pubkey + .derive_ephemeral_ivk() + .unwrap() + .derive_ephemeral_address( + NonHardenedChildIndex::from_index(address_index) + .expect("all non-hardened address indexes in use!"), + ) + .unwrap() +} + +fn encode_address

(consensus_parameters: &P, address: TransparentAddress) -> String +where + P: consensus::Parameters, +{ + let zcash_address = match address { + TransparentAddress::PublicKeyHash(data) => { + ZcashAddress::from_transparent_p2pkh(consensus_parameters.network_type(), data) + } + TransparentAddress::ScriptHash(data) => { + ZcashAddress::from_transparent_p2sh(consensus_parameters.network_type(), data) + } + }; + zcash_address.to_string() +} + +// TODO: process memo encoded address indexes. +// 1. return any memo address ids from scan in ScanResults +// 2. derive the addresses up to that index, add to wallet addresses and send them to GetTaddressTxids +// 3. for each transaction returned: +// a) if the tx is in a range that is not scanned, add locator to sync_state +// b) if the range is scanned and the tx is already in the wallet, rescan the zcash transaction transparent bundles in +// the wallet transaction +// c) if the range is scanned and the tx does not exist in the wallet, fetch the compact block if its not in the wallet +// and scan the transparent bundles diff --git a/zingo-sync/src/traits.rs b/zingo-sync/src/traits.rs index 2db810e70..ff7d9473c 100644 --- a/zingo-sync/src/traits.rs +++ b/zingo-sync/src/traits.rs @@ -8,27 +8,40 @@ use zcash_primitives::consensus::BlockHeight; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::AccountId; +use crate::keys::TransparentAddressId; use crate::primitives::{NullifierMap, SyncState, WalletBlock, WalletTransaction}; use crate::witness::{ShardTreeData, ShardTrees}; +// TODO: clean up interface and move many default impls out of traits. consider merging to a simplified SyncWallet interface. + /// Temporary dump for all neccessary wallet functionality for PoC pub trait SyncWallet { /// Errors associated with interfacing the sync engine with wallet data type Error: Debug; - /// Returns block height wallet was created + /// Returns the block height wallet was created. fn get_birthday(&self) -> Result; - /// Returns reference to wallet sync state + /// Returns a reference to wallet sync state. fn get_sync_state(&self) -> Result<&SyncState, Self::Error>; - /// Returns mutable reference to wallet sync state + /// Returns a mutable reference to wallet sync state. fn get_sync_state_mut(&mut self) -> Result<&mut SyncState, Self::Error>; /// Returns all unified full viewing keys known to this wallet. fn get_unified_full_viewing_keys( &self, ) -> Result, Self::Error>; + + /// Returns a reference to all the transparent addresses known to this wallet. + fn get_transparent_addresses( + &self, + ) -> Result<&BTreeMap, Self::Error>; + + /// Returns a mutable reference to all the transparent addresses known to this wallet. + fn get_transparent_addresses_mut( + &mut self, + ) -> Result<&mut BTreeMap, Self::Error>; } /// Trait for interfacing [`crate::primitives::WalletBlock`]s with wallet data diff --git a/zingolib/src/wallet.rs b/zingolib/src/wallet.rs index 13fa9b013..40a08fb09 100644 --- a/zingolib/src/wallet.rs +++ b/zingolib/src/wallet.rs @@ -15,6 +15,7 @@ use rand::Rng; #[cfg(feature = "sync")] use zingo_sync::{ + keys::TransparentAddressId, primitives::{NullifierMap, SyncState, WalletBlock, WalletTransaction}, witness::ShardTrees, }; @@ -235,6 +236,10 @@ pub struct LightWallet { /// Sync state #[cfg(feature = "sync")] pub sync_state: SyncState, + + /// Transparent addresses + #[cfg(feature = "sync")] + pub transparent_addresses: BTreeMap, } impl LightWallet { @@ -416,6 +421,8 @@ impl LightWallet { shard_trees: zingo_sync::witness::ShardTrees::new(), #[cfg(feature = "sync")] sync_state: zingo_sync::primitives::SyncState::new(), + #[cfg(feature = "sync")] + transparent_addresses: BTreeMap::new(), }) } diff --git a/zingolib/src/wallet/disk.rs b/zingolib/src/wallet/disk.rs index fa641545b..a51e3ee22 100644 --- a/zingolib/src/wallet/disk.rs +++ b/zingolib/src/wallet/disk.rs @@ -301,6 +301,8 @@ impl LightWallet { shard_trees: zingo_sync::witness::ShardTrees::new(), #[cfg(feature = "sync")] sync_state: zingo_sync::primitives::SyncState::new(), + #[cfg(feature = "sync")] + transparent_addresses: BTreeMap::new(), }; Ok(lw) diff --git a/zingolib/src/wallet/sync.rs b/zingolib/src/wallet/sync.rs index 56bf21a30..dddf20797 100644 --- a/zingolib/src/wallet/sync.rs +++ b/zingolib/src/wallet/sync.rs @@ -8,6 +8,7 @@ use std::{ use zcash_keys::keys::{UnifiedFullViewingKey, UnifiedSpendingKey}; use zcash_primitives::consensus::BlockHeight; use zingo_sync::{ + keys::TransparentAddressId, primitives::{NullifierMap, SyncState, WalletBlock}, traits::{SyncBlocks, SyncNullifiers, SyncShardTrees, SyncTransactions, SyncWallet}, witness::ShardTrees, @@ -53,6 +54,18 @@ impl SyncWallet for LightWallet { Ok(ufvk_map) } + + fn get_transparent_addresses( + &self, + ) -> Result<&BTreeMap, Self::Error> { + Ok(&self.transparent_addresses) + } + + fn get_transparent_addresses_mut( + &mut self, + ) -> Result<&mut BTreeMap, Self::Error> { + Ok(&mut self.transparent_addresses) + } } impl SyncBlocks for LightWallet {