Zingo sync transparent with get taddress txids

libtonode-tests/tests/sync.rs

@@ -55,16 +55,29 @@ async fn sync_test() {
     let (regtest_manager, _cph, faucet, mut recipient, _txid) =
         scenarios::orchard_funded_recipient(5_000_000).await;
     from_inputs::quick_send(
-        &recipient,
+        &faucet,
         vec![(
-            &get_base_address_macro!(&faucet, "unified"),
+            &get_base_address_macro!(&recipient, "transparent"),
             100_000,
-            Some("Outgoing decrypt test"),
+            None,
         )],
     )
     .await
     .unwrap();
+    // from_inputs::quick_send(
+    //     &recipient,
+    //     vec![(
+    //         &get_base_address_macro!(&faucet, "unified"),
+    //         100_000,
+    //         Some("Outgoing decrypt test"),
+    //     )],
+    // )
+    // .await
+    // .unwrap();
 
+    increase_server_height(&regtest_manager, 1).await;
+    recipient.do_sync(false).await.unwrap();
+    recipient.quick_shield().await.unwrap();
     increase_server_height(&regtest_manager, 1).await;
 
     let uri = recipient.config().lightwalletd_uri.read().unwrap().clone();

zingo-sync/Cargo.toml

@@ -17,6 +17,7 @@ sapling-crypto.workspace = true
 orchard.workspace = true
 incrementalmerkletree.workspace = true
 shardtree.workspace = true
+zip32.workspace = true
 
 # Async
 futures.workspace = true

zingo-sync/src/client.rs

@@ -8,7 +8,7 @@ use zcash_client_backend::{
     data_api::chain::ChainState,
     proto::{
         compact_formats::CompactBlock,
-        service::{BlockId, TreeState},
+        service::{BlockId, GetAddressUtxosReply, TreeState},
     },
 };
 use zcash_primitives::{
@@ -27,10 +27,20 @@ pub enum FetchRequest {
     ChainTip(oneshot::Sender<BlockId>),
     /// Gets the specified range of compact blocks from the server (end exclusive).
     CompactBlockRange(oneshot::Sender<Vec<CompactBlock>>, Range<BlockHeight>),
-    /// Gets the tree states for a specified block height..
+    /// Gets the tree states for a specified block height.
     TreeState(oneshot::Sender<TreeState>, BlockHeight),
     /// Get a full transaction by txid.
     Transaction(oneshot::Sender<(Transaction, BlockHeight)>, TxId),
+    /// Get a list of unspent transparent output metadata for a given list of transparent addresses and start height.
+    UtxoMetadata(
+        oneshot::Sender<Vec<GetAddressUtxosReply>>,
+        (Vec<String>, BlockHeight),
+    ),
+    /// Get a list of transactions for a given transparent address and block range.
+    TransparentAddressTxs(
+        oneshot::Sender<Vec<(BlockHeight, Transaction)>>,
+        (String, Range<BlockHeight>),
+    ),
 }
 
 /// Gets the height of the blockchain from the server.
@@ -47,6 +57,7 @@ pub async fn get_chain_height(
 
     Ok(BlockHeight::from_u32(chain_tip.height as u32))
 }
+
 /// Gets the specified range of compact blocks from the server (end exclusive).
 ///
 /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel.
@@ -62,6 +73,7 @@ pub async fn get_compact_block_range(
 
     Ok(compact_blocks)
 }
+
 /// Gets the frontiers for a specified block height.
 ///
 /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel.
@@ -78,6 +90,7 @@ pub async fn get_frontiers(
 
     Ok(frontiers)
 }
+
 /// Gets a full transaction for a specified txid.
 ///
 /// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel.
@@ -93,3 +106,47 @@ pub async fn get_transaction_and_block_height(
 
     Ok(transaction_and_block_height)
 }
+
+/// Gets unspent transparent output metadata for a list of `transparent addresses` from the specified `start_height`.
+///
+/// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel.
+pub async fn get_utxo_metadata(
+    fetch_request_sender: UnboundedSender<FetchRequest>,
+    transparent_addresses: Vec<String>,
+    start_height: BlockHeight,
+) -> Result<Vec<GetAddressUtxosReply>, ()> {
+    if transparent_addresses.is_empty() {
+        panic!("addresses must be non-empty!");
+    }
+
+    let (sender, receiver) = oneshot::channel();
+    fetch_request_sender
+        .send(FetchRequest::UtxoMetadata(
+            sender,
+            (transparent_addresses, start_height),
+        ))
+        .unwrap();
+    let transparent_output_metadata = receiver.await.unwrap();
+
+    Ok(transparent_output_metadata)
+}
+
+/// Gets transactions relevant to a given `transparent address` in the specified `block_range`.
+///
+/// Requires [`crate::client::fetch::fetch`] to be running concurrently, connected via the `fetch_request` channel.
+pub async fn get_transparent_address_transactions(
+    fetch_request_sender: UnboundedSender<FetchRequest>,
+    transparent_address: String,
+    block_range: Range<BlockHeight>,
+) -> Result<Vec<(BlockHeight, Transaction)>, ()> {
+    let (sender, receiver) = oneshot::channel();
+    fetch_request_sender
+        .send(FetchRequest::TransparentAddressTxs(
+            sender,
+            (transparent_address, block_range),
+        ))
+        .unwrap();
+    let transactions = receiver.await.unwrap();
+
+    Ok(transactions)
+}

zingo-sync/src/client/fetch.rs

@@ -8,6 +8,7 @@ use zcash_client_backend::proto::{
     compact_formats::CompactBlock,
     service::{
         compact_tx_streamer_client::CompactTxStreamerClient, BlockId, BlockRange, ChainSpec,
+        GetAddressUtxosArg, GetAddressUtxosReply, RawTransaction, TransparentAddressBlockFilter,
         TreeState, TxFilter,
     },
 };
@@ -121,6 +122,28 @@ async fn fetch_from_server(
             let transaction = get_transaction(client, parameters, txid).await.unwrap();
             sender.send(transaction).unwrap();
         }
+        FetchRequest::UtxoMetadata(sender, (addresses, start_height)) => {
+            tracing::info!(
+                "Fetching unspent transparent output metadata from {:?} for addresses:\n{:?}",
+                &start_height,
+                &addresses
+            );
+            let utxo_metadata = get_address_utxos(client, addresses, start_height, 0)
+                .await
+                .unwrap();
+            sender.send(utxo_metadata).unwrap();
+        }
+        FetchRequest::TransparentAddressTxs(sender, (address, block_range)) => {
+            tracing::info!(
+                "Fetching raw transactions in block range {:?} for address {:?}",
+                &block_range,
+                &address
+            );
+            let transactions = get_taddress_txs(client, parameters, address, block_range)
+                .await
+                .unwrap();
+            sender.send(transactions).unwrap();
+        }
     }
 
     Ok(())
@@ -182,7 +205,7 @@ async fn get_transaction(
     });
 
     let raw_transaction = client.get_transaction(request).await.unwrap().into_inner();
-    let block_height = BlockHeight::from_u32(raw_transaction.height as u32);
+    let block_height = BlockHeight::from_u32(u32::try_from(raw_transaction.height).unwrap());
 
     let transaction = Transaction::read(
         &raw_transaction.data[..],
@@ -192,3 +215,74 @@ async fn get_transaction(
 
     Ok((transaction, block_height))
 }
+
+async fn get_address_utxos(
+    client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
+    addresses: Vec<String>,
+    start_height: BlockHeight,
+    max_entries: u32,
+) -> Result<Vec<GetAddressUtxosReply>, ()> {
+    let start_height: u64 = start_height.into();
+    let request = tonic::Request::new(GetAddressUtxosArg {
+        addresses,
+        start_height,
+        max_entries,
+    });
+
+    Ok(client
+        .get_address_utxos(request)
+        .await
+        .unwrap()
+        .into_inner()
+        .address_utxos)
+}
+
+async fn get_taddress_txs(
+    client: &mut CompactTxStreamerClient<zingo_netutils::UnderlyingService>,
+    parameters: &impl consensus::Parameters,
+    address: String,
+    block_range: Range<BlockHeight>,
+) -> Result<Vec<(BlockHeight, Transaction)>, ()> {
+    let mut raw_transactions: Vec<RawTransaction> = Vec::new();
+
+    let range = Some(BlockRange {
+        start: Some(BlockId {
+            height: block_range.start.into(),
+            hash: vec![],
+        }),
+        end: Some(BlockId {
+            height: u64::from(block_range.end) - 1,
+            hash: vec![],
+        }),
+    });
+
+    let request = tonic::Request::new(TransparentAddressBlockFilter { address, range });
+
+    let mut raw_tx_stream = client
+        .get_taddress_txids(request)
+        .await
+        .unwrap()
+        .into_inner();
+
+    while let Some(raw_tx) = raw_tx_stream.message().await.unwrap() {
+        raw_transactions.push(raw_tx);
+    }
+
+    let transactions: Vec<(BlockHeight, Transaction)> = raw_transactions
+        .into_iter()
+        .map(|raw_transaction| {
+            let block_height =
+                BlockHeight::from_u32(u32::try_from(raw_transaction.height).unwrap());
+
+            let transaction = Transaction::read(
+                &raw_transaction.data[..],
+                BranchId::for_height(parameters, block_height),
+            )
+            .unwrap();
+
+            (block_height, transaction)
+        })
+        .collect();
+
+    Ok(transactions)
+}

zingo-sync/src/keys.rs

@@ -5,23 +5,29 @@ use std::collections::HashMap;
 use getset::Getters;
 use incrementalmerkletree::Position;
 use orchard::{
-    keys::{FullViewingKey, IncomingViewingKey, Scope},
+    keys::{FullViewingKey, IncomingViewingKey},
     note_encryption::OrchardDomain,
 };
 use sapling_crypto::{
     self as sapling, note_encryption::SaplingDomain, NullifierDerivingKey, SaplingIvk,
 };
 use zcash_keys::keys::UnifiedFullViewingKey;
 use zcash_note_encryption::Domain;
+use zcash_primitives::zip32::AccountId;
+use zip32::Scope;
 
+/// Child index for the `address_index` path level in the BIP44 hierarchy.
+pub type AddressIndex = u32;
+
+/// Unique ID for shielded keys.
 #[derive(Debug, Clone, Copy, Hash, PartialEq, Eq)]
 pub struct KeyId {
     account_id: zcash_primitives::zip32::AccountId,
     scope: Scope,
 }
 
 impl KeyId {
-    pub fn from_parts(account_id: zcash_primitives::zip32::AccountId, scope: Scope) -> Self {
+    pub(crate) fn from_parts(account_id: zcash_primitives::zip32::AccountId, scope: Scope) -> Self {
         Self { account_id, scope }
     }
 }
@@ -36,8 +42,56 @@ impl memuse::DynamicUsage for KeyId {
     }
 }
 
+/// Unique ID for transparent addresses.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
+pub struct TransparentAddressId {
+    account_id: AccountId,
+    scope: TransparentScope,
+    address_index: AddressIndex,
+}
+
+impl TransparentAddressId {
+    pub(crate) fn from_parts(
+        account_id: zcash_primitives::zip32::AccountId,
+        scope: TransparentScope,
+        address_index: AddressIndex,
+    ) -> Self {
+        Self {
+            account_id,
+            scope,
+            address_index,
+        }
+    }
+
+    /// Gets address account ID
+    pub fn account_id(&self) -> AccountId {
+        self.account_id
+    }
+
+    /// Gets address scope
+    pub fn scope(&self) -> TransparentScope {
+        self.scope
+    }
+
+    /// Gets address index
+    pub fn address_index(&self) -> AddressIndex {
+        self.address_index
+    }
+}
+
+/// Child index for the `change` path level in the BIP44 hierarchy (a.k.a. scope/chain).
+#[derive(Debug, Clone, Copy, Hash, PartialEq, Eq, PartialOrd, Ord)]
+pub enum TransparentScope {
+    /// External scope
+    External,
+    /// Internal scope (a.k.a. change)
+    Internal,
+    /// Refund scope (a.k.a. ephemeral)
+    Refund,
+}
+
 /// A key that can be used to perform trial decryption and nullifier
-/// computation for a [`CompactSaplingOutput`] or [`CompactOrchardAction`].
+/// computation for a CompactSaplingOutput or CompactOrchardAction.
 pub trait ScanningKeyOps<D: Domain, Nf> {
     /// Prepare the key for use in batch trial decryption.
     fn prepare(&self) -> D::IncomingViewingKey;

zingo-sync/src/lib.rs

@@ -2,10 +2,16 @@
 //! Zingo sync engine prototype
 //!
 //! Entrypoint: [`crate::sync::sync`]
+//!
+//! Terminology:
+//! Chain height - highest block height of best chain from the server
+//! Wallet height - highest block height of blockchain known to the wallet. Commonly used, to determine the chain height
+//!                 of the previous sync, before the server is contacted to update the wallet height to the new chain height.
+//! Fully scanned height - block height in which the wallet has completed scanning all blocks equal to and below this height.
 
 pub mod client;
 pub mod error;
-pub(crate) mod keys;
+pub mod keys;
 #[allow(missing_docs)]
 pub mod primitives;
 pub(crate) mod scan;