From dc9839f67873271012fe545ac9bb37904ed14491 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 14 Jul 2022 18:10:06 +0700 Subject: [PATCH 01/52] WIP init_native_client,RpcClient, ZcoinNativeClientInitError added --- mm2src/coins/z_coin.rs | 18 +++++-- mm2src/coins/z_coin/z_coin_errors.rs | 25 ++++++++- mm2src/coins/z_coin/z_rpc.rs | 76 ++++++++++++++++++++++++---- 3 files changed, 104 insertions(+), 15 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 7d859fca8f..aee4221804 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,6 +1,6 @@ use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawInProgressStatus, WithdrawTaskHandle}; -use crate::utxo::rpc_clients::{ElectrumRpcRequest, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, - UtxoRpcResult}; +use crate::utxo::rpc_clients::{ElectrumRpcRequest, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, + UtxoRpcFut, UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{big_decimal_from_sat_unsigned, payment_script}; @@ -65,7 +65,7 @@ use z_htlc::{z_p2sh_spend, z_send_dex_fee, z_send_htlc}; mod z_rpc; pub use z_rpc::SyncStatus; -use z_rpc::{init_light_client, SaplingSyncConnector, SaplingSyncGuard, WalletDbShared}; +use z_rpc::{init_light_client, init_native_client, SaplingSyncConnector, SaplingSyncGuard, WalletDbShared}; mod z_coin_errors; pub use z_coin_errors::*; @@ -480,7 +480,17 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { let evk = ExtendedFullViewingKey::from(&self.z_spending_key); let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { ZcoinRpcMode::Native => { - return MmError::err(ZCoinBuildError::NativeModeIsNotSupportedYet); + let native_client: NativeClient = self.native_client().unwrap(); + let cache_db_path = self.db_dir_path.join(format!("{}_native_cache.db", self.ticker)); + let wallet_db_path = self.db_dir_path.join(format!("{}_native_wallet.db", self.ticker)); + init_native_client( + native_client, + cache_db_path, + wallet_db_path, + self.consensus_params.clone(), + evk, + ) + .await? }, ZcoinRpcMode::Light { light_wallet_d_servers, .. diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 1b1333aa4b..a2902bf158 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -16,6 +16,7 @@ use zcash_primitives::transaction::builder::Error as ZTxBuilderError; pub enum UpdateBlocksCacheErr { GrpcError(tonic::Status), DbError(SqliteError), + JsonRpsErr(UtxoRpcError), } impl From for UpdateBlocksCacheErr { @@ -26,6 +27,10 @@ impl From for UpdateBlocksCacheErr { fn from(err: SqliteError) -> Self { UpdateBlocksCacheErr::DbError(err) } } +impl From for UpdateBlocksCacheErr { + fn from(err: UtxoRpcError) -> Self { UpdateBlocksCacheErr::JsonRpsErr(err) } +} + #[derive(Debug, Display)] #[non_exhaustive] pub enum ZcoinLightClientInitError { @@ -36,10 +41,24 @@ pub enum ZcoinLightClientInitError { ZcashSqliteError(ZcashClientError), } +#[derive(Debug, Display)] +#[non_exhaustive] +pub enum ZcoinNativeClientInitError { + TlsConfigFailure(tonic::transport::Error), + ConnectionFailure(tonic::transport::Error), + BlocksDbInitFailure(SqliteError), + WalletDbInitFailure(SqliteError), + ZcashSqliteError(ZcashClientError), +} + impl From for ZcoinLightClientInitError { fn from(err: ZcashClientError) -> Self { ZcoinLightClientInitError::ZcashSqliteError(err) } } +impl From for ZcoinNativeClientInitError { + fn from(err: ZcashClientError) -> Self { ZcoinNativeClientInitError::ZcashSqliteError(err) } +} + #[derive(Debug, Display)] #[display(fmt = "Blockchain scan process stopped")] pub struct BlockchainScanStopped {} @@ -176,9 +195,9 @@ pub enum ZCoinBuildError { }, Io(std::io::Error), EmptyLightwalletdUris, - NativeModeIsNotSupportedYet, InvalidLightwalletdUri(InvalidUri), LightClientInitErr(ZcoinLightClientInitError), + NativeClientInitError(ZcoinNativeClientInitError), ZCashParamsNotFound, } @@ -205,3 +224,7 @@ impl From for ZCoinBuildError { impl From for ZCoinBuildError { fn from(err: ZcoinLightClientInitError) -> Self { ZCoinBuildError::LightClientInitErr(err) } } + +impl From for ZCoinBuildError { + fn from(err: ZcoinNativeClientInitError) -> Self { ZCoinBuildError::NativeClientInitError(err) } +} diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6a752bde42..d8a683e42b 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -2,7 +2,7 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; use crate::utxo::utxo_common; use common::executor::Timer; use common::log::{debug, error, info}; -use common::{async_blocking, spawn_abortable, AbortOnDropHandle}; +use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; @@ -12,7 +12,6 @@ use futures::StreamExt; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; -use prost::Message; use protobuf::Message as ProtobufMessage; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -32,6 +31,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } +use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -174,7 +174,6 @@ pub(super) async fn init_light_client( .connect() .await .map_to_mm(ZcoinLightClientInitError::ConnectionFailure)?; - let grpc_client = CompactTxStreamerClient::new(tonic_channel); let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -182,7 +181,55 @@ pub(super) async fn init_light_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - grpc_client, + rpc_client: RpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), + blocks_db, + wallet_db: wallet_db.clone(), + consensus_params, + sync_status_notifier, + on_tx_gen_watcher, + watch_for_tx: None, + }; + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle)); + + Ok(( + SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), + wallet_db, + )) +} + +pub(super) async fn init_native_client( + native_client: NativeClient, + cache_db_path: PathBuf, + wallet_db_path: PathBuf, + consensus_params: ZcoinConsensusParams, + evk: ExtendedFullViewingKey, +) -> Result<(AsyncMutex, WalletDbShared), MmError> { + let blocks_db = + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinNativeClientInitError::BlocksDbInitFailure)) + .await?; + + let wallet_db = async_blocking({ + let consensus_params = consensus_params.clone(); + move || -> Result<_, MmError> { + let db = WalletDb::for_path(wallet_db_path, consensus_params) + .map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + if db.get_extended_full_viewing_keys()?.is_empty() { + init_accounts_table(&db, &[evk])?; + } + Ok(db) + } + }) + .await?; + + let (sync_status_notifier, sync_watcher) = channel(1); + let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); + + let wallet_db = Arc::new(Mutex::new(wallet_db)); + let sync_handle = SaplingSyncLoopHandle { + current_block: BlockHeight::from_u32(0), + rpc_client: RpcClient::Native(native_client), blocks_db, wallet_db: wallet_db.clone(), consensus_params, @@ -241,9 +288,14 @@ pub enum SyncStatus { }, } +enum RpcClient { + Native(NativeClient), + Light(CompactTxStreamerClient), +} + pub struct SaplingSyncLoopHandle { current_block: BlockHeight, - grpc_client: CompactTxStreamerClient, + rpc_client: RpcClient, blocks_db: BlockDb, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, @@ -292,13 +344,17 @@ impl SaplingSyncLoopHandle { } async fn update_blocks_cache(&mut self) -> Result<(), MmError> { - let request = tonic::Request::new(ChainSpec {}); - let current_blockchain_block = self.grpc_client.get_latest_block(request).await?; + let current_block = match &self.rpc_client { + RpcClient::Native(client) => client.get_block_count().compat().await?, + RpcClient::Light(client) => { + let request = tonic::Request::new(ChainSpec {}); + let block = client.get_latest_block(request).await?; + let res: u64 = block.into_inner().height; + res + }, + }; let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; - let from_block = current_block_in_db as u64 + 1; - let current_block: u64 = current_blockchain_block.into_inner().height; - if current_block >= from_block { let request = tonic::Request::new(BlockRange { start: Some(BlockId { From b76ff673e58a0cc5eb216f85a05bbfdc40c09dcd Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 15 Jul 2022 11:35:25 +0700 Subject: [PATCH 02/52] WIP match added in current_block check --- mm2src/coins/z_coin/z_rpc.rs | 64 ++++++++++++++++++++++-------------- 1 file changed, 39 insertions(+), 25 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index d8a683e42b..8099d1944b 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -12,6 +12,7 @@ use futures::StreamExt; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; +use prost::Message; use protobuf::Message as ProtobufMessage; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -181,7 +182,7 @@ pub(super) async fn init_light_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - rpc_client: RpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), + rpc_client: ZRpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), blocks_db, wallet_db: wallet_db.clone(), consensus_params, @@ -229,7 +230,7 @@ pub(super) async fn init_native_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - rpc_client: RpcClient::Native(native_client), + rpc_client: ZRpcClient::Native(native_client), blocks_db, wallet_db: wallet_db.clone(), consensus_params, @@ -288,14 +289,24 @@ pub enum SyncStatus { }, } -enum RpcClient { +enum ZRpcClient { Native(NativeClient), Light(CompactTxStreamerClient), } +struct BlockIDNative { + height: u32, + hash: Vec, +} + +struct BlockRangeNative { + start: BlockIDNative, + end: BlockIDNative, +} + pub struct SaplingSyncLoopHandle { current_block: BlockHeight, - rpc_client: RpcClient, + rpc_client: ZRpcClient, blocks_db: BlockDb, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, @@ -344,9 +355,9 @@ impl SaplingSyncLoopHandle { } async fn update_blocks_cache(&mut self) -> Result<(), MmError> { - let current_block = match &self.rpc_client { - RpcClient::Native(client) => client.get_block_count().compat().await?, - RpcClient::Light(client) => { + let current_block = match &mut self.rpc_client { + ZRpcClient::Native(client) => client.get_block_count().compat().await?, + ZRpcClient::Light(client) => { let request = tonic::Request::new(ChainSpec {}); let block = client.get_latest_block(request).await?; let res: u64 = block.into_inner().height; @@ -356,23 +367,26 @@ impl SaplingSyncLoopHandle { let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; let from_block = current_block_in_db as u64 + 1; if current_block >= from_block { - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: from_block, - hash: Vec::new(), - }), - end: Some(BlockId { - height: current_block, - hash: Vec::new(), - }), - }); - - let mut response = self.grpc_client.get_block_range(request).await?; - - while let Some(block) = response.get_mut().message().await? { - debug!("Got block {:?}", block); - block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; - self.notify_blocks_cache_status(block.height, current_block); + match &mut self.rpc_client { + ZRpcClient::Light(client) => { + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height: from_block, + hash: Vec::new(), + }), + end: Some(BlockId { + height: current_block, + hash: Vec::new(), + }), + }); + let mut response = client.get_block_range(request).await?; + while let Some(block) = response.get_mut().message().await? { + debug!("Got block {:?}", block); + block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; + self.notify_blocks_cache_status(block.height, current_block); + } + }, + ZRpcClient::Native(client) => unreachable!(), } } @@ -419,7 +433,7 @@ impl SaplingSyncLoopHandle { hash: tx_id.0.into(), }; let request = tonic::Request::new(filter); - match self.grpc_client.get_transaction(request).await { + match self.rpc_client.get_transaction(request).await { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); From ae6a842fd94fb4ada9390fa6f08d30cccc44d13f Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 15 Jul 2022 14:50:20 +0700 Subject: [PATCH 03/52] WIP trying trait Client --- mm2src/coins/z_coin/z_rpc.rs | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 8099d1944b..abd6c6c92a 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -318,6 +318,28 @@ pub struct SaplingSyncLoopHandle { watch_for_tx: Option, } +// #[async_trait] +// trait Client { +// async fn get_block_height(&mut self) -> Result>; +// } +// +// #[async_trait] +// impl Client for NativeClient { +// async fn get_block_height(&mut self) -> Result> { +// self.get_block_count().compat().await +// } +// } +// +// #[async_trait] +// impl Client for CompactTxStreamerClient { +// async fn get_block_height(&mut self) -> Result> { +// let request = tonic::Request::new(ChainSpec {}); +// let block = self.get_latest_block(request).await?; +// let res = block.into_inner().height; +// Ok(res) +// } +// } + impl SaplingSyncLoopHandle { fn notify_blocks_cache_status(&mut self, current_scanned_block: u64, latest_block: u64) { if self From 646126d8a236c4a3247d21d11437e717585046f0 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 15 Jul 2022 16:38:40 +0700 Subject: [PATCH 04/52] WIP serialized_block added --- mm2src/coins/utxo/rpc_clients.rs | 4 ++-- mm2src/coins/z_coin/z_coin_errors.rs | 1 + mm2src/coins/z_coin/z_rpc.rs | 13 +++++++++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index fe9d5f819c..4b5e016db8 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -483,14 +483,14 @@ pub enum EstimateFeeMethod { SmartFee, } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] #[serde(untagged)] pub enum BlockNonce { String(String), U64(u64), } -#[derive(Debug, Deserialize)] +#[derive(Debug, Deserialize, Serialize)] pub struct VerboseBlock { /// Block hash pub hash: H256Json, diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index a2902bf158..cc9e7b0708 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -17,6 +17,7 @@ pub enum UpdateBlocksCacheErr { GrpcError(tonic::Status), DbError(SqliteError), JsonRpsErr(UtxoRpcError), + InternalError(String), } impl From for UpdateBlocksCacheErr { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index abd6c6c92a..a94c53c607 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,5 +1,6 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; use crate::utxo::utxo_common; +use bincode::serialize; use common::executor::Timer; use common::log::{debug, error, info}; use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; @@ -408,10 +409,18 @@ impl SaplingSyncLoopHandle { self.notify_blocks_cache_status(block.height, current_block); } }, - ZRpcClient::Native(client) => unreachable!(), + ZRpcClient::Native(client) => { + for height in from_block..=current_block { + let block = client.get_block_by_height(height).await?; + debug!("Got block {:?}", block); + let serialized_block = + serialize(&block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; + block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; + self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); + } + }, } } - self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) } From ddcffdb4d55b8b066cf72032930f73d76fe0ae26 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 15 Jul 2022 16:48:58 +0700 Subject: [PATCH 05/52] WIP remove BlockIDNative, BlockRangeNative --- mm2src/coins/z_coin/z_rpc.rs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index a94c53c607..af2cec51fb 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -295,16 +295,6 @@ enum ZRpcClient { Light(CompactTxStreamerClient), } -struct BlockIDNative { - height: u32, - hash: Vec, -} - -struct BlockRangeNative { - start: BlockIDNative, - end: BlockIDNative, -} - pub struct SaplingSyncLoopHandle { current_block: BlockHeight, rpc_client: ZRpcClient, From f70493f3802a18434147ea881157b67e4eb31af7 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 18 Jul 2022 13:53:18 +0700 Subject: [PATCH 06/52] WIP use if let --- mm2src/coins/z_coin/z_rpc.rs | 56 +++++++++++++++++------------------- 1 file changed, 27 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index af2cec51fb..7a49c2c465 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -380,35 +380,33 @@ impl SaplingSyncLoopHandle { let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; let from_block = current_block_in_db as u64 + 1; if current_block >= from_block { - match &mut self.rpc_client { - ZRpcClient::Light(client) => { - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: from_block, - hash: Vec::new(), - }), - end: Some(BlockId { - height: current_block, - hash: Vec::new(), - }), - }); - let mut response = client.get_block_range(request).await?; - while let Some(block) = response.get_mut().message().await? { - debug!("Got block {:?}", block); - block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; - self.notify_blocks_cache_status(block.height, current_block); - } - }, - ZRpcClient::Native(client) => { - for height in from_block..=current_block { - let block = client.get_block_by_height(height).await?; - debug!("Got block {:?}", block); - let serialized_block = - serialize(&block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; - block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; - self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); - } - }, + if let ZRpcClient::Light(client) = &mut self.rpc_client { + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height: from_block, + hash: Vec::new(), + }), + end: Some(BlockId { + height: current_block, + hash: Vec::new(), + }), + }); + let mut response = client.get_block_range(request).await?; + while let Some(block) = response.get_mut().message().await? { + debug!("Got block {:?}", block); + block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; + self.notify_blocks_cache_status(block.height, current_block); + } + } + if let ZRpcClient::Native(client) = &self.rpc_client { + for height in from_block..=current_block { + let block = client.get_block_by_height(height).await?; + debug!("Got block {:?}", block); + let serialized_block = + serialize(&block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; + block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; + self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); + } } } self.current_block = BlockHeight::from_u32(current_block as u32); From 7724cb76cd732ed3d9a2949b30fddacb00cfe7e0 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 18 Jul 2022 14:49:40 +0700 Subject: [PATCH 07/52] WIP merge fix --- mm2src/coins/z_coin/z_coin_errors.rs | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index bfe13b1b19..6497d07e2e 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -18,6 +18,8 @@ use zcash_primitives::transaction::builder::Error as ZTxBuilderError; pub enum UpdateBlocksCacheErr { GrpcError(tonic::Status), DbError(SqliteError), + JsonRpsErr(UtxoRpcError), + InternalError(String), } impl From for UpdateBlocksCacheErr { @@ -28,6 +30,10 @@ impl From for UpdateBlocksCacheErr { fn from(err: SqliteError) -> Self { UpdateBlocksCacheErr::DbError(err) } } +impl From for UpdateBlocksCacheErr { + fn from(err: UtxoRpcError) -> Self { UpdateBlocksCacheErr::JsonRpsErr(err) } +} + #[derive(Debug, Display)] #[non_exhaustive] pub enum ZcoinLightClientInitError { @@ -38,10 +44,24 @@ pub enum ZcoinLightClientInitError { ZcashSqliteError(ZcashClientError), } +#[derive(Debug, Display)] +#[non_exhaustive] +pub enum ZcoinNativeClientInitError { + TlsConfigFailure(tonic::transport::Error), + ConnectionFailure(tonic::transport::Error), + BlocksDbInitFailure(SqliteError), + WalletDbInitFailure(SqliteError), + ZcashSqliteError(ZcashClientError), +} + impl From for ZcoinLightClientInitError { fn from(err: ZcashClientError) -> Self { ZcoinLightClientInitError::ZcashSqliteError(err) } } +impl From for ZcoinNativeClientInitError { + fn from(err: ZcashClientError) -> Self { ZcoinNativeClientInitError::ZcashSqliteError(err) } +} + #[derive(Debug, Display)] #[display(fmt = "Blockchain scan process stopped")] pub struct BlockchainScanStopped {} @@ -181,6 +201,7 @@ pub enum ZCoinBuildError { NativeModeIsNotSupportedYet, InvalidLightwalletdUri(InvalidUri), LightClientInitErr(ZcoinLightClientInitError), + NativeClientInitError(ZcoinNativeClientInitError), ZCashParamsNotFound, } @@ -208,6 +229,10 @@ impl From for ZCoinBuildError { fn from(err: ZcoinLightClientInitError) -> Self { ZCoinBuildError::LightClientInitErr(err) } } +impl From for ZCoinBuildError { + fn from(err: ZcoinNativeClientInitError) -> Self { ZCoinBuildError::NativeClientInitError(err) } +} + pub(super) enum SqlTxHistoryError { Sql(SqlError), FromIdDoesNotExist(i64), From 7f0000531ce0565991cb6022b141f49e6b897e77 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 18 Jul 2022 17:23:51 +0700 Subject: [PATCH 08/52] WIP if let added in check_watch_for_tx_existence --- mm2src/coins/z_coin/z_rpc.rs | 43 ++++++++++++++++++++---------------- 1 file changed, 24 insertions(+), 19 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 7a49c2c465..00d120dc98 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -445,28 +445,33 @@ impl SaplingSyncLoopHandle { async fn check_watch_for_tx_existence(&mut self) { if let Some(tx_id) = self.watch_for_tx { let mut attempts = 0; - loop { - let filter = TxFilter { - block: None, - index: 0, - hash: tx_id.0.into(), - }; - let request = tonic::Request::new(filter); - match self.rpc_client.get_transaction(request).await { - Ok(_) => break, - Err(e) => { - error!("Error on getting tx {}", tx_id); - if e.message().contains(utxo_common::NO_TX_ERROR_CODE) { - if attempts >= 3 { - self.watch_for_tx = None; - return; + if let ZRpcClient::Light(client) = &mut self.rpc_client { + loop { + let filter = TxFilter { + block: None, + index: 0, + hash: tx_id.0.into(), + }; + let request = tonic::Request::new(filter); + match client.get_transaction(request).await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.message().contains(utxo_common::NO_TX_ERROR_CODE) { + if attempts >= 3 { + self.watch_for_tx = None; + return; + } + attempts += 1; } - attempts += 1; - } - Timer::sleep(30.).await; - }, + Timer::sleep(30.).await; + }, + } } } + if let ZRpcClient::Native(client) = &self.rpc_client { + todo!() + } } } } From 187bb7d20dff4974f231452845d31e84bfd9b494 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 19 Jul 2022 13:26:03 +0700 Subject: [PATCH 09/52] WIP client.clone() added --- mm2src/coins/z_coin/z_rpc.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 00d120dc98..830fef407e 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -399,6 +399,7 @@ impl SaplingSyncLoopHandle { } } if let ZRpcClient::Native(client) = &self.rpc_client { + let client = client.clone(); for height in from_block..=current_block { let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); @@ -469,7 +470,7 @@ impl SaplingSyncLoopHandle { } } } - if let ZRpcClient::Native(client) = &self.rpc_client { + if let ZRpcClient::Native(_client) = &self.rpc_client { todo!() } } From 10e97901a9ef328f4da281bbca517a9954edc984 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 19 Jul 2022 17:31:35 +0700 Subject: [PATCH 10/52] WIP match for get_raw_tx added --- mm2src/coins/z_coin/z_rpc.rs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 830fef407e..3b1024aaa0 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -470,8 +470,16 @@ impl SaplingSyncLoopHandle { } } } - if let ZRpcClient::Native(_client) = &self.rpc_client { - todo!() + if let ZRpcClient::Native(client) = &self.rpc_client { + loop { + match client.get_raw_transaction_bytes(&tx_id.0.into()).compat().await { + Ok(_) => break, + Err(_e) => { + error!("Error on getting tx {}", tx_id); + todo!() + }, + } + } } } } From df68b93e3642397d969f00fcb7e038cfb4875cec Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 19 Jul 2022 18:51:13 +0700 Subject: [PATCH 11/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 3b1024aaa0..74cbad4334 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -474,9 +474,16 @@ impl SaplingSyncLoopHandle { loop { match client.get_raw_transaction_bytes(&tx_id.0.into()).compat().await { Ok(_) => break, - Err(_e) => { + Err(e) => { error!("Error on getting tx {}", tx_id); - todo!() + if e.to_string().contains(utxo_common::NO_TX_ERROR_CODE) { + if attempts >= 3 { + self.watch_for_tx = None; + return; + } + attempts += 1; + } + Timer::sleep(30.).await; }, } } From 3fc0ed98e55aafd8c0c53d68bad7237adc9432cf Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 20 Jul 2022 16:57:22 +0700 Subject: [PATCH 12/52] [r2r] trait impl comment removed --- mm2src/coins/z_coin/z_rpc.rs | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 74cbad4334..229dde628a 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -309,28 +309,6 @@ pub struct SaplingSyncLoopHandle { watch_for_tx: Option, } -// #[async_trait] -// trait Client { -// async fn get_block_height(&mut self) -> Result>; -// } -// -// #[async_trait] -// impl Client for NativeClient { -// async fn get_block_height(&mut self) -> Result> { -// self.get_block_count().compat().await -// } -// } -// -// #[async_trait] -// impl Client for CompactTxStreamerClient { -// async fn get_block_height(&mut self) -> Result> { -// let request = tonic::Request::new(ChainSpec {}); -// let block = self.get_latest_block(request).await?; -// let res = block.into_inner().height; -// Ok(res) -// } -// } - impl SaplingSyncLoopHandle { fn notify_blocks_cache_status(&mut self, current_scanned_block: u64, latest_block: u64) { if self From f7cb333b176d6e5fa30ea810253902e867fcdf4d Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Jul 2022 15:31:38 +0700 Subject: [PATCH 13/52] WIP --- mm2src/coins/z_coin/z_coin_errors.rs | 10 ++++- mm2src/coins/z_coin/z_rpc.rs | 58 +++++++++++++++++++++++++++- 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 6497d07e2e..208c77eb19 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -3,6 +3,7 @@ use crate::utxo::rpc_clients::UtxoRpcError; use crate::utxo::utxo_builder::UtxoCoinBuildError; use crate::WithdrawError; use crate::{NumConversError, PrivKeyNotAllowed}; +use common::jsonrpc_client::JsonRpcError; use db_common::sqlite::rusqlite::Error as SqliteError; use db_common::sqlite::rusqlite::Error as SqlError; use derive_more::Display; @@ -18,8 +19,9 @@ use zcash_primitives::transaction::builder::Error as ZTxBuilderError; pub enum UpdateBlocksCacheErr { GrpcError(tonic::Status), DbError(SqliteError), - JsonRpsErr(UtxoRpcError), + UtxoRpcError(UtxoRpcError), InternalError(String), + JsonRpcError(JsonRpcError), } impl From for UpdateBlocksCacheErr { @@ -31,7 +33,11 @@ impl From for UpdateBlocksCacheErr { } impl From for UpdateBlocksCacheErr { - fn from(err: UtxoRpcError) -> Self { UpdateBlocksCacheErr::JsonRpsErr(err) } + fn from(err: UtxoRpcError) -> Self { UpdateBlocksCacheErr::UtxoRpcError(err) } +} + +impl From for UpdateBlocksCacheErr { + fn from(err: JsonRpcError) -> Self { UpdateBlocksCacheErr::JsonRpcError(err) } } #[derive(Debug, Display)] diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 229dde628a..fbc87079fe 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -34,6 +34,7 @@ mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; +use crate::{BytesJson, H256Json}; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -290,6 +291,45 @@ pub enum SyncStatus { }, } +#[derive(Debug, Deserialize, Serialize)] +struct CompactBlockNative { + height: u64, // the height of this block + hash: H256Json, + prev_hash: H256Json, + time: u32, + header: BytesJson, // (hash, prev_hash, and time) OR (full header) + compact_tx: Vec, // compact transactions from this block +} + +#[derive(Debug, Deserialize, Serialize)] +struct CompactTxNative { + // Index and hash will allow the receiver to call out to chain + // explorers or other data structures to retrieve more information + // about this transaction. + index: H256Json, + hash: H256Json, + // The transaction fee: present if server can provide. In the case of a + // stateless server and a transaction with transparent inputs, this will be + // unset because the calculation requires reference to prior transactions. + // in a pure-Sapling context, the fee will be calculable as: + // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) + fee: u32, + spends: Vec, + outputs: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct CompactSpendNative { + nf: Vec, +} + +#[derive(Debug, Deserialize, Serialize)] +struct CompactOutputNative { + cmu: Vec, + epk: Vec, + ciphertext: Vec, +} + enum ZRpcClient { Native(NativeClient), Light(CompactTxStreamerClient), @@ -381,8 +421,24 @@ impl SaplingSyncLoopHandle { for height in from_block..=current_block { let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); + let header = client.get_block_header_bytes(block.hash).compat().await?; + let compact_tx = CompactTxNative { + index: Default::default(), + hash: Default::default(), + fee: 0, + spends: vec![], + outputs: vec![], + }; + let compact_block = CompactBlockNative { + height, + hash: block.hash, + prev_hash: block.previousblockhash.unwrap(), + time: block.time, + header, + compact_tx: vec![compact_tx], + }; let serialized_block = - serialize(&block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; + serialize(&compact_block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); } From 3265803874b914e9263e03c05861dce1ff16813a Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Jul 2022 18:14:32 +0700 Subject: [PATCH 14/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 39 +++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 12 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index fbc87079fe..9a5b17d9f4 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -15,6 +15,7 @@ use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; +use serialization::deserialize; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -34,7 +35,7 @@ mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; -use crate::{BytesJson, H256Json}; +use crate::{BytesJson, H256Json, UtxoTx}; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -297,8 +298,8 @@ struct CompactBlockNative { hash: H256Json, prev_hash: H256Json, time: u32, - header: BytesJson, // (hash, prev_hash, and time) OR (full header) - compact_tx: Vec, // compact transactions from this block + header: BytesJson, // (hash, prev_hash, and time) OR (full header) + compact_txs: Vec, // compact transactions from this block } #[derive(Debug, Deserialize, Serialize)] @@ -306,7 +307,7 @@ struct CompactTxNative { // Index and hash will allow the receiver to call out to chain // explorers or other data structures to retrieve more information // about this transaction. - index: H256Json, + index: u64, hash: H256Json, // The transaction fee: present if server can provide. In the case of a // stateless server and a transaction with transparent inputs, this will be @@ -421,21 +422,35 @@ impl SaplingSyncLoopHandle { for height in from_block..=current_block { let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); + let mut tx_id: u64 = 0; + let mut compact_txs: Vec = Vec::new(); + // create and push compact_tx during iteration + for hash in &block.tx { + let tx = client.get_transaction_bytes(hash).compat().await?; + let tx: UtxoTx = deserialize(tx.as_slice()).expect("Panic here to avoid invalid tree state"); + let mut spends: Vec = Vec::new(); + let mut outputs: Vec = Vec::new(); + // create outs, spends + for spend in tx.shielded_spends {} + for out in tx.shielded_outputs {} + tx_id += 1; + let compact_tx = CompactTxNative { + index: tx_id, + hash: *hash, + fee: 0, + spends, + outputs, + }; + compact_txs.push(compact_tx); + } let header = client.get_block_header_bytes(block.hash).compat().await?; - let compact_tx = CompactTxNative { - index: Default::default(), - hash: Default::default(), - fee: 0, - spends: vec![], - outputs: vec![], - }; let compact_block = CompactBlockNative { height, hash: block.hash, prev_hash: block.previousblockhash.unwrap(), time: block.time, header, - compact_tx: vec![compact_tx], + compact_txs, }; let serialized_block = serialize(&compact_block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; From 5d5ce178fd9e0e2453ad7475b7e8a25dcec25695 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Jul 2022 18:18:40 +0700 Subject: [PATCH 15/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 9a5b17d9f4..6409281349 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -15,7 +15,6 @@ use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; -use serialization::deserialize; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -35,7 +34,7 @@ mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; -use crate::{BytesJson, H256Json, UtxoTx}; +use crate::{BytesJson, H256Json, ZTransaction}; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -426,13 +425,13 @@ impl SaplingSyncLoopHandle { let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration for hash in &block.tx { - let tx = client.get_transaction_bytes(hash).compat().await?; - let tx: UtxoTx = deserialize(tx.as_slice()).expect("Panic here to avoid invalid tree state"); + let tx_bytes = client.get_transaction_bytes(hash).compat().await?; + let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); // create outs, spends - for spend in tx.shielded_spends {} - for out in tx.shielded_outputs {} + for spend in &tx.shielded_spends {} + for out in &tx.shielded_outputs {} tx_id += 1; let compact_tx = CompactTxNative { index: tx_id, From c200dab38c050f497a10a3766f1a717d8d3a08c5 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 26 Jul 2022 18:26:27 +0700 Subject: [PATCH 16/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6409281349..aa4af0ddae 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -429,10 +429,12 @@ impl SaplingSyncLoopHandle { let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); - // create outs, spends + // create and push outs and spends during iteration for spend in &tx.shielded_spends {} for out in &tx.shielded_outputs {} tx_id += 1; + let spends = spends; + let outputs= outputs; let compact_tx = CompactTxNative { index: tx_id, hash: *hash, @@ -443,6 +445,7 @@ impl SaplingSyncLoopHandle { compact_txs.push(compact_tx); } let header = client.get_block_header_bytes(block.hash).compat().await?; + let compact_txs = compact_txs; let compact_block = CompactBlockNative { height, hash: block.hash, From efbe95b4c5cd1cb3343deecbaf83c251741de920 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 27 Jul 2022 15:10:20 +0700 Subject: [PATCH 17/52] WIP no method named `to_bytes`jubjub --- mm2src/coins/z_coin/z_rpc.rs | 41 +++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index aa4af0ddae..28530acb66 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -291,7 +291,7 @@ pub enum SyncStatus { }, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Serialize)] struct CompactBlockNative { height: u64, // the height of this block hash: H256Json, @@ -301,7 +301,7 @@ struct CompactBlockNative { compact_txs: Vec, // compact transactions from this block } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Serialize)] struct CompactTxNative { // Index and hash will allow the receiver to call out to chain // explorers or other data structures to retrieve more information @@ -318,16 +318,17 @@ struct CompactTxNative { outputs: Vec, } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Serialize)] struct CompactSpendNative { - nf: Vec, + nf: [u8; 32], } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Serialize)] struct CompactOutputNative { - cmu: Vec, - epk: Vec, - ciphertext: Vec, + cmu: [u8; 32], + epk: [u8; 32], + #[serde(serialize_with = "<[_]>::serialize")] + ciphertext: [u8; 80], } enum ZRpcClient { @@ -424,20 +425,30 @@ impl SaplingSyncLoopHandle { let mut tx_id: u64 = 0; let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration - for hash in &block.tx { - let tx_bytes = client.get_transaction_bytes(hash).compat().await?; + for hash_tx in &block.tx { + let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); - // create and push outs and spends during iteration - for spend in &tx.shielded_spends {} - for out in &tx.shielded_outputs {} + // create and push outs and spends during iterations + for spend in &tx.shielded_spends { + let compact_spend = CompactSpendNative { nf: spend.nullifier.0 }; + spends.push(compact_spend); + } + for out in &tx.shielded_outputs { + let compact_out = CompactOutputNative { + cmu: out.cmu.to_bytes(), + epk: out.ephemeral_key.to_bytes(), + ciphertext: out.out_ciphertext, + }; + outputs.push(compact_out); + } tx_id += 1; let spends = spends; - let outputs= outputs; + let outputs = outputs; let compact_tx = CompactTxNative { index: tx_id, - hash: *hash, + hash: *hash_tx, fee: 0, spends, outputs, From e235c03f70de94ae8863ba19204a0b5e2749a670 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 27 Jul 2022 16:37:09 +0700 Subject: [PATCH 18/52] WIP group crate added --- Cargo.lock | 1 + mm2src/coins/Cargo.toml | 1 + mm2src/coins/z_coin/z_rpc.rs | 1 + 3 files changed, 3 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ec11587fa3..b959664ab7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1054,6 +1054,7 @@ dependencies = [ "ethkey", "futures 0.1.29", "futures 0.3.15", + "group 0.8.0", "gstuff", "hex 0.4.2", "http 0.2.7", diff --git a/mm2src/coins/Cargo.toml b/mm2src/coins/Cargo.toml index d541a1153b..19b83e0d96 100644 --- a/mm2src/coins/Cargo.toml +++ b/mm2src/coins/Cargo.toml @@ -41,6 +41,7 @@ ethkey = { git = "https://github.com/artemii235/parity-ethereum.git" } futures01 = { version = "0.1", package = "futures" } # using select macro requires the crate to be named futures, compilation failed with futures03 name futures = { version = "0.3", package = "futures", features = ["compat", "async-await"] } +group = "0.8.0" gstuff = { version = "0.7", features = ["nightly"] } hex = "0.4.2" http = "0.2" diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 28530acb66..0028e901d4 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -10,6 +10,7 @@ use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as Async use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; +use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; From 408504695e7d92e10af4673294260f9d80d2d946 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 27 Jul 2022 17:44:15 +0700 Subject: [PATCH 19/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 0028e901d4..b37582474f 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -28,6 +28,7 @@ use zcash_client_sqlite::error::SqliteClientError as ZcashClientError; use zcash_client_sqlite::wallet::init::{init_accounts_table, init_wallet_db}; use zcash_client_sqlite::WalletDb; use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; @@ -294,19 +295,16 @@ pub enum SyncStatus { #[derive(Debug, Serialize)] struct CompactBlockNative { - height: u64, // the height of this block + height: u64, hash: H256Json, prev_hash: H256Json, time: u32, - header: BytesJson, // (hash, prev_hash, and time) OR (full header) - compact_txs: Vec, // compact transactions from this block + header: BytesJson, + compact_txs: Vec, } #[derive(Debug, Serialize)] struct CompactTxNative { - // Index and hash will allow the receiver to call out to chain - // explorers or other data structures to retrieve more information - // about this transaction. index: u64, hash: H256Json, // The transaction fee: present if server can provide. In the case of a @@ -314,7 +312,7 @@ struct CompactTxNative { // unset because the calculation requires reference to prior transactions. // in a pure-Sapling context, the fee will be calculable as: // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) - fee: u32, + fee: u64, spends: Vec, outputs: Vec, } @@ -447,10 +445,22 @@ impl SaplingSyncLoopHandle { tx_id += 1; let spends = spends; let outputs = outputs; + let mut transparent_input_amount = Amount::zero(); + for input in tx.vin.iter() { + let hash = H256Json::from(*input.prevout.hash()); + let prev_tx_bytes = client.get_transaction_bytes(&hash).compat().await?; + let prev_tx = ZTransaction::read(prev_tx_bytes.as_slice()).unwrap(); + if let Some(spent_output) = prev_tx.vout.get(input.prevout.n() as usize) { + transparent_input_amount += spent_output.value; + } + } + let transparent_output_amount = + tx.vout.iter().fold(Amount::zero(), |current, out| current + out.value); + let fee_amount = tx.value_balance + transparent_input_amount - transparent_output_amount; let compact_tx = CompactTxNative { index: tx_id, hash: *hash_tx, - fee: 0, + fee: fee_amount.into(), spends, outputs, }; From c08e3ffd59453e43a21464092666adb3af47bfd3 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 29 Jul 2022 12:05:15 +0700 Subject: [PATCH 20/52] WIP merge fixes --- mm2src/coins/z_coin/z_rpc.rs | 275 +++++++++++++++++++++++++++++------ 1 file changed, 229 insertions(+), 46 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 404f229405..6150f14328 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,14 +1,16 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; use crate::utxo::rpc_clients; +use bincode::serialize; use common::executor::Timer; use common::log::{debug, error, info}; -use common::{async_blocking, spawn_abortable, AbortOnDropHandle}; +use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; +use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; @@ -26,12 +28,15 @@ use zcash_client_sqlite::error::SqliteClientError as ZcashClientError; use zcash_client_sqlite::wallet::init::{init_accounts_table, init_wallet_db}; use zcash_client_sqlite::WalletDb; use zcash_primitives::consensus::BlockHeight; +use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } +use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; +use crate::{BytesJson, H256Json, ZTransaction}; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -174,7 +179,6 @@ pub(super) async fn init_light_client( .connect() .await .map_to_mm(ZcoinLightClientInitError::ConnectionFailure)?; - let grpc_client = CompactTxStreamerClient::new(tonic_channel); let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -182,7 +186,55 @@ pub(super) async fn init_light_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - grpc_client, + rpc_client: ZRpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), + blocks_db, + wallet_db: wallet_db.clone(), + consensus_params, + sync_status_notifier, + on_tx_gen_watcher, + watch_for_tx: None, + }; + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle)); + + Ok(( + SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), + wallet_db, + )) +} + +pub(super) async fn init_native_client( + native_client: NativeClient, + cache_db_path: PathBuf, + wallet_db_path: PathBuf, + consensus_params: ZcoinConsensusParams, + evk: ExtendedFullViewingKey, +) -> Result<(AsyncMutex, WalletDbShared), MmError> { + let blocks_db = + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinNativeClientInitError::BlocksDbInitFailure)) + .await?; + + let wallet_db = async_blocking({ + let consensus_params = consensus_params.clone(); + move || -> Result<_, MmError> { + let db = WalletDb::for_path(wallet_db_path, consensus_params) + .map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + if db.get_extended_full_viewing_keys()?.is_empty() { + init_accounts_table(&db, &[evk])?; + } + Ok(db) + } + }) + .await?; + + let (sync_status_notifier, sync_watcher) = channel(1); + let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); + + let wallet_db = Arc::new(Mutex::new(wallet_db)); + let sync_handle = SaplingSyncLoopHandle { + current_block: BlockHeight::from_u32(0), + rpc_client: ZRpcClient::Native(native_client), blocks_db, wallet_db: wallet_db.clone(), consensus_params, @@ -241,9 +293,51 @@ pub enum SyncStatus { }, } +#[derive(Debug, Serialize)] +struct CompactBlockNative { + height: u64, + hash: H256Json, + prev_hash: H256Json, + time: u32, + header: BytesJson, + compact_txs: Vec, +} + +#[derive(Debug, Serialize)] +struct CompactTxNative { + index: u64, + hash: H256Json, + // The transaction fee: present if server can provide. In the case of a + // stateless server and a transaction with transparent inputs, this will be + // unset because the calculation requires reference to prior transactions. + // in a pure-Sapling context, the fee will be calculable as: + // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) + fee: u64, + spends: Vec, + outputs: Vec, +} + +#[derive(Debug, Serialize)] +struct CompactSpendNative { + nf: [u8; 32], +} + +#[derive(Debug, Serialize)] +struct CompactOutputNative { + cmu: [u8; 32], + epk: [u8; 32], + #[serde(serialize_with = "<[_]>::serialize")] + ciphertext: [u8; 80], +} + +enum ZRpcClient { + Native(NativeClient), + Light(CompactTxStreamerClient), +} + pub struct SaplingSyncLoopHandle { current_block: BlockHeight, - grpc_client: CompactTxStreamerClient, + rpc_client: ZRpcClient, blocks_db: BlockDb, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, @@ -292,34 +386,103 @@ impl SaplingSyncLoopHandle { } async fn update_blocks_cache(&mut self) -> Result<(), MmError> { - let request = tonic::Request::new(ChainSpec {}); - let current_blockchain_block = self.grpc_client.get_latest_block(request).await?; + let current_block = match &mut self.rpc_client { + ZRpcClient::Native(client) => client.get_block_count().compat().await?, + ZRpcClient::Light(client) => { + let request = tonic::Request::new(ChainSpec {}); + let block = client.get_latest_block(request).await?; + let res: u64 = block.into_inner().height; + res + }, + }; let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; - let from_block = current_block_in_db as u64 + 1; - let current_block: u64 = current_blockchain_block.into_inner().height; - if current_block >= from_block { - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: from_block, - hash: Vec::new(), - }), - end: Some(BlockId { - height: current_block, - hash: Vec::new(), - }), - }); - - let mut response = self.grpc_client.get_block_range(request).await?; - - while let Some(block) = response.get_mut().message().await? { - debug!("Got block {:?}", block); - block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; - self.notify_blocks_cache_status(block.height, current_block); + if let ZRpcClient::Light(client) = &mut self.rpc_client { + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height: from_block, + hash: Vec::new(), + }), + end: Some(BlockId { + height: current_block, + hash: Vec::new(), + }), + }); + let mut response = client.get_block_range(request).await?; + while let Some(block) = response.get_mut().message().await? { + debug!("Got block {:?}", block); + block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; + self.notify_blocks_cache_status(block.height, current_block); + } + } + if let ZRpcClient::Native(client) = &self.rpc_client { + let client = client.clone(); + for height in from_block..=current_block { + let block = client.get_block_by_height(height).await?; + debug!("Got block {:?}", block); + let mut tx_id: u64 = 0; + let mut compact_txs: Vec = Vec::new(); + // create and push compact_tx during iteration + for hash_tx in &block.tx { + let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; + let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); + let mut spends: Vec = Vec::new(); + let mut outputs: Vec = Vec::new(); + // create and push outs and spends during iterations + for spend in &tx.shielded_spends { + let compact_spend = CompactSpendNative { nf: spend.nullifier.0 }; + spends.push(compact_spend); + } + for out in &tx.shielded_outputs { + let compact_out = CompactOutputNative { + cmu: out.cmu.to_bytes(), + epk: out.ephemeral_key.to_bytes(), + ciphertext: out.out_ciphertext, + }; + outputs.push(compact_out); + } + tx_id += 1; + let spends = spends; + let outputs = outputs; + let mut transparent_input_amount = Amount::zero(); + for input in tx.vin.iter() { + let hash = H256Json::from(*input.prevout.hash()); + let prev_tx_bytes = client.get_transaction_bytes(&hash).compat().await?; + let prev_tx = ZTransaction::read(prev_tx_bytes.as_slice()).unwrap(); + if let Some(spent_output) = prev_tx.vout.get(input.prevout.n() as usize) { + transparent_input_amount += spent_output.value; + } + } + let transparent_output_amount = + tx.vout.iter().fold(Amount::zero(), |current, out| current + out.value); + let fee_amount = tx.value_balance + transparent_input_amount - transparent_output_amount; + let compact_tx = CompactTxNative { + index: tx_id, + hash: *hash_tx, + fee: fee_amount.into(), + spends, + outputs, + }; + compact_txs.push(compact_tx); + } + let header = client.get_block_header_bytes(block.hash).compat().await?; + let compact_txs = compact_txs; + let compact_block = CompactBlockNative { + height, + hash: block.hash, + prev_hash: block.previousblockhash.unwrap(), + time: block.time, + header, + compact_txs, + }; + let serialized_block = + serialize(&compact_block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; + block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; + self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); + } } } - self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) } @@ -356,26 +519,46 @@ impl SaplingSyncLoopHandle { async fn check_watch_for_tx_existence(&mut self) { if let Some(tx_id) = self.watch_for_tx { let mut attempts = 0; - loop { - let filter = TxFilter { - block: None, - index: 0, - hash: tx_id.0.into(), - }; - let request = tonic::Request::new(filter); - match self.grpc_client.get_transaction(request).await { - Ok(_) => break, - Err(e) => { - error!("Error on getting tx {}", tx_id); - if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { - if attempts >= 3 { - self.watch_for_tx = None; - return; + if let ZRpcClient::Light(client) = &mut self.rpc_client { + loop { + let filter = TxFilter { + block: None, + index: 0, + hash: tx_id.0.into(), + }; + let request = tonic::Request::new(filter); + match client.get_transaction(request).await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { + if attempts >= 3 { + self.watch_for_tx = None; + return; + } + attempts += 1; } - attempts += 1; - } - Timer::sleep(30.).await; - }, + Timer::sleep(30.).await; + }, + } + } + } + if let ZRpcClient::Native(client) = &self.rpc_client { + loop { + match client.get_raw_transaction_bytes(&tx_id.0.into()).compat().await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.to_string().contains(rpc_clients::NO_TX_ERROR_CODE) { + if attempts >= 3 { + self.watch_for_tx = None; + return; + } + attempts += 1; + } + Timer::sleep(30.).await; + }, + } } } } From a8cbc72b7d186fba38b088bf2e7baec1f49818ef Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 1 Aug 2022 15:09:18 +0700 Subject: [PATCH 21/52] WIP log! added --- mm2src/coins/z_coin/z_rpc.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6150f14328..fcda28bfb1 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -395,6 +395,7 @@ impl SaplingSyncLoopHandle { res }, }; + log!("current_block = {:?}", current_block); let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; let from_block = current_block_in_db as u64 + 1; if current_block >= from_block { @@ -419,8 +420,10 @@ impl SaplingSyncLoopHandle { if let ZRpcClient::Native(client) = &self.rpc_client { let client = client.clone(); for height in from_block..=current_block { + log!("Height in current_block = {:?}", height); let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); + log!("Got block = {:?}", block); let mut tx_id: u64 = 0; let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration @@ -448,7 +451,9 @@ impl SaplingSyncLoopHandle { let mut transparent_input_amount = Amount::zero(); for input in tx.vin.iter() { let hash = H256Json::from(*input.prevout.hash()); + log!("Hash from prevout = {:?}", hash); let prev_tx_bytes = client.get_transaction_bytes(&hash).compat().await?; + log!("prev_tx_bytes = {:?}", prev_tx_bytes); let prev_tx = ZTransaction::read(prev_tx_bytes.as_slice()).unwrap(); if let Some(spent_output) = prev_tx.vout.get(input.prevout.n() as usize) { transparent_input_amount += spent_output.value; From fc78a139b94d42845e12243928ada4eeafbdc376 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 1 Aug 2022 18:16:19 +0700 Subject: [PATCH 22/52] WIP log --- mm2src/coins/z_coin/z_rpc.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index fcda28bfb1..fc7ec6840c 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -398,6 +398,7 @@ impl SaplingSyncLoopHandle { log!("current_block = {:?}", current_block); let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; let from_block = current_block_in_db as u64 + 1; + log!("from_block = {:?}", from_block); if current_block >= from_block { if let ZRpcClient::Light(client) = &mut self.rpc_client { let request = tonic::Request::new(BlockRange { @@ -420,7 +421,7 @@ impl SaplingSyncLoopHandle { if let ZRpcClient::Native(client) = &self.rpc_client { let client = client.clone(); for height in from_block..=current_block { - log!("Height in current_block = {:?}", height); + log!("Height in block = {:?}", height); let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); log!("Got block = {:?}", block); @@ -428,8 +429,10 @@ impl SaplingSyncLoopHandle { let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration for hash_tx in &block.tx { + log!("hash_tx in block.tx = {:?}", hash_tx); let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); + log!("tx ZTransaction = {:?}", tx); let mut spends: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); // create and push outs and spends during iterations @@ -446,10 +449,14 @@ impl SaplingSyncLoopHandle { outputs.push(compact_out); } tx_id += 1; + // making spends and outputs immutable let spends = spends; + log!("spends = {:?}", spends); let outputs = outputs; + log!("outputs = {:?}", outputs); let mut transparent_input_amount = Amount::zero(); for input in tx.vin.iter() { + log!("Input from tx.vin.iter = {:?}", input); let hash = H256Json::from(*input.prevout.hash()); log!("Hash from prevout = {:?}", hash); let prev_tx_bytes = client.get_transaction_bytes(&hash).compat().await?; From 82595c906b0398706c7253b04c1e5cbe24e82d24 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 1 Aug 2022 19:13:25 +0700 Subject: [PATCH 23/52] WIP verbose in getblockheader should be boolean --- mm2src/coins/utxo/rpc_clients.rs | 2 +- mm2src/coins/z_coin/z_rpc.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/utxo/rpc_clients.rs b/mm2src/coins/utxo/rpc_clients.rs index 0b2eb12aec..66a23f36a3 100644 --- a/mm2src/coins/utxo/rpc_clients.rs +++ b/mm2src/coins/utxo/rpc_clients.rs @@ -1124,7 +1124,7 @@ impl NativeClientImpl { /// https://developer.bitcoin.org/reference/rpc/getblockheader.html pub fn get_block_header_bytes(&self, block_hash: H256Json) -> RpcRes { - let verbose = 0; + let verbose = false; rpc_func!(self, "getblockheader", block_hash, verbose) } } diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index fc7ec6840c..9715f9136e 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -395,7 +395,7 @@ impl SaplingSyncLoopHandle { res }, }; - log!("current_block = {:?}", current_block); + log!("latest block height = {:?}", current_block); let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; let from_block = current_block_in_db as u64 + 1; log!("from_block = {:?}", from_block); @@ -421,7 +421,7 @@ impl SaplingSyncLoopHandle { if let ZRpcClient::Native(client) = &self.rpc_client { let client = client.clone(); for height in from_block..=current_block { - log!("Height in block = {:?}", height); + log!("Height = {:?}", height); let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); log!("Got block = {:?}", block); From 3bc4b3b29e78aef8f28e57af87dd16e2222b82fb Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 2 Aug 2022 14:51:23 +0700 Subject: [PATCH 24/52] WIP fee 0 for transparent inputs --- mm2src/coins/z_coin/z_rpc.rs | 26 ++++---------------------- 1 file changed, 4 insertions(+), 22 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 9715f9136e..fc1d0972a0 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -28,7 +28,6 @@ use zcash_client_sqlite::error::SqliteClientError as ZcashClientError; use zcash_client_sqlite::wallet::init::{init_accounts_table, init_wallet_db}; use zcash_client_sqlite::WalletDb; use zcash_primitives::consensus::BlockHeight; -use zcash_primitives::transaction::components::Amount; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; @@ -424,15 +423,15 @@ impl SaplingSyncLoopHandle { log!("Height = {:?}", height); let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); - log!("Got block = {:?}", block); + // log!("Got block = {:?}", block); let mut tx_id: u64 = 0; let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration for hash_tx in &block.tx { - log!("hash_tx in block.tx = {:?}", hash_tx); + // log!("hash_tx in block.tx = {:?}", hash_tx); let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); - log!("tx ZTransaction = {:?}", tx); + // log!("tx ZTransaction = {:?}", tx); let mut spends: Vec = Vec::new(); let mut outputs: Vec = Vec::new(); // create and push outs and spends during iterations @@ -451,28 +450,11 @@ impl SaplingSyncLoopHandle { tx_id += 1; // making spends and outputs immutable let spends = spends; - log!("spends = {:?}", spends); let outputs = outputs; - log!("outputs = {:?}", outputs); - let mut transparent_input_amount = Amount::zero(); - for input in tx.vin.iter() { - log!("Input from tx.vin.iter = {:?}", input); - let hash = H256Json::from(*input.prevout.hash()); - log!("Hash from prevout = {:?}", hash); - let prev_tx_bytes = client.get_transaction_bytes(&hash).compat().await?; - log!("prev_tx_bytes = {:?}", prev_tx_bytes); - let prev_tx = ZTransaction::read(prev_tx_bytes.as_slice()).unwrap(); - if let Some(spent_output) = prev_tx.vout.get(input.prevout.n() as usize) { - transparent_input_amount += spent_output.value; - } - } - let transparent_output_amount = - tx.vout.iter().fold(Amount::zero(), |current, out| current + out.value); - let fee_amount = tx.value_balance + transparent_input_amount - transparent_output_amount; let compact_tx = CompactTxNative { index: tx_id, hash: *hash_tx, - fee: fee_amount.into(), + fee: 0, spends, outputs, }; From 1754dc61fce750ae0ba2e7ab9505507d196655f0 Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 2 Aug 2022 15:15:37 +0700 Subject: [PATCH 25/52] WIP Deafult added for CompactTxNative --- mm2src/coins/z_coin/z_rpc.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index fc1d0972a0..c3e7b395ab 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -302,7 +302,7 @@ struct CompactBlockNative { compact_txs: Vec, } -#[derive(Debug, Serialize)] +#[derive(Default, Debug, Serialize)] struct CompactTxNative { index: u64, hash: H256Json, @@ -454,9 +454,9 @@ impl SaplingSyncLoopHandle { let compact_tx = CompactTxNative { index: tx_id, hash: *hash_tx, - fee: 0, spends, outputs, + ..Default::default() }; compact_txs.push(compact_tx); } From 59b9b527237c058dc7c892b80be344ee03787a23 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 3 Aug 2022 12:26:12 +0700 Subject: [PATCH 26/52] WIP custom structures were removed --- mm2src/coins/z_coin/z_rpc.rs | 101 ++++++++++------------------------- 1 file changed, 28 insertions(+), 73 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index c3e7b395ab..b7a9eac024 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,6 +1,5 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; use crate::utxo::rpc_clients; -use bincode::serialize; use common::executor::Timer; use common::log::{debug, error, info}; use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; @@ -15,7 +14,7 @@ use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; -use protobuf::Message as ProtobufMessage; +use protobuf::{Message as ProtobufMessage, RepeatedField}; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -23,7 +22,7 @@ use tonic::transport::{Channel, ClientTlsConfig}; use zcash_client_backend::data_api::chain::{scan_cached_blocks, validate_chain}; use zcash_client_backend::data_api::error::Error as ChainError; use zcash_client_backend::data_api::{BlockSource, WalletRead, WalletWrite}; -use zcash_client_backend::proto::compact_formats::CompactBlock; +use zcash_client_backend::proto::compact_formats::{CompactBlock, CompactOutput, CompactSpend, CompactTx}; use zcash_client_sqlite::error::SqliteClientError as ZcashClientError; use zcash_client_sqlite::wallet::init::{init_accounts_table, init_wallet_db}; use zcash_client_sqlite::WalletDb; @@ -35,7 +34,7 @@ mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; -use crate::{BytesJson, H256Json, ZTransaction}; +use crate::ZTransaction; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; @@ -292,43 +291,6 @@ pub enum SyncStatus { }, } -#[derive(Debug, Serialize)] -struct CompactBlockNative { - height: u64, - hash: H256Json, - prev_hash: H256Json, - time: u32, - header: BytesJson, - compact_txs: Vec, -} - -#[derive(Default, Debug, Serialize)] -struct CompactTxNative { - index: u64, - hash: H256Json, - // The transaction fee: present if server can provide. In the case of a - // stateless server and a transaction with transparent inputs, this will be - // unset because the calculation requires reference to prior transactions. - // in a pure-Sapling context, the fee will be calculable as: - // valueBalance + (sum(vPubNew) - sum(vPubOld) - sum(tOut)) - fee: u64, - spends: Vec, - outputs: Vec, -} - -#[derive(Debug, Serialize)] -struct CompactSpendNative { - nf: [u8; 32], -} - -#[derive(Debug, Serialize)] -struct CompactOutputNative { - cmu: [u8; 32], - epk: [u8; 32], - #[serde(serialize_with = "<[_]>::serialize")] - ciphertext: [u8; 80], -} - enum ZRpcClient { Native(NativeClient), Light(CompactTxStreamerClient), @@ -425,53 +387,46 @@ impl SaplingSyncLoopHandle { debug!("Got block {:?}", block); // log!("Got block = {:?}", block); let mut tx_id: u64 = 0; - let mut compact_txs: Vec = Vec::new(); + let mut compact_txs: Vec = Vec::new(); // create and push compact_tx during iteration for hash_tx in &block.tx { // log!("hash_tx in block.tx = {:?}", hash_tx); let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); // log!("tx ZTransaction = {:?}", tx); - let mut spends: Vec = Vec::new(); - let mut outputs: Vec = Vec::new(); + let mut spends: Vec = Vec::new(); + let mut outputs: Vec = Vec::new(); // create and push outs and spends during iterations for spend in &tx.shielded_spends { - let compact_spend = CompactSpendNative { nf: spend.nullifier.0 }; - spends.push(compact_spend); + let mut compact_spend = CompactSpend::default_instance().clone(); + compact_spend.set_nf(spend.nullifier.0.to_vec()); + spends.push(compact_spend.clone()); } for out in &tx.shielded_outputs { - let compact_out = CompactOutputNative { - cmu: out.cmu.to_bytes(), - epk: out.ephemeral_key.to_bytes(), - ciphertext: out.out_ciphertext, - }; - outputs.push(compact_out); + let mut compact_out = CompactOutput::default_instance().clone(); + compact_out.set_cmu(Vec::from(out.cmu.to_bytes())); + compact_out.set_epk(Vec::from(out.ephemeral_key.to_bytes())); + compact_out.set_ciphertext(Vec::from(out.out_ciphertext)); + outputs.push(compact_out.clone()); } tx_id += 1; - // making spends and outputs immutable - let spends = spends; - let outputs = outputs; - let compact_tx = CompactTxNative { - index: tx_id, - hash: *hash_tx, - spends, - outputs, - ..Default::default() - }; - compact_txs.push(compact_tx); + let mut compact_tx = CompactTx::default_instance().clone(); + compact_tx.set_index(tx_id); + compact_tx.set_hash(hash_tx.0.to_vec()); + compact_tx.set_spends(RepeatedField::from_vec(spends)); + compact_tx.set_outputs(RepeatedField::from_vec(outputs)); + compact_txs.push(compact_tx.clone()); } let header = client.get_block_header_bytes(block.hash).compat().await?; let compact_txs = compact_txs; - let compact_block = CompactBlockNative { - height, - hash: block.hash, - prev_hash: block.previousblockhash.unwrap(), - time: block.time, - header, - compact_txs, - }; - let serialized_block = - serialize(&compact_block).map_to_mm(|e| UpdateBlocksCacheErr::InternalError(e.to_string()))?; + let mut compact_block = CompactBlock::default_instance().clone(); + compact_block.set_height(height); + compact_block.set_hash(block.hash.0.to_vec()); + compact_block.set_prevHash(block.previousblockhash.unwrap().0.to_vec()); + compact_block.set_time(block.time); + compact_block.set_header(header.0); + compact_block.set_vtx(RepeatedField::from_vec(compact_txs)); + let serialized_block = Vec::new(); block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); } From 2195b0d86a7ed414aaa5f747c94f57aeb12af954 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 3 Aug 2022 16:06:04 +0700 Subject: [PATCH 27/52] WIP z_coin_grpc compact structs --- mm2src/coins/z_coin/z_rpc.rs | 68 +++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 29 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index b7a9eac024..2f7e1d8c88 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -14,7 +14,7 @@ use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; -use protobuf::{Message as ProtobufMessage, RepeatedField}; +use protobuf::Message as ProtobufMessage; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -22,7 +22,7 @@ use tonic::transport::{Channel, ClientTlsConfig}; use zcash_client_backend::data_api::chain::{scan_cached_blocks, validate_chain}; use zcash_client_backend::data_api::error::Error as ChainError; use zcash_client_backend::data_api::{BlockSource, WalletRead, WalletWrite}; -use zcash_client_backend::proto::compact_formats::{CompactBlock, CompactOutput, CompactSpend, CompactTx}; +use zcash_client_backend::proto::compact_formats::CompactBlock; use zcash_client_sqlite::error::SqliteClientError as ZcashClientError; use zcash_client_sqlite::wallet::init::{init_accounts_table, init_wallet_db}; use zcash_client_sqlite::WalletDb; @@ -36,7 +36,9 @@ mod z_coin_grpc { use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps}; use crate::ZTransaction; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; -use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; +use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, + CompactOutput as TonicCompactOutput, CompactSpend as TonicCompactSpend, CompactTx as TonicCompactTx, + TxFilter}; pub type WalletDbShared = Arc>>; @@ -387,47 +389,55 @@ impl SaplingSyncLoopHandle { debug!("Got block {:?}", block); // log!("Got block = {:?}", block); let mut tx_id: u64 = 0; - let mut compact_txs: Vec = Vec::new(); + let mut compact_txs = Vec::new(); // create and push compact_tx during iteration for hash_tx in &block.tx { // log!("hash_tx in block.tx = {:?}", hash_tx); let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); // log!("tx ZTransaction = {:?}", tx); - let mut spends: Vec = Vec::new(); - let mut outputs: Vec = Vec::new(); + let mut spends = Vec::new(); + let mut outputs = Vec::new(); // create and push outs and spends during iterations for spend in &tx.shielded_spends { - let mut compact_spend = CompactSpend::default_instance().clone(); - compact_spend.set_nf(spend.nullifier.0.to_vec()); - spends.push(compact_spend.clone()); + let compact_spend = TonicCompactSpend { + nf: spend.nullifier.0.to_vec(), + }; + spends.push(compact_spend); } for out in &tx.shielded_outputs { - let mut compact_out = CompactOutput::default_instance().clone(); - compact_out.set_cmu(Vec::from(out.cmu.to_bytes())); - compact_out.set_epk(Vec::from(out.ephemeral_key.to_bytes())); - compact_out.set_ciphertext(Vec::from(out.out_ciphertext)); - outputs.push(compact_out.clone()); + let compact_out = TonicCompactOutput { + cmu: out.cmu.to_bytes().to_vec(), + epk: out.ephemeral_key.to_bytes().to_vec(), + ciphertext: out.out_ciphertext.to_vec(), + }; + outputs.push(compact_out); } tx_id += 1; - let mut compact_tx = CompactTx::default_instance().clone(); - compact_tx.set_index(tx_id); - compact_tx.set_hash(hash_tx.0.to_vec()); - compact_tx.set_spends(RepeatedField::from_vec(spends)); - compact_tx.set_outputs(RepeatedField::from_vec(outputs)); - compact_txs.push(compact_tx.clone()); + let compact_tx = TonicCompactTx { + index: tx_id, + hash: hash_tx.0.to_vec(), + fee: 0, + spends, + outputs, + }; + compact_txs.push(compact_tx); } let header = client.get_block_header_bytes(block.hash).compat().await?; let compact_txs = compact_txs; - let mut compact_block = CompactBlock::default_instance().clone(); - compact_block.set_height(height); - compact_block.set_hash(block.hash.0.to_vec()); - compact_block.set_prevHash(block.previousblockhash.unwrap().0.to_vec()); - compact_block.set_time(block.time); - compact_block.set_header(header.0); - compact_block.set_vtx(RepeatedField::from_vec(compact_txs)); - let serialized_block = Vec::new(); - block_in_place(|| self.blocks_db.insert_block(block.height.unwrap(), serialized_block))?; + let compact_block = TonicCompactBlock { + proto_version: 3, + height, + hash: block.hash.0.to_vec(), + prev_hash: block.previousblockhash.unwrap().0.to_vec(), + time: block.time, + header: header.0, + vtx: compact_txs, + }; + block_in_place(|| { + self.blocks_db + .insert_block(block.height.unwrap(), compact_block.encode_to_vec()) + })?; self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); } } From a7b9c663145db55d09f149a8097e69249be6d6fc Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 3 Aug 2022 18:32:33 +0700 Subject: [PATCH 28/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 2f7e1d8c88..4030135f3e 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -29,6 +29,7 @@ use zcash_client_sqlite::WalletDb; use zcash_primitives::consensus::BlockHeight; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; +use rpc::v1::types::{H256 as H256Json}; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); @@ -387,18 +388,15 @@ impl SaplingSyncLoopHandle { log!("Height = {:?}", height); let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); - // log!("Got block = {:?}", block); let mut tx_id: u64 = 0; let mut compact_txs = Vec::new(); // create and push compact_tx during iteration for hash_tx in &block.tx { - // log!("hash_tx in block.tx = {:?}", hash_tx); let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); - // log!("tx ZTransaction = {:?}", tx); let mut spends = Vec::new(); let mut outputs = Vec::new(); - // create and push outs and spends during iterations + // create and push spends with outs during iterations for spend in &tx.shielded_spends { let compact_spend = TonicCompactSpend { nf: spend.nullifier.0.to_vec(), @@ -414,6 +412,9 @@ impl SaplingSyncLoopHandle { outputs.push(compact_out); } tx_id += 1; + // Shadowing mut variables as immutable. No longer need to update them. + let spends = spends; + let outputs = outputs; let compact_tx = TonicCompactTx { index: tx_id, hash: hash_tx.0.to_vec(), @@ -504,7 +505,7 @@ impl SaplingSyncLoopHandle { } if let ZRpcClient::Native(client) = &self.rpc_client { loop { - match client.get_raw_transaction_bytes(&tx_id.0.into()).compat().await { + match client.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); From 52e16ac254515a4c89e8aae2d6d6658797dbc6c0 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 3 Aug 2022 18:35:54 +0700 Subject: [PATCH 29/52] WIP fix fmt --- mm2src/coins/z_coin/z_rpc.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 4030135f3e..ec0ba56430 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -15,6 +15,7 @@ use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; +use rpc::v1::types::H256 as H256Json; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -29,7 +30,6 @@ use zcash_client_sqlite::WalletDb; use zcash_primitives::consensus::BlockHeight; use zcash_primitives::transaction::TxId; use zcash_primitives::zip32::ExtendedFullViewingKey; -use rpc::v1::types::{H256 as H256Json}; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); @@ -505,7 +505,11 @@ impl SaplingSyncLoopHandle { } if let ZRpcClient::Native(client) = &self.rpc_client { loop { - match client.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { + match client + .get_raw_transaction_bytes(&H256Json::from(tx_id.0)) + .compat() + .await + { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); From 5a8d823512de8d1b879e49e45b5436f8f2364de8 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 5 Aug 2022 15:34:13 +0700 Subject: [PATCH 30/52] WIP --- mm2src/coins/z_coin/z_rpc.rs | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index ec0ba56430..23db838dc5 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -361,8 +361,14 @@ impl SaplingSyncLoopHandle { }; log!("latest block height = {:?}", current_block); let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; + log!("current_block_in_db = {:?}", current_block_in_db); let from_block = current_block_in_db as u64 + 1; log!("from_block = {:?}", from_block); + let last_block_in_handler = self.current_block; + log!( + "LAST block in handler before if current_block = {:?}", + last_block_in_handler + ); if current_block >= from_block { if let ZRpcClient::Light(client) = &mut self.rpc_client { let request = tonic::Request::new(BlockRange { @@ -385,7 +391,9 @@ impl SaplingSyncLoopHandle { if let ZRpcClient::Native(client) = &self.rpc_client { let client = client.clone(); for height in from_block..=current_block { - log!("Height = {:?}", height); + if height == 500 || height == 150_000 || height == 200_000 || height == 214_000 { + log!("****************Height reached = {:?}", height); + } let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); let mut tx_id: u64 = 0; @@ -399,7 +407,7 @@ impl SaplingSyncLoopHandle { // create and push spends with outs during iterations for spend in &tx.shielded_spends { let compact_spend = TonicCompactSpend { - nf: spend.nullifier.0.to_vec(), + nf: spend.nullifier.to_vec(), }; spends.push(compact_spend); } @@ -407,7 +415,10 @@ impl SaplingSyncLoopHandle { let compact_out = TonicCompactOutput { cmu: out.cmu.to_bytes().to_vec(), epk: out.ephemeral_key.to_bytes().to_vec(), - ciphertext: out.out_ciphertext.to_vec(), + // https://zips.z.cash/zip-0307#output-compression + // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, + // which is all of the data needed to spend the note and to verify that the note is spendable. + ciphertext: out.enc_ciphertext[0..52].to_vec(), }; outputs.push(compact_out); } @@ -443,6 +454,8 @@ impl SaplingSyncLoopHandle { } } } + let block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; + log!("LAST block in db after current_block = {:?}", block_in_db); self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) } @@ -471,6 +484,7 @@ impl SaplingSyncLoopHandle { e => return MmError::err(e), } } + log!("****************AFTER validate_chain****************"); scan_cached_blocks(&self.consensus_params, &self.blocks_db, &mut wallet_ops, None)?; Ok(()) @@ -568,8 +582,10 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } sync_handle.notify_sync_finished(); + log!("****************AFTER notify_sync_finished****************"); sync_handle.check_watch_for_tx_existence().await; + log!("****************AFTER check_watch_for_tx_existence****************"); if let Some(tx_id) = sync_handle.watch_for_tx { if !block_in_place(|| is_tx_imported(sync_handle.wallet_db.lock().sql_conn(), tx_id)) { @@ -579,6 +595,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } sync_handle.watch_for_tx = None; } + log!("****************AFTER sync_handle.watch_for_tx****************"); if let Ok(Some(sender)) = sync_handle.on_tx_gen_watcher.try_next() { match sender.send(sync_handle) { @@ -588,6 +605,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { }, } } + log!("****************AFTER sync_handle.on_tx_gen_watcher.try_next****************"); Timer::sleep(10.).await; } From 0eb2517f6236d2582d918ae9386d8a27e03b1a55 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 5 Aug 2022 18:28:41 +0700 Subject: [PATCH 31/52] WIP empty header --- mm2src/coins/z_coin/z_rpc.rs | 97 +++++++++++++++++------------------- 1 file changed, 45 insertions(+), 52 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 23db838dc5..ced07636df 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -359,16 +359,8 @@ impl SaplingSyncLoopHandle { res }, }; - log!("latest block height = {:?}", current_block); let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; - log!("current_block_in_db = {:?}", current_block_in_db); let from_block = current_block_in_db as u64 + 1; - log!("from_block = {:?}", from_block); - let last_block_in_handler = self.current_block; - log!( - "LAST block in handler before if current_block = {:?}", - last_block_in_handler - ); if current_block >= from_block { if let ZRpcClient::Light(client) = &mut self.rpc_client { let request = tonic::Request::new(BlockRange { @@ -382,7 +374,12 @@ impl SaplingSyncLoopHandle { }), }); let mut response = client.get_block_range(request).await?; + let mut count = 0; while let Some(block) = response.get_mut().message().await? { + if count < 300 { + log!("WORKING LIGHT = {:?}", block); + count += 1; + } debug!("Got block {:?}", block); block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; self.notify_blocks_cache_status(block.height, current_block); @@ -391,61 +388,64 @@ impl SaplingSyncLoopHandle { if let ZRpcClient::Native(client) = &self.rpc_client { let client = client.clone(); for height in from_block..=current_block { - if height == 500 || height == 150_000 || height == 200_000 || height == 214_000 { - log!("****************Height reached = {:?}", height); - } let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); - let mut tx_id: u64 = 0; let mut compact_txs = Vec::new(); // create and push compact_tx during iteration - for hash_tx in &block.tx { + for (tx_id, hash_tx) in block.tx.iter().enumerate() { let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends = Vec::new(); let mut outputs = Vec::new(); // create and push spends with outs during iterations - for spend in &tx.shielded_spends { - let compact_spend = TonicCompactSpend { - nf: spend.nullifier.to_vec(), - }; - spends.push(compact_spend); - } - for out in &tx.shielded_outputs { - let compact_out = TonicCompactOutput { - cmu: out.cmu.to_bytes().to_vec(), - epk: out.ephemeral_key.to_bytes().to_vec(), - // https://zips.z.cash/zip-0307#output-compression - // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, - // which is all of the data needed to spend the note and to verify that the note is spendable. - ciphertext: out.enc_ciphertext[0..52].to_vec(), + if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { + for spend in &tx.shielded_spends { + let compact_spend = TonicCompactSpend { + nf: spend.nullifier.to_vec(), + }; + spends.push(compact_spend); + } + for out in &tx.shielded_outputs { + let compact_out = TonicCompactOutput { + cmu: out.cmu.to_bytes().to_vec(), + epk: out.ephemeral_key.to_bytes().to_vec(), + // https://zips.z.cash/zip-0307#output-compression + // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, + // which is all of the data needed to spend the note and to verify that the note is spendable. + ciphertext: out.enc_ciphertext[0..52].to_vec(), + }; + outputs.push(compact_out); + } + // Shadowing mut variables as immutable. No longer need to update them. + let spends = spends; + let outputs = outputs; + let compact_tx = TonicCompactTx { + index: tx_id as u64, + hash: hash_tx.0.to_vec(), + fee: 0, + spends, + outputs, }; - outputs.push(compact_out); + compact_txs.push(compact_tx); } - tx_id += 1; - // Shadowing mut variables as immutable. No longer need to update them. - let spends = spends; - let outputs = outputs; - let compact_tx = TonicCompactTx { - index: tx_id, - hash: hash_tx.0.to_vec(), - fee: 0, - spends, - outputs, - }; - compact_txs.push(compact_tx); } - let header = client.get_block_header_bytes(block.hash).compat().await?; + let mut hash = block.hash.0.to_vec(); + hash.reverse(); + let mut prev_hash = block.previousblockhash.unwrap().0.to_vec(); + prev_hash.reverse(); let compact_txs = compact_txs; let compact_block = TonicCompactBlock { - proto_version: 3, + proto_version: 0, height, - hash: block.hash.0.to_vec(), - prev_hash: block.previousblockhash.unwrap().0.to_vec(), + hash, + prev_hash, time: block.time, - header: header.0, + header: Vec::new(), vtx: compact_txs, }; + if height < 300 { + log!("WORKING NATIVE = {:?}", compact_block); + } block_in_place(|| { self.blocks_db .insert_block(block.height.unwrap(), compact_block.encode_to_vec()) @@ -454,8 +454,6 @@ impl SaplingSyncLoopHandle { } } } - let block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; - log!("LAST block in db after current_block = {:?}", block_in_db); self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) } @@ -484,7 +482,6 @@ impl SaplingSyncLoopHandle { e => return MmError::err(e), } } - log!("****************AFTER validate_chain****************"); scan_cached_blocks(&self.consensus_params, &self.blocks_db, &mut wallet_ops, None)?; Ok(()) @@ -582,10 +579,8 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } sync_handle.notify_sync_finished(); - log!("****************AFTER notify_sync_finished****************"); sync_handle.check_watch_for_tx_existence().await; - log!("****************AFTER check_watch_for_tx_existence****************"); if let Some(tx_id) = sync_handle.watch_for_tx { if !block_in_place(|| is_tx_imported(sync_handle.wallet_db.lock().sql_conn(), tx_id)) { @@ -595,7 +590,6 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } sync_handle.watch_for_tx = None; } - log!("****************AFTER sync_handle.watch_for_tx****************"); if let Ok(Some(sender)) = sync_handle.on_tx_gen_watcher.try_next() { match sender.send(sync_handle) { @@ -605,7 +599,6 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { }, } } - log!("****************AFTER sync_handle.on_tx_gen_watcher.try_next****************"); Timer::sleep(10.).await; } From 29af60c8b937b97f40bea5ca598f19cace4c37d1 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 8 Aug 2022 14:32:33 +0700 Subject: [PATCH 32/52] r2r --- mm2src/coins/z_coin/z_rpc.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index ced07636df..fb726c0d12 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -374,12 +374,7 @@ impl SaplingSyncLoopHandle { }), }); let mut response = client.get_block_range(request).await?; - let mut count = 0; while let Some(block) = response.get_mut().message().await? { - if count < 300 { - log!("WORKING LIGHT = {:?}", block); - count += 1; - } debug!("Got block {:?}", block); block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; self.notify_blocks_cache_status(block.height, current_block); @@ -391,13 +386,14 @@ impl SaplingSyncLoopHandle { let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); let mut compact_txs = Vec::new(); - // create and push compact_tx during iteration + // Create and push compact_tx during iteration. for (tx_id, hash_tx) in block.tx.iter().enumerate() { let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends = Vec::new(); let mut outputs = Vec::new(); - // create and push spends with outs during iterations + // Create and push spends with outs for compact_tx during iterations. + // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { for spend in &tx.shielded_spends { let compact_spend = TonicCompactSpend { @@ -419,9 +415,11 @@ impl SaplingSyncLoopHandle { // Shadowing mut variables as immutable. No longer need to update them. let spends = spends; let outputs = outputs; + let mut hash_tx_vec = hash_tx.0.to_vec(); + hash_tx_vec.reverse(); let compact_tx = TonicCompactTx { index: tx_id as u64, - hash: hash_tx.0.to_vec(), + hash: hash_tx_vec, fee: 0, spends, outputs, @@ -440,12 +438,10 @@ impl SaplingSyncLoopHandle { hash, prev_hash, time: block.time, + // (hash, prevHash, and time) OR (full header) header: Vec::new(), vtx: compact_txs, }; - if height < 300 { - log!("WORKING NATIVE = {:?}", compact_block); - } block_in_place(|| { self.blocks_db .insert_block(block.height.unwrap(), compact_block.encode_to_vec()) From f35a0727e6898db146fa64f2dd76eefe3fc6b126 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 8 Aug 2022 14:59:14 +0700 Subject: [PATCH 33/52] r2r --- mm2src/coins/z_coin/z_rpc.rs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index fb726c0d12..0a53366914 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -386,15 +386,15 @@ impl SaplingSyncLoopHandle { let block = client.get_block_by_height(height).await?; debug!("Got block {:?}", block); let mut compact_txs = Vec::new(); + // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. // Create and push compact_tx during iteration. for (tx_id, hash_tx) in block.tx.iter().enumerate() { let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); let mut spends = Vec::new(); let mut outputs = Vec::new(); - // Create and push spends with outs for compact_tx during iterations. - // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { + // Create and push spends with outs for compact_tx during iterations. for spend in &tx.shielded_spends { let compact_spend = TonicCompactSpend { nf: spend.nullifier.to_vec(), @@ -417,6 +417,7 @@ impl SaplingSyncLoopHandle { let outputs = outputs; let mut hash_tx_vec = hash_tx.0.to_vec(); hash_tx_vec.reverse(); + let compact_tx = TonicCompactTx { index: tx_id as u64, hash: hash_tx_vec, @@ -431,7 +432,11 @@ impl SaplingSyncLoopHandle { hash.reverse(); let mut prev_hash = block.previousblockhash.unwrap().0.to_vec(); prev_hash.reverse(); + // Shadowing mut variables as immutable. + let hash = hash; + let prev_hash = prev_hash; let compact_txs = compact_txs; + let compact_block = TonicCompactBlock { proto_version: 0, height, From 283f906d6566ca4a14ef83f2d45e570a2055607c Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 8 Aug 2022 15:18:10 +0700 Subject: [PATCH 34/52] r2r ZcoinClientInitError --- mm2src/coins/z_coin/z_coin_errors.rs | 32 ++++++---------------------- mm2src/coins/z_coin/z_rpc.rs | 28 ++++++++++++------------ 2 files changed, 20 insertions(+), 40 deletions(-) diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 208c77eb19..04b16e7717 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -42,7 +42,7 @@ impl From for UpdateBlocksCacheErr { #[derive(Debug, Display)] #[non_exhaustive] -pub enum ZcoinLightClientInitError { +pub enum ZcoinClientInitError { TlsConfigFailure(tonic::transport::Error), ConnectionFailure(tonic::transport::Error), BlocksDbInitFailure(SqliteError), @@ -50,22 +50,8 @@ pub enum ZcoinLightClientInitError { ZcashSqliteError(ZcashClientError), } -#[derive(Debug, Display)] -#[non_exhaustive] -pub enum ZcoinNativeClientInitError { - TlsConfigFailure(tonic::transport::Error), - ConnectionFailure(tonic::transport::Error), - BlocksDbInitFailure(SqliteError), - WalletDbInitFailure(SqliteError), - ZcashSqliteError(ZcashClientError), -} - -impl From for ZcoinLightClientInitError { - fn from(err: ZcashClientError) -> Self { ZcoinLightClientInitError::ZcashSqliteError(err) } -} - -impl From for ZcoinNativeClientInitError { - fn from(err: ZcashClientError) -> Self { ZcoinNativeClientInitError::ZcashSqliteError(err) } +impl From for ZcoinClientInitError { + fn from(err: ZcashClientError) -> Self { ZcoinClientInitError::ZcashSqliteError(err) } } #[derive(Debug, Display)] @@ -204,10 +190,8 @@ pub enum ZCoinBuildError { }, Io(std::io::Error), EmptyLightwalletdUris, - NativeModeIsNotSupportedYet, InvalidLightwalletdUri(InvalidUri), - LightClientInitErr(ZcoinLightClientInitError), - NativeClientInitError(ZcoinNativeClientInitError), + ClientInitErr(ZcoinClientInitError), ZCashParamsNotFound, } @@ -231,12 +215,8 @@ impl From for ZCoinBuildError { fn from(err: InvalidUri) -> Self { ZCoinBuildError::InvalidLightwalletdUri(err) } } -impl From for ZCoinBuildError { - fn from(err: ZcoinLightClientInitError) -> Self { ZCoinBuildError::LightClientInitErr(err) } -} - -impl From for ZCoinBuildError { - fn from(err: ZcoinNativeClientInitError) -> Self { ZCoinBuildError::NativeClientInitError(err) } +impl From for ZCoinBuildError { + fn from(err: ZcoinClientInitError) -> Self { ZCoinBuildError::ClientInitErr(err) } } pub(super) enum SqlTxHistoryError { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 0a53366914..0499461145 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -154,18 +154,18 @@ pub(super) async fn init_light_client( wallet_db_path: PathBuf, consensus_params: ZcoinConsensusParams, evk: ExtendedFullViewingKey, -) -> Result<(AsyncMutex, WalletDbShared), MmError> { +) -> Result<(AsyncMutex, WalletDbShared), MmError> { let blocks_db = - async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinLightClientInitError::BlocksDbInitFailure)) + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; let wallet_db = async_blocking({ let consensus_params = consensus_params.clone(); - move || -> Result<_, MmError> { + move || -> Result<_, MmError> { let db = WalletDb::for_path(wallet_db_path, consensus_params) - .map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; + .map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; if db.get_extended_full_viewing_keys()?.is_empty() { init_accounts_table(&db, &[evk])?; } @@ -176,10 +176,10 @@ pub(super) async fn init_light_client( let tonic_channel = Channel::builder(lightwalletd_url) .tls_config(ClientTlsConfig::new()) - .map_to_mm(ZcoinLightClientInitError::TlsConfigFailure)? + .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? .connect() .await - .map_to_mm(ZcoinLightClientInitError::ConnectionFailure)?; + .map_to_mm(ZcoinClientInitError::ConnectionFailure)?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -209,18 +209,18 @@ pub(super) async fn init_native_client( wallet_db_path: PathBuf, consensus_params: ZcoinConsensusParams, evk: ExtendedFullViewingKey, -) -> Result<(AsyncMutex, WalletDbShared), MmError> { +) -> Result<(AsyncMutex, WalletDbShared), MmError> { let blocks_db = - async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinNativeClientInitError::BlocksDbInitFailure)) + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; let wallet_db = async_blocking({ let consensus_params = consensus_params.clone(); - move || -> Result<_, MmError> { + move || -> Result<_, MmError> { let db = WalletDb::for_path(wallet_db_path, consensus_params) - .map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinNativeClientInitError::WalletDbInitFailure)?; + .map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; if db.get_extended_full_viewing_keys()?.is_empty() { init_accounts_table(&db, &[evk])?; } From 601e06a7e74ab774a9da224f2dbf3c248e459052 Mon Sep 17 00:00:00 2001 From: laruh Date: Mon, 8 Aug 2022 17:56:05 +0700 Subject: [PATCH 35/52] First review iteration --- mm2src/coins/z_coin/z_rpc.rs | 52 +++++++++++++----------------------- 1 file changed, 19 insertions(+), 33 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 0499461145..53e6c26f92 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -148,6 +148,21 @@ impl BlockSource for BlockDb { } } +async fn create_wallet_db( + wallet_db_path: PathBuf, + consensus_params: ZcoinConsensusParams, + evk: ExtendedFullViewingKey, +) -> Result, MmError> { + let db = + WalletDb::for_path(wallet_db_path, consensus_params).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + if db.get_extended_full_viewing_keys()?.is_empty() { + init_accounts_table(&db, &[evk])?; + } + Ok(db) +} + pub(super) async fn init_light_client( lightwalletd_url: Uri, cache_db_path: PathBuf, @@ -158,22 +173,7 @@ pub(super) async fn init_light_client( let blocks_db = async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; - - let wallet_db = async_blocking({ - let consensus_params = consensus_params.clone(); - move || -> Result<_, MmError> { - let db = WalletDb::for_path(wallet_db_path, consensus_params) - .map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - if db.get_extended_full_viewing_keys()?.is_empty() { - init_accounts_table(&db, &[evk])?; - } - Ok(db) - } - }) - .await?; - + let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), evk).await?; let tonic_channel = Channel::builder(lightwalletd_url) .tls_config(ClientTlsConfig::new()) .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? @@ -213,22 +213,7 @@ pub(super) async fn init_native_client( let blocks_db = async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; - - let wallet_db = async_blocking({ - let consensus_params = consensus_params.clone(); - move || -> Result<_, MmError> { - let db = WalletDb::for_path(wallet_db_path, consensus_params) - .map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - if db.get_extended_full_viewing_keys()?.is_empty() { - init_accounts_table(&db, &[evk])?; - } - Ok(db) - } - }) - .await?; - + let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), evk).await?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -430,7 +415,8 @@ impl SaplingSyncLoopHandle { } let mut hash = block.hash.0.to_vec(); hash.reverse(); - let mut prev_hash = block.previousblockhash.unwrap().0.to_vec(); + // Set 0 in vector in the case of genesis block. + let mut prev_hash = block.previousblockhash.unwrap_or_default().0.to_vec(); prev_hash.reverse(); // Shadowing mut variables as immutable. let hash = hash; From 08df23e60b8dc0fac552e797dd53ca739a4551ae Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 Aug 2022 13:01:14 +0700 Subject: [PATCH 36/52] WIP trait added --- mm2src/coins/z_coin/z_rpc.rs | 291 +++++++++++++++++++++-------------- 1 file changed, 173 insertions(+), 118 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 53e6c26f92..2e59f577f8 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,5 +1,6 @@ use super::{z_coin_errors::*, ZcoinConsensusParams}; use crate::utxo::rpc_clients; +use async_trait::async_trait; use common::executor::Timer; use common::log::{debug, error, info}; use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; @@ -16,6 +17,7 @@ use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; use rpc::v1::types::H256 as H256Json; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -48,6 +50,123 @@ struct CompactBlockRow { data: Vec, } +#[async_trait] +pub trait ZRpcOps { + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError>; +} + +#[async_trait] +impl ZRpcOps for CompactTxStreamerClient { + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError> { + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height: start_block, + hash: Vec::new(), + }), + end: Some(BlockId { + height: last_block, + hash: Vec::new(), + }), + }); + let mut response = self.get_block_range(request).await?; + while let Some(block) = response.get_mut().message().await? { + debug!("Got block {:?}", block); + on_block(block)?; + } + Ok(()) + } +} + +#[async_trait] +impl ZRpcOps for NativeClient { + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError> { + for height in start_block..=last_block { + let block = self.get_block_by_height(height).await?; + debug!("Got block {:?}", block); + let mut compact_txs = Vec::new(); + // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. + // Create and push compact_tx during iteration. + for (tx_id, hash_tx) in block.tx.iter().enumerate() { + let tx_bytes = self.get_transaction_bytes(hash_tx).compat().await?; + let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); + let mut spends = Vec::new(); + let mut outputs = Vec::new(); + if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { + // Create and push spends with outs for compact_tx during iterations. + for spend in &tx.shielded_spends { + let compact_spend = TonicCompactSpend { + nf: spend.nullifier.to_vec(), + }; + spends.push(compact_spend); + } + for out in &tx.shielded_outputs { + let compact_out = TonicCompactOutput { + cmu: out.cmu.to_bytes().to_vec(), + epk: out.ephemeral_key.to_bytes().to_vec(), + // https://zips.z.cash/zip-0307#output-compression + // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, + // which is all of the data needed to spend the note and to verify that the note is spendable. + ciphertext: out.enc_ciphertext[0..52].to_vec(), + }; + outputs.push(compact_out); + } + // Shadowing mut variables as immutable. No longer need to update them. + let spends = spends; + let outputs = outputs; + let mut hash_tx_vec = hash_tx.0.to_vec(); + hash_tx_vec.reverse(); + + let compact_tx = TonicCompactTx { + index: tx_id as u64, + hash: hash_tx_vec, + fee: 0, + spends, + outputs, + }; + compact_txs.push(compact_tx); + } + } + let mut hash = block.hash.0.to_vec(); + hash.reverse(); + // Set 0 in vector in the case of genesis block. + let mut prev_hash = block.previousblockhash.unwrap_or_default().0.to_vec(); + prev_hash.reverse(); + // Shadowing mut variables as immutable. + let hash = hash; + let prev_hash = prev_hash; + let compact_txs = compact_txs; + + let compact_block = TonicCompactBlock { + proto_version: 0, + height, + hash, + prev_hash, + time: block.time, + // (hash, prevHash, and time) OR (full header) + header: Vec::new(), + vtx: compact_txs, + }; + on_block(compact_block)?; + } + Ok(()) + } +} + /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -174,7 +293,7 @@ pub(super) async fn init_light_client( async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), evk).await?; - let tonic_channel = Channel::builder(lightwalletd_url) + let tonic_channel = Channel::builder(lightwalletd_url.clone()) .tls_config(ClientTlsConfig::new()) .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? .connect() @@ -188,14 +307,22 @@ pub(super) async fn init_light_client( let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), rpc_client: ZRpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), - blocks_db, + blocks_db: Mutex::new(blocks_db), wallet_db: wallet_db.clone(), consensus_params, sync_status_notifier, on_tx_gen_watcher, watch_for_tx: None, }; - let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle)); + + let tonic_channel = Channel::builder(lightwalletd_url) + .tls_config(ClientTlsConfig::new()) + .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? + .connect() + .await + .map_to_mm(ZcoinClientInitError::ConnectionFailure)?; + let rpc_copy = CompactTxStreamerClient::new(tonic_channel); + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(rpc_copy))); Ok(( SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), @@ -220,15 +347,15 @@ pub(super) async fn init_native_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - rpc_client: ZRpcClient::Native(native_client), - blocks_db, + rpc_client: ZRpcClient::Native(native_client.clone()), + blocks_db: Mutex::new(blocks_db), wallet_db: wallet_db.clone(), consensus_params, sync_status_notifier, on_tx_gen_watcher, watch_for_tx: None, }; - let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle)); + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(native_client))); Ok(( SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), @@ -245,14 +372,14 @@ fn is_tx_imported(conn: &Connection, tx_id: TxId) -> bool { } pub struct SaplingSyncRespawnGuard { - pub(super) sync_handle: Option, + pub(super) sync_handle: Option<(SaplingSyncLoopHandle, Box)>, pub(super) abort_handle: Arc>, } impl Drop for SaplingSyncRespawnGuard { fn drop(&mut self) { - if let Some(handle) = self.sync_handle.take() { - *self.abort_handle.lock() = spawn_abortable(light_wallet_db_sync_loop(handle)); + if let Some((handle, rpc)) = self.sync_handle.take() { + *self.abort_handle.lock() = spawn_abortable(light_wallet_db_sync_loop(handle, rpc)); } } } @@ -260,12 +387,14 @@ impl Drop for SaplingSyncRespawnGuard { impl SaplingSyncRespawnGuard { pub(super) fn watch_for_tx(&mut self, tx_id: TxId) { if let Some(ref mut handle) = self.sync_handle { - handle.watch_for_tx = Some(tx_id); + handle.0.watch_for_tx = Some(tx_id); } } #[inline] - pub(super) fn current_block(&self) -> BlockHeight { self.sync_handle.as_ref().expect("always Some").current_block } + pub(super) fn current_block(&self) -> BlockHeight { + self.sync_handle.as_ref().expect("always Some").0.current_block + } } pub enum SyncStatus { @@ -287,14 +416,14 @@ enum ZRpcClient { pub struct SaplingSyncLoopHandle { current_block: BlockHeight, rpc_client: ZRpcClient, - blocks_db: BlockDb, + blocks_db: Mutex, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, /// Notifies about sync status without stopping the loop, e.g. on coin activation sync_status_notifier: AsyncSender, /// If new tx is required to be generated, we stop the sync and respawn it after tx is sent /// This watcher waits for such notification - on_tx_gen_watcher: AsyncReceiver>, + on_tx_gen_watcher: AsyncReceiver)>>, watch_for_tx: Option, } @@ -334,7 +463,10 @@ impl SaplingSyncLoopHandle { } } - async fn update_blocks_cache(&mut self) -> Result<(), MmError> { + async fn update_blocks_cache( + &mut self, + rpc: &mut (dyn ZRpcOps + Send), + ) -> Result<(), MmError> { let current_block = match &mut self.rpc_client { ZRpcClient::Native(client) => client.get_block_count().compat().await?, ZRpcClient::Light(client) => { @@ -344,102 +476,19 @@ impl SaplingSyncLoopHandle { res }, }; - let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; + let current_block_in_db = block_in_place(|| self.blocks_db.lock().get_latest_block())?; let from_block = current_block_in_db as u64 + 1; if current_block >= from_block { - if let ZRpcClient::Light(client) = &mut self.rpc_client { - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: from_block, - hash: Vec::new(), - }), - end: Some(BlockId { - height: current_block, - hash: Vec::new(), - }), - }); - let mut response = client.get_block_range(request).await?; - while let Some(block) = response.get_mut().message().await? { - debug!("Got block {:?}", block); - block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; - self.notify_blocks_cache_status(block.height, current_block); - } - } - if let ZRpcClient::Native(client) = &self.rpc_client { - let client = client.clone(); - for height in from_block..=current_block { - let block = client.get_block_by_height(height).await?; - debug!("Got block {:?}", block); - let mut compact_txs = Vec::new(); - // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. - // Create and push compact_tx during iteration. - for (tx_id, hash_tx) in block.tx.iter().enumerate() { - let tx_bytes = client.get_transaction_bytes(hash_tx).compat().await?; - let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); - let mut spends = Vec::new(); - let mut outputs = Vec::new(); - if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { - // Create and push spends with outs for compact_tx during iterations. - for spend in &tx.shielded_spends { - let compact_spend = TonicCompactSpend { - nf: spend.nullifier.to_vec(), - }; - spends.push(compact_spend); - } - for out in &tx.shielded_outputs { - let compact_out = TonicCompactOutput { - cmu: out.cmu.to_bytes().to_vec(), - epk: out.ephemeral_key.to_bytes().to_vec(), - // https://zips.z.cash/zip-0307#output-compression - // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, - // which is all of the data needed to spend the note and to verify that the note is spendable. - ciphertext: out.enc_ciphertext[0..52].to_vec(), - }; - outputs.push(compact_out); - } - // Shadowing mut variables as immutable. No longer need to update them. - let spends = spends; - let outputs = outputs; - let mut hash_tx_vec = hash_tx.0.to_vec(); - hash_tx_vec.reverse(); - - let compact_tx = TonicCompactTx { - index: tx_id as u64, - hash: hash_tx_vec, - fee: 0, - spends, - outputs, - }; - compact_txs.push(compact_tx); - } - } - let mut hash = block.hash.0.to_vec(); - hash.reverse(); - // Set 0 in vector in the case of genesis block. - let mut prev_hash = block.previousblockhash.unwrap_or_default().0.to_vec(); - prev_hash.reverse(); - // Shadowing mut variables as immutable. - let hash = hash; - let prev_hash = prev_hash; - let compact_txs = compact_txs; - - let compact_block = TonicCompactBlock { - proto_version: 0, - height, - hash, - prev_hash, - time: block.time, - // (hash, prevHash, and time) OR (full header) - header: Vec::new(), - vtx: compact_txs, - }; - block_in_place(|| { - self.blocks_db - .insert_block(block.height.unwrap(), compact_block.encode_to_vec()) - })?; - self.notify_blocks_cache_status(block.height.unwrap() as u64, current_block); - } - } + rpc.scan_blocks(from_block, current_block, &mut |block: TonicCompactBlock| { + block_in_place(|| { + self.blocks_db + .lock() + .insert_block(block.height as u32, block.encode_to_vec()) + })?; + self.notify_blocks_cache_status(block.height, current_block); + Ok(()) + }) + .await?; } self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) @@ -453,7 +502,7 @@ impl SaplingSyncLoopHandle { if let Err(e) = validate_chain( &self.consensus_params, - &self.blocks_db, + &self.blocks_db.lock().deref(), wallet_ops.get_max_height_hash()?, ) { match e { @@ -464,13 +513,18 @@ impl SaplingSyncLoopHandle { BlockHeight::from_u32(0) }; wallet_ops.rewind_to_height(rewind_height)?; - self.blocks_db.rewind_to_height(rewind_height.into())?; + self.blocks_db.lock().rewind_to_height(rewind_height.into())?; }, e => return MmError::err(e), } } - scan_cached_blocks(&self.consensus_params, &self.blocks_db, &mut wallet_ops, None)?; + scan_cached_blocks( + &self.consensus_params, + self.blocks_db.lock().deref(), + &mut wallet_ops, + None, + )?; Ok(()) } @@ -548,10 +602,10 @@ impl SaplingSyncLoopHandle { /// 6. Once the transaction is generated and sent, `SaplingSyncRespawnGuard::watch_for_tx` is called to update `SaplingSyncLoopHandle` state. /// 7. Once the loop is respawned, it will check that broadcast tx is imported (or not available anymore) before stopping in favor of /// next wait_for_gen_tx_blockchain_sync call. -async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { +async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle, mut client: Box) { // this loop is spawned as standalone task so it's safe to use block_in_place here loop { - if let Err(e) = sync_handle.update_blocks_cache().await { + if let Err(e) = sync_handle.update_blocks_cache(client.as_mut()).await { error!("Error {} on blocks cache update", e); Timer::sleep(10.).await; continue; @@ -579,10 +633,11 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } if let Ok(Some(sender)) = sync_handle.on_tx_gen_watcher.try_next() { - match sender.send(sync_handle) { + match sender.send((sync_handle, client)) { Ok(_) => break, - Err(handle_from_channel) => { + Err((handle_from_channel, rpc_from_channel)) => { sync_handle = handle_from_channel; + client = rpc_from_channel; }, } } @@ -592,7 +647,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } type SyncWatcher = AsyncReceiver; -type NewTxNotifier = AsyncSender>; +type NewTxNotifier = AsyncSender)>>; pub(super) struct SaplingSyncConnector { sync_watcher: SyncWatcher, @@ -628,8 +683,8 @@ impl SaplingSyncConnector { .map_to_mm(|_| BlockchainScanStopped {})?; receiver .await - .map(|handle| SaplingSyncRespawnGuard { - sync_handle: Some(handle), + .map(|(handle, rpc)| SaplingSyncRespawnGuard { + sync_handle: Some((handle, rpc)), abort_handle: self.abort_handle.clone(), }) .map_to_mm(|_| BlockchainScanStopped {}) From 089f22e6efd73076e9d102a397ac2dcfedd6f65f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 Aug 2022 13:14:25 +0700 Subject: [PATCH 37/52] WIP reference removed --- mm2src/coins/z_coin/z_rpc.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 2e59f577f8..4f432a80e6 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -502,7 +502,7 @@ impl SaplingSyncLoopHandle { if let Err(e) = validate_chain( &self.consensus_params, - &self.blocks_db.lock().deref(), + self.blocks_db.lock().deref(), wallet_ops.get_max_height_hash()?, ) { match e { From 9dc0d7838838fc17a87b7adbb3938b64aca2481f Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 9 Aug 2022 22:33:52 +0700 Subject: [PATCH 38/52] remove rpc_client, impl additional method --- mm2src/coins/z_coin/z_rpc.rs | 135 +++++++++++++++++------------------ 1 file changed, 64 insertions(+), 71 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 4f432a80e6..0c39428dd1 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -52,16 +52,27 @@ struct CompactBlockRow { #[async_trait] pub trait ZRpcOps { + async fn get_block_height(&mut self) -> Result>; + async fn scan_blocks( &mut self, start_block: u64, last_block: u64, on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), ) -> Result<(), MmError>; + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option); } #[async_trait] impl ZRpcOps for CompactTxStreamerClient { + async fn get_block_height(&mut self) -> Result> { + let request = tonic::Request::new(ChainSpec {}); + let block = self.get_latest_block(request).await?; + let res = block.into_inner().height; + Ok(res) + } + async fn scan_blocks( &mut self, start_block: u64, @@ -85,10 +96,39 @@ impl ZRpcOps for CompactTxStreamerClient { } Ok(()) } + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + loop { + let filter = TxFilter { + block: None, + index: 0, + hash: tx_id.0.into(), + }; + let request = tonic::Request::new(filter); + match self.get_transaction(request).await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { + if *attempts >= 3 { + *watch_for_tx = None; + return; + } + *attempts += 1; + } + Timer::sleep(30.).await; + }, + } + } + } } #[async_trait] impl ZRpcOps for NativeClient { + async fn get_block_height(&mut self) -> Result> { + Ok(self.get_block_count().compat().await?) + } + async fn scan_blocks( &mut self, start_block: u64, @@ -165,6 +205,25 @@ impl ZRpcOps for NativeClient { } Ok(()) } + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + loop { + match self.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.to_string().contains(rpc_clients::NO_TX_ERROR_CODE) { + if *attempts >= 3 { + *watch_for_tx = None; + return; + } + *attempts += 1; + } + Timer::sleep(30.).await; + }, + } + } + } } /// A wrapper for the SQLite connection to the block cache database. @@ -293,12 +352,6 @@ pub(super) async fn init_light_client( async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), evk).await?; - let tonic_channel = Channel::builder(lightwalletd_url.clone()) - .tls_config(ClientTlsConfig::new()) - .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? - .connect() - .await - .map_to_mm(ZcoinClientInitError::ConnectionFailure)?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -306,7 +359,6 @@ pub(super) async fn init_light_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - rpc_client: ZRpcClient::Light(CompactTxStreamerClient::new(tonic_channel)), blocks_db: Mutex::new(blocks_db), wallet_db: wallet_db.clone(), consensus_params, @@ -347,7 +399,6 @@ pub(super) async fn init_native_client( let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - rpc_client: ZRpcClient::Native(native_client.clone()), blocks_db: Mutex::new(blocks_db), wallet_db: wallet_db.clone(), consensus_params, @@ -408,14 +459,8 @@ pub enum SyncStatus { }, } -enum ZRpcClient { - Native(NativeClient), - Light(CompactTxStreamerClient), -} - pub struct SaplingSyncLoopHandle { current_block: BlockHeight, - rpc_client: ZRpcClient, blocks_db: Mutex, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, @@ -467,15 +512,7 @@ impl SaplingSyncLoopHandle { &mut self, rpc: &mut (dyn ZRpcOps + Send), ) -> Result<(), MmError> { - let current_block = match &mut self.rpc_client { - ZRpcClient::Native(client) => client.get_block_count().compat().await?, - ZRpcClient::Light(client) => { - let request = tonic::Request::new(ChainSpec {}); - let block = client.get_latest_block(request).await?; - let res: u64 = block.into_inner().height; - res - }, - }; + let current_block = rpc.get_block_height().await?; let current_block_in_db = block_in_place(|| self.blocks_db.lock().get_latest_block())?; let from_block = current_block_in_db as u64 + 1; if current_block >= from_block { @@ -528,55 +565,11 @@ impl SaplingSyncLoopHandle { Ok(()) } - async fn check_watch_for_tx_existence(&mut self) { + async fn check_watch_for_tx_existence(&mut self, rpc: &mut (dyn ZRpcOps + Send)) { if let Some(tx_id) = self.watch_for_tx { let mut attempts = 0; - if let ZRpcClient::Light(client) = &mut self.rpc_client { - loop { - let filter = TxFilter { - block: None, - index: 0, - hash: tx_id.0.into(), - }; - let request = tonic::Request::new(filter); - match client.get_transaction(request).await { - Ok(_) => break, - Err(e) => { - error!("Error on getting tx {}", tx_id); - if e.message().contains(rpc_clients::NO_TX_ERROR_CODE) { - if attempts >= 3 { - self.watch_for_tx = None; - return; - } - attempts += 1; - } - Timer::sleep(30.).await; - }, - } - } - } - if let ZRpcClient::Native(client) = &self.rpc_client { - loop { - match client - .get_raw_transaction_bytes(&H256Json::from(tx_id.0)) - .compat() - .await - { - Ok(_) => break, - Err(e) => { - error!("Error on getting tx {}", tx_id); - if e.to_string().contains(rpc_clients::NO_TX_ERROR_CODE) { - if attempts >= 3 { - self.watch_for_tx = None; - return; - } - attempts += 1; - } - Timer::sleep(30.).await; - }, - } - } - } + rpc.check_watch_for_tx(tx_id, &mut attempts, &mut self.watch_for_tx) + .await; } } } @@ -621,7 +614,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle, mut c sync_handle.notify_sync_finished(); - sync_handle.check_watch_for_tx_existence().await; + sync_handle.check_watch_for_tx_existence(client.as_mut()).await; if let Some(tx_id) = sync_handle.watch_for_tx { if !block_in_place(|| is_tx_imported(sync_handle.wallet_db.lock().sql_conn(), tx_id)) { From 1f46d9875239e1c6753b988a95ee1e6e8b9bda43 Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 11 Aug 2022 13:34:26 +0700 Subject: [PATCH 39/52] fix errors --- mm2src/coins/z_coin.rs | 3 +- mm2src/coins/z_coin/z_rpc.rs | 402 ++++++++++++++++++++++++++--------- 2 files changed, 300 insertions(+), 105 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 1b2e1b132f..b32bc8c3a5 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -775,7 +775,8 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { native_client, cache_db_path, wallet_db_path, - self.consensus_params.clone(), + self.protocol_info.consensus_params.clone(), + self.protocol_info.check_point_block, evk, ) .await? diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index b0495cf6e3..ecce4d8f76 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,19 +1,22 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; -use crate::utxo::rpc_clients::NO_TX_ERROR_CODE; +use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; +use async_trait::async_trait; use common::executor::Timer; use common::log::{debug, error, info, LogOnError}; -use common::{async_blocking, spawn_abortable, AbortOnDropHandle}; +use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; +use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; +use std::ops::Deref; use std::path::{Path, PathBuf}; use std::sync::Arc; use tokio::task::block_in_place; @@ -33,8 +36,12 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } +use crate::ZTransaction; +use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; -use z_coin_grpc::{BlockId, BlockRange, ChainSpec, TxFilter}; +use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, + CompactOutput as TonicCompactOutput, CompactSpend as TonicCompactSpend, CompactTx as TonicCompactTx, + TxFilter}; pub type WalletDbShared = Arc>>; @@ -43,6 +50,182 @@ struct CompactBlockRow { data: Vec, } +#[async_trait] +pub trait ZRpcOps { + async fn get_block_height(&mut self) -> Result>; + + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError>; + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option); +} + +#[async_trait] +impl ZRpcOps for CompactTxStreamerClient { + async fn get_block_height(&mut self) -> Result> { + let request = tonic::Request::new(ChainSpec {}); + let block = self.get_latest_block(request).await?; + let res = block.into_inner().height; + Ok(res) + } + + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError> { + let request = tonic::Request::new(BlockRange { + start: Some(BlockId { + height: start_block, + hash: Vec::new(), + }), + end: Some(BlockId { + height: last_block, + hash: Vec::new(), + }), + }); + let mut response = self.get_block_range(request).await?; + while let Some(block) = response.get_mut().message().await? { + debug!("Got block {:?}", block); + on_block(block)?; + } + Ok(()) + } + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + loop { + let filter = TxFilter { + block: None, + index: 0, + hash: tx_id.0.into(), + }; + let request = tonic::Request::new(filter); + match self.get_transaction(request).await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.message().contains(NO_TX_ERROR_CODE) { + if *attempts >= 3 { + *watch_for_tx = None; + return; + } + *attempts += 1; + } + Timer::sleep(30.).await; + }, + } + } + } +} + +#[async_trait] +impl ZRpcOps for NativeClient { + async fn get_block_height(&mut self) -> Result> { + Ok(self.get_block_count().compat().await?) + } + + async fn scan_blocks( + &mut self, + start_block: u64, + last_block: u64, + on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + ) -> Result<(), MmError> { + for height in start_block..=last_block { + let block = self.get_block_by_height(height).await?; + debug!("Got block {:?}", block); + let mut compact_txs = Vec::new(); + // By default, CompactBlocks only contain CompactTxs for transactions that contain Sapling spends or outputs. + // Create and push compact_tx during iteration. + for (tx_id, hash_tx) in block.tx.iter().enumerate() { + let tx_bytes = self.get_transaction_bytes(hash_tx).compat().await?; + let tx = ZTransaction::read(tx_bytes.as_slice()).unwrap(); + let mut spends = Vec::new(); + let mut outputs = Vec::new(); + if !tx.shielded_spends.is_empty() || !tx.shielded_outputs.is_empty() { + // Create and push spends with outs for compact_tx during iterations. + for spend in &tx.shielded_spends { + let compact_spend = TonicCompactSpend { + nf: spend.nullifier.to_vec(), + }; + spends.push(compact_spend); + } + for out in &tx.shielded_outputs { + let compact_out = TonicCompactOutput { + cmu: out.cmu.to_bytes().to_vec(), + epk: out.ephemeral_key.to_bytes().to_vec(), + // https://zips.z.cash/zip-0307#output-compression + // The first 52 bytes of the ciphertext contain the contents and opening of the note commitment, + // which is all of the data needed to spend the note and to verify that the note is spendable. + ciphertext: out.enc_ciphertext[0..52].to_vec(), + }; + outputs.push(compact_out); + } + // Shadowing mut variables as immutable. No longer need to update them. + let spends = spends; + let outputs = outputs; + let mut hash_tx_vec = hash_tx.0.to_vec(); + hash_tx_vec.reverse(); + + let compact_tx = TonicCompactTx { + index: tx_id as u64, + hash: hash_tx_vec, + fee: 0, + spends, + outputs, + }; + compact_txs.push(compact_tx); + } + } + let mut hash = block.hash.0.to_vec(); + hash.reverse(); + // Set 0 in vector in the case of genesis block. + let mut prev_hash = block.previousblockhash.unwrap_or_default().0.to_vec(); + prev_hash.reverse(); + // Shadowing mut variables as immutable. + let hash = hash; + let prev_hash = prev_hash; + let compact_txs = compact_txs; + + let compact_block = TonicCompactBlock { + proto_version: 0, + height, + hash, + prev_hash, + time: block.time, + // (hash, prevHash, and time) OR (full header) + header: Vec::new(), + vtx: compact_txs, + }; + on_block(compact_block)?; + } + Ok(()) + } + + async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + loop { + match self.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { + Ok(_) => break, + Err(e) => { + error!("Error on getting tx {}", tx_id); + if e.to_string().contains(NO_TX_ERROR_CODE) { + if *attempts >= 3 { + *watch_for_tx = None; + return; + } + *attempts += 1; + } + Timer::sleep(30.).await; + }, + } + } + } +} + /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -108,7 +291,7 @@ impl BlockDb { Ok(query_single_row( &self.0, "SELECT height FROM compactblocks ORDER BY height DESC LIMIT 1", - db_common::sqlite::rusqlite::NO_PARAMS, + NO_PARAMS, |row| row.get(0), )? .unwrap_or(0)) @@ -143,6 +326,31 @@ impl BlockSource for BlockDb { } } +async fn create_wallet_db( + wallet_db_path: PathBuf, + consensus_params: ZcoinConsensusParams, + check_point_block: Option, + evk: ExtendedFullViewingKey, +) -> Result, MmError> { + let db = + WalletDb::for_path(wallet_db_path, consensus_params).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + if db.get_extended_full_viewing_keys()?.is_empty() { + init_accounts_table(&db, &[evk])?; + if let Some(check_point) = check_point_block { + init_blocks_table( + &db, + BlockHeight::from_u32(check_point.height), + BlockHash(check_point.hash.0), + check_point.time, + &check_point.sapling_tree.0, + )?; + } + } + Ok(db) +} + pub(super) async fn init_light_client( lightwalletd_url: Uri, cache_db_path: PathBuf, @@ -150,58 +358,68 @@ pub(super) async fn init_light_client( consensus_params: ZcoinConsensusParams, check_point_block: Option, evk: ExtendedFullViewingKey, -) -> Result<(AsyncMutex, WalletDbShared), MmError> { +) -> Result<(AsyncMutex, WalletDbShared), MmError> { let blocks_db = - async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinLightClientInitError::BlocksDbInitFailure)) + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; - let wallet_db = async_blocking({ - let consensus_params = consensus_params.clone(); - move || -> Result<_, MmError> { - let db = WalletDb::for_path(wallet_db_path, consensus_params) - .map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinLightClientInitError::WalletDbInitFailure)?; - if db.get_extended_full_viewing_keys()?.is_empty() { - init_accounts_table(&db, &[evk])?; - if let Some(check_point) = check_point_block { - init_blocks_table( - &db, - BlockHeight::from_u32(check_point.height), - BlockHash(check_point.hash.0), - check_point.time, - &check_point.sapling_tree.0, - )?; - } - } - Ok(db) - } - }) - .await?; + let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; + + let (sync_status_notifier, sync_watcher) = channel(1); + let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); + + let wallet_db = Arc::new(Mutex::new(wallet_db)); + let sync_handle = SaplingSyncLoopHandle { + current_block: BlockHeight::from_u32(0), + blocks_db: Mutex::new(blocks_db), + wallet_db: wallet_db.clone(), + consensus_params, + sync_status_notifier, + on_tx_gen_watcher, + watch_for_tx: None, + }; let tonic_channel = Channel::builder(lightwalletd_url) .tls_config(ClientTlsConfig::new()) - .map_to_mm(ZcoinLightClientInitError::TlsConfigFailure)? + .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? .connect() .await - .map_to_mm(ZcoinLightClientInitError::ConnectionFailure)?; - let grpc_client = CompactTxStreamerClient::new(tonic_channel); + .map_to_mm(ZcoinClientInitError::ConnectionFailure)?; + let rpc_copy = CompactTxStreamerClient::new(tonic_channel); + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(rpc_copy))); + + Ok(( + SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), + wallet_db, + )) +} +pub(super) async fn init_native_client( + native_client: NativeClient, + cache_db_path: PathBuf, + wallet_db_path: PathBuf, + consensus_params: ZcoinConsensusParams, + check_point_block: Option, + evk: ExtendedFullViewingKey, +) -> Result<(AsyncMutex, WalletDbShared), MmError> { + let blocks_db = + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) + .await?; + let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), - grpc_client, - blocks_db, + blocks_db: Mutex::new(blocks_db), wallet_db: wallet_db.clone(), consensus_params, sync_status_notifier, on_tx_gen_watcher, watch_for_tx: None, }; - let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle)); + let abort_handle = spawn_abortable(light_wallet_db_sync_loop(sync_handle, Box::new(native_client))); Ok(( SaplingSyncConnector::new_mutex_wrapped(sync_watcher, on_tx_gen_notifier, abort_handle), @@ -218,14 +436,14 @@ fn is_tx_imported(conn: &Connection, tx_id: TxId) -> bool { } pub struct SaplingSyncRespawnGuard { - pub(super) sync_handle: Option, + pub(super) sync_handle: Option<(SaplingSyncLoopHandle, Box)>, pub(super) abort_handle: Arc>, } impl Drop for SaplingSyncRespawnGuard { fn drop(&mut self) { - if let Some(handle) = self.sync_handle.take() { - *self.abort_handle.lock() = spawn_abortable(light_wallet_db_sync_loop(handle)); + if let Some((handle, rpc)) = self.sync_handle.take() { + *self.abort_handle.lock() = spawn_abortable(light_wallet_db_sync_loop(handle, rpc)); } } } @@ -233,12 +451,14 @@ impl Drop for SaplingSyncRespawnGuard { impl SaplingSyncRespawnGuard { pub(super) fn watch_for_tx(&mut self, tx_id: TxId) { if let Some(ref mut handle) = self.sync_handle { - handle.watch_for_tx = Some(tx_id); + handle.0.watch_for_tx = Some(tx_id); } } #[inline] - pub(super) fn current_block(&self) -> BlockHeight { self.sync_handle.as_ref().expect("always Some").current_block } + pub(super) fn current_block(&self) -> BlockHeight { + self.sync_handle.as_ref().expect("always Some").0.current_block + } } pub enum SyncStatus { @@ -258,15 +478,14 @@ pub enum SyncStatus { pub struct SaplingSyncLoopHandle { current_block: BlockHeight, - grpc_client: CompactTxStreamerClient, - blocks_db: BlockDb, + blocks_db: Mutex, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, /// Notifies about sync status without stopping the loop, e.g. on coin activation sync_status_notifier: AsyncSender, /// If new tx is required to be generated, we stop the sync and respawn it after tx is sent /// This watcher waits for such notification - on_tx_gen_watcher: AsyncReceiver>, + on_tx_gen_watcher: AsyncReceiver)>>, watch_for_tx: Option, } @@ -303,12 +522,13 @@ impl SaplingSyncLoopHandle { .debug_log_with_msg("No one seems interested in SyncStatus"); } - async fn update_blocks_cache(&mut self) -> Result<(), MmError> { - let request = tonic::Request::new(ChainSpec {}); - let current_blockchain_block = self.grpc_client.get_latest_block(request).await?; - let current_block_in_db = block_in_place(|| self.blocks_db.get_latest_block())?; + async fn update_blocks_cache( + &mut self, + rpc: &mut (dyn ZRpcOps + Send), + ) -> Result<(), MmError> { + let current_block = rpc.get_block_height().await?; + let current_block_in_db = block_in_place(|| self.blocks_db.lock().get_latest_block())?; let extrema = block_in_place(|| self.wallet_db.lock().block_height_extrema())?; - let mut from_block = self .consensus_params .sapling_activation_height @@ -317,30 +537,18 @@ impl SaplingSyncLoopHandle { if let Some((_, max_in_wallet)) = extrema { from_block = from_block.max(max_in_wallet.into()); } - - let current_block: u64 = current_blockchain_block.into_inner().height; - if current_block >= from_block { - let request = tonic::Request::new(BlockRange { - start: Some(BlockId { - height: from_block, - hash: Vec::new(), - }), - end: Some(BlockId { - height: current_block, - hash: Vec::new(), - }), - }); - - let mut response = self.grpc_client.get_block_range(request).await?; - - while let Some(block) = response.get_mut().message().await? { - debug!("Got block {:?}", block); - block_in_place(|| self.blocks_db.insert_block(block.height as u32, block.encode_to_vec()))?; + rpc.scan_blocks(from_block, current_block, &mut |block: TonicCompactBlock| { + block_in_place(|| { + self.blocks_db + .lock() + .insert_block(block.height as u32, block.encode_to_vec()) + })?; self.notify_blocks_cache_status(block.height, current_block); - } + Ok(()) + }) + .await?; } - self.current_block = BlockHeight::from_u32(current_block as u32); Ok(()) } @@ -355,7 +563,7 @@ impl SaplingSyncLoopHandle { if let Err(e) = validate_chain( &self.consensus_params, - &self.blocks_db, + self.blocks_db.lock().deref(), wallet_ops.get_max_height_hash()?, ) { match e { @@ -366,13 +574,13 @@ impl SaplingSyncLoopHandle { BlockHeight::from_u32(0) }; wallet_ops.rewind_to_height(rewind_height)?; - self.blocks_db.rewind_to_height(rewind_height.into())?; + self.blocks_db.lock().rewind_to_height(rewind_height.into())?; }, e => return MmError::err(e), } } - let current_block = BlockHeight::from_u32(self.blocks_db.get_latest_block()?); + let current_block = BlockHeight::from_u32(self.blocks_db.lock().get_latest_block()?); loop { match wallet_ops.block_height_extrema()? { Some((_, max_in_wallet)) => { @@ -384,36 +592,21 @@ impl SaplingSyncLoopHandle { }, None => self.notify_building_wallet_db(0, current_block.into()), } - scan_cached_blocks(&self.consensus_params, &self.blocks_db, &mut wallet_ops, Some(1000))?; + scan_cached_blocks( + &self.consensus_params, + self.blocks_db.lock().deref(), + &mut wallet_ops, + Some(1000), + )?; } Ok(()) } - async fn check_watch_for_tx_existence(&mut self) { + async fn check_watch_for_tx_existence(&mut self, rpc: &mut (dyn ZRpcOps + Send)) { if let Some(tx_id) = self.watch_for_tx { let mut attempts = 0; - loop { - let filter = TxFilter { - block: None, - index: 0, - hash: tx_id.0.into(), - }; - let request = tonic::Request::new(filter); - match self.grpc_client.get_transaction(request).await { - Ok(_) => break, - Err(e) => { - error!("Error on getting tx {}", tx_id); - if e.message().contains(NO_TX_ERROR_CODE) { - if attempts >= 3 { - self.watch_for_tx = None; - return; - } - attempts += 1; - } - Timer::sleep(30.).await; - }, - } - } + rpc.check_watch_for_tx(tx_id, &mut attempts, &mut self.watch_for_tx) + .await; } } } @@ -439,10 +632,10 @@ impl SaplingSyncLoopHandle { /// 6. Once the transaction is generated and sent, `SaplingSyncRespawnGuard::watch_for_tx` is called to update `SaplingSyncLoopHandle` state. /// 7. Once the loop is respawned, it will check that broadcast tx is imported (or not available anymore) before stopping in favor of /// next wait_for_gen_tx_blockchain_sync call. -async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { +async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle, mut client: Box) { // this loop is spawned as standalone task so it's safe to use block_in_place here loop { - if let Err(e) = sync_handle.update_blocks_cache().await { + if let Err(e) = sync_handle.update_blocks_cache(client.as_mut()).await { error!("Error {} on blocks cache update", e); sync_handle.notify_on_error(e.to_string()); Timer::sleep(10.).await; @@ -458,7 +651,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { sync_handle.notify_sync_finished(); - sync_handle.check_watch_for_tx_existence().await; + sync_handle.check_watch_for_tx_existence(client.as_mut()).await; if let Some(tx_id) = sync_handle.watch_for_tx { if !block_in_place(|| is_tx_imported(sync_handle.wallet_db.lock().sql_conn(), tx_id)) { @@ -470,10 +663,11 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } if let Ok(Some(sender)) = sync_handle.on_tx_gen_watcher.try_next() { - match sender.send(sync_handle) { + match sender.send((sync_handle, client)) { Ok(_) => break, - Err(handle_from_channel) => { + Err((handle_from_channel, rpc_from_channel)) => { sync_handle = handle_from_channel; + client = rpc_from_channel; }, } } @@ -483,7 +677,7 @@ async fn light_wallet_db_sync_loop(mut sync_handle: SaplingSyncLoopHandle) { } type SyncWatcher = AsyncReceiver; -type NewTxNotifier = AsyncSender>; +type NewTxNotifier = AsyncSender)>>; pub(super) struct SaplingSyncConnector { sync_watcher: SyncWatcher, @@ -519,8 +713,8 @@ impl SaplingSyncConnector { .map_to_mm(|_| BlockchainScanStopped {})?; receiver .await - .map(|handle| SaplingSyncRespawnGuard { - sync_handle: Some(handle), + .map(|(handle, rpc)| SaplingSyncRespawnGuard { + sync_handle: Some((handle, rpc)), abort_handle: self.abort_handle.clone(), }) .map_to_mm(|_| BlockchainScanStopped {}) From 83c450c9c4f3a171af83b037590d92b62b35369f Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 26 Aug 2022 18:42:48 +0700 Subject: [PATCH 40/52] WIP LightwalletdImpl and LightwalletdConnection added --- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_rpc.rs | 153 +++++++++++++++++++++++++++++++++-- 2 files changed, 148 insertions(+), 7 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a358c3dda2..dae976eded 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -794,7 +794,7 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { )?; init_light_client( - uri, + light_wallet_d_servers, cache_db_path, wallet_db_path, self.protocol_info.consensus_params.clone(), diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index ecce4d8f76..d0b73337a0 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,23 +1,28 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; -use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; +use crate::utxo::rpc_clients::{JsonRpcPendingRequests, JsonRpcPendingRequestsShared, NativeClient, UtxoRpcClientOps, + NO_TX_ERROR_CODE}; use async_trait::async_trait; -use common::executor::Timer; -use common::log::{debug, error, info, LogOnError}; -use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; +use common::executor::{spawn, Timer}; +use common::log::{debug, error, info, warn, LogOnError}; +use common::{async_blocking, small_rng, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; +use futures::future::{select as select_func, FutureExt}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; +use futures01::sync::{mpsc, oneshot}; use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; +use rand::seq::SliceRandom; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::sync::atomic::AtomicU64; use std::sync::Arc; use tokio::task::block_in_place; use tonic::transport::{Channel, ClientTlsConfig}; @@ -36,7 +41,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } -use crate::ZTransaction; +use crate::{RpcTransportEventHandlerShared, ZTransaction}; use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, @@ -226,6 +231,127 @@ impl ZRpcOps for NativeClient { } } +#[derive(Debug)] +struct LightwalletdConnection { + /// The lightwalletd connected to this SocketAddr + addr: String, + /// The Sender forwarding requests to writing part of underlying stream + tx: Arc>>>>, + /// The Sender used to shutdown the background connection loop when LightwalletdConnection is dropped + shutdown_tx: Option>, + /// Responses are stored here + responses: JsonRpcPendingRequestsShared, +} + +impl LightwalletdConnection { + async fn is_connected(&self) -> bool { self.tx.lock().await.is_some() } +} + +impl Drop for LightwalletdConnection { + fn drop(&mut self) { + if let Some(shutdown_tx) = self.shutdown_tx.take() { + if shutdown_tx.send(()).is_err() { + warn!("lightwalletd_connection_drop] Warning, shutdown_tx already closed"); + } + } + } +} + +struct LightwalletdImpl { + connections: AsyncMutex>, + next_id: AtomicU64, + event_handlers: Vec, +} + +impl LightwalletdImpl { + fn new() -> LightwalletdImpl { + LightwalletdImpl { + connections: AsyncMutex::new(vec![]), + next_id: 0.into(), + event_handlers: vec![], + } + } + + // Create an Lightwalletd connection and spawn a green thread actor to handle it. + pub async fn add_server(&self, url: &String) -> Result<(), String> { + let connection = try_s!(spawn_connect(url, self.event_handlers.clone())); + self.connections.lock().await.push(connection); + Ok(()) + } + + /// Remove an Lightwalletd connection and stop corresponding spawned actor. + pub async fn remove_server(&self, server_addr: &str) -> Result<(), String> { + let mut connections = self.connections.lock().await; + // do not use retain, we would have to return an error if we did not find connection by the passd address + let pos = connections + .iter() + .position(|con| con.addr == server_addr) + .ok_or(ERRL!("Unknown lightwalletd address {}", server_addr))?; + // shutdown_tx will be closed immediately on the connection drop + connections.remove(pos); + Ok(()) + } + + /// Moves the lightwalletd servers that fail in a multi request to the end. + pub async fn rotate_servers(&self, no_of_rotations: usize) { + let mut connections = self.connections.lock().await; + connections.rotate_left(no_of_rotations); + } + + /// Check if one of the spawned connections is connected. + pub async fn is_connected(&self) -> bool { + for connection in self.connections.lock().await.iter() { + if connection.is_connected().await { + return true; + } + } + false + } + + pub async fn count_connections(&self) -> usize { self.connections.lock().await.len() } +} + +/// Attempts to process the request (parse url, etc), build up the config and create new lightwalletd connection +#[cfg(not(target_arch = "wasm32"))] +fn spawn_connect( + url: &String, + event_handlers: Vec, +) -> Result { + let uri: Uri = try_s!(url.parse()); + uri.host().ok_or(ERRL!("Couldn't retrieve host from addr {}", url))?; + + Ok(lightwalletd_connect(url.clone(), event_handlers)) +} + +/// Builds up the lightwalletd connection, spawns endless loop that attempts to reconnect to the server +/// in case of connection errors +fn lightwalletd_connect(addr: String, event_handlers: Vec) -> LightwalletdConnection { + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); + let tx = Arc::new(AsyncMutex::new(None)); + + let connect_loop = connect_loop(addr.clone(), responses.clone(), tx.clone(), event_handlers); + + let connect_loop = select_func(connect_loop.boxed(), shutdown_rx.compat()); + spawn(connect_loop.map(|_| ())); + LightwalletdConnection { + addr, + tx, + shutdown_tx: Some(shutdown_tx), + responses, + } +} + +#[cfg(not(target_arch = "wasm32"))] +async fn connect_loop( + addr: String, + responses: JsonRpcPendingRequestsShared, + connection_tx: Arc>>>>, + event_handlers: Vec, +) -> Result<(), ()> { + todo!() +} + /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -352,7 +478,7 @@ async fn create_wallet_db( } pub(super) async fn init_light_client( - lightwalletd_url: Uri, + mut lightwalletd_urls: &Vec, cache_db_path: PathBuf, wallet_db_path: PathBuf, consensus_params: ZcoinConsensusParams, @@ -379,6 +505,21 @@ pub(super) async fn init_light_client( watch_for_tx: None, }; + let mut rng = small_rng(); + lightwalletd_urls.as_mut_slice().shuffle(&mut rng); + + let lightwalletd_impl = LightwalletdImpl::new(); + + for lightwalletd_server in lightwalletd_urls.iter() { + match lightwalletd_impl.add_server(lightwalletd_server).await { + Ok(_) => (), + Err(e) => error!( + "Error {:?} connecting to {:?}. Address won't be used", + e, lightwalletd_server + ), + } + } + let tonic_channel = Channel::builder(lightwalletd_url) .tls_config(ClientTlsConfig::new()) .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? From 1816d60019b432d9c78a30a8c3cb84501778b067 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 2 Sep 2022 14:58:20 +0700 Subject: [PATCH 41/52] r2r native mode support for zcoin --- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_rpc.rs | 153 ++--------------------------------- 2 files changed, 7 insertions(+), 148 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index dae976eded..a358c3dda2 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -794,7 +794,7 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { )?; init_light_client( - light_wallet_d_servers, + uri, cache_db_path, wallet_db_path, self.protocol_info.consensus_params.clone(), diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index d0b73337a0..ecce4d8f76 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,28 +1,23 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; -use crate::utxo::rpc_clients::{JsonRpcPendingRequests, JsonRpcPendingRequestsShared, NativeClient, UtxoRpcClientOps, - NO_TX_ERROR_CODE}; +use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; use async_trait::async_trait; -use common::executor::{spawn, Timer}; -use common::log::{debug, error, info, warn, LogOnError}; -use common::{async_blocking, small_rng, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; +use common::executor::Timer; +use common::log::{debug, error, info, LogOnError}; +use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; -use futures::future::{select as select_func, FutureExt}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; -use futures01::sync::{mpsc, oneshot}; use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; use parking_lot::Mutex; use prost::Message; use protobuf::Message as ProtobufMessage; -use rand::seq::SliceRandom; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::atomic::AtomicU64; use std::sync::Arc; use tokio::task::block_in_place; use tonic::transport::{Channel, ClientTlsConfig}; @@ -41,7 +36,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } -use crate::{RpcTransportEventHandlerShared, ZTransaction}; +use crate::ZTransaction; use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, @@ -231,127 +226,6 @@ impl ZRpcOps for NativeClient { } } -#[derive(Debug)] -struct LightwalletdConnection { - /// The lightwalletd connected to this SocketAddr - addr: String, - /// The Sender forwarding requests to writing part of underlying stream - tx: Arc>>>>, - /// The Sender used to shutdown the background connection loop when LightwalletdConnection is dropped - shutdown_tx: Option>, - /// Responses are stored here - responses: JsonRpcPendingRequestsShared, -} - -impl LightwalletdConnection { - async fn is_connected(&self) -> bool { self.tx.lock().await.is_some() } -} - -impl Drop for LightwalletdConnection { - fn drop(&mut self) { - if let Some(shutdown_tx) = self.shutdown_tx.take() { - if shutdown_tx.send(()).is_err() { - warn!("lightwalletd_connection_drop] Warning, shutdown_tx already closed"); - } - } - } -} - -struct LightwalletdImpl { - connections: AsyncMutex>, - next_id: AtomicU64, - event_handlers: Vec, -} - -impl LightwalletdImpl { - fn new() -> LightwalletdImpl { - LightwalletdImpl { - connections: AsyncMutex::new(vec![]), - next_id: 0.into(), - event_handlers: vec![], - } - } - - // Create an Lightwalletd connection and spawn a green thread actor to handle it. - pub async fn add_server(&self, url: &String) -> Result<(), String> { - let connection = try_s!(spawn_connect(url, self.event_handlers.clone())); - self.connections.lock().await.push(connection); - Ok(()) - } - - /// Remove an Lightwalletd connection and stop corresponding spawned actor. - pub async fn remove_server(&self, server_addr: &str) -> Result<(), String> { - let mut connections = self.connections.lock().await; - // do not use retain, we would have to return an error if we did not find connection by the passd address - let pos = connections - .iter() - .position(|con| con.addr == server_addr) - .ok_or(ERRL!("Unknown lightwalletd address {}", server_addr))?; - // shutdown_tx will be closed immediately on the connection drop - connections.remove(pos); - Ok(()) - } - - /// Moves the lightwalletd servers that fail in a multi request to the end. - pub async fn rotate_servers(&self, no_of_rotations: usize) { - let mut connections = self.connections.lock().await; - connections.rotate_left(no_of_rotations); - } - - /// Check if one of the spawned connections is connected. - pub async fn is_connected(&self) -> bool { - for connection in self.connections.lock().await.iter() { - if connection.is_connected().await { - return true; - } - } - false - } - - pub async fn count_connections(&self) -> usize { self.connections.lock().await.len() } -} - -/// Attempts to process the request (parse url, etc), build up the config and create new lightwalletd connection -#[cfg(not(target_arch = "wasm32"))] -fn spawn_connect( - url: &String, - event_handlers: Vec, -) -> Result { - let uri: Uri = try_s!(url.parse()); - uri.host().ok_or(ERRL!("Couldn't retrieve host from addr {}", url))?; - - Ok(lightwalletd_connect(url.clone(), event_handlers)) -} - -/// Builds up the lightwalletd connection, spawns endless loop that attempts to reconnect to the server -/// in case of connection errors -fn lightwalletd_connect(addr: String, event_handlers: Vec) -> LightwalletdConnection { - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); - let tx = Arc::new(AsyncMutex::new(None)); - - let connect_loop = connect_loop(addr.clone(), responses.clone(), tx.clone(), event_handlers); - - let connect_loop = select_func(connect_loop.boxed(), shutdown_rx.compat()); - spawn(connect_loop.map(|_| ())); - LightwalletdConnection { - addr, - tx, - shutdown_tx: Some(shutdown_tx), - responses, - } -} - -#[cfg(not(target_arch = "wasm32"))] -async fn connect_loop( - addr: String, - responses: JsonRpcPendingRequestsShared, - connection_tx: Arc>>>>, - event_handlers: Vec, -) -> Result<(), ()> { - todo!() -} - /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -478,7 +352,7 @@ async fn create_wallet_db( } pub(super) async fn init_light_client( - mut lightwalletd_urls: &Vec, + lightwalletd_url: Uri, cache_db_path: PathBuf, wallet_db_path: PathBuf, consensus_params: ZcoinConsensusParams, @@ -505,21 +379,6 @@ pub(super) async fn init_light_client( watch_for_tx: None, }; - let mut rng = small_rng(); - lightwalletd_urls.as_mut_slice().shuffle(&mut rng); - - let lightwalletd_impl = LightwalletdImpl::new(); - - for lightwalletd_server in lightwalletd_urls.iter() { - match lightwalletd_impl.add_server(lightwalletd_server).await { - Ok(_) => (), - Err(e) => error!( - "Error {:?} connecting to {:?}. Address won't be used", - e, lightwalletd_server - ), - } - } - let tonic_channel = Channel::builder(lightwalletd_url) .tls_config(ClientTlsConfig::new()) .map_to_mm(ZcoinClientInitError::TlsConfigFailure)? From b80ea9b3ac6e3fd02e3ceabf717df58bf8e24180 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 2 Sep 2022 18:03:38 +0700 Subject: [PATCH 42/52] WIP multi lightwalletd servers support --- mm2src/coins/z_coin.rs | 1 + mm2src/coins/z_coin/z_rpc.rs | 230 ++++++++++++++++++++++++++++++++++- 2 files changed, 226 insertions(+), 5 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a358c3dda2..c70446345a 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -794,6 +794,7 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { )?; init_light_client( + light_wallet_d_servers.clone(), uri, cache_db_path, wallet_db_path, diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index ecce4d8f76..1f494785d2 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,15 +1,18 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; -use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; +use crate::utxo::rpc_clients::{JsonRpcPendingRequests, JsonRpcPendingRequestsShared, NativeClient, UtxoRpcClientOps, + NO_TX_ERROR_CODE}; use async_trait::async_trait; -use common::executor::Timer; -use common::log::{debug, error, info, LogOnError}; -use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; +use common::executor::{spawn, Timer}; +use common::log::{debug, error, info, warn, LogOnError}; +use common::{async_blocking, now_float, now_ms, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; +use futures::future::{select as select_func, FutureExt}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; +use futures01::sync::{mpsc, oneshot}; use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; @@ -18,6 +21,8 @@ use prost::Message; use protobuf::Message as ProtobufMessage; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::str::FromStr; +use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::Arc; use tokio::task::block_in_place; use tonic::transport::{Channel, ClientTlsConfig}; @@ -36,7 +41,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } -use crate::ZTransaction; +use crate::{RpcTransportEventHandler, RpcTransportEventHandlerShared, ZTransaction}; use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, @@ -226,6 +231,203 @@ impl ZRpcOps for NativeClient { } } +#[allow(dead_code)] +fn increase_delay(delay: &AtomicU64) { + if delay.load(AtomicOrdering::Relaxed) < 60 { + delay.fetch_add(5, AtomicOrdering::Relaxed); + } +} + +macro_rules! try_loop { + ($e:expr, $addr: ident, $delay: ident) => { + match $e { + Ok(res) => res, + Err(e) => { + error!("{:?} error {:?}", $addr, e); + increase_delay(&$delay); + continue; + }, + } + }; +} + +#[allow(dead_code)] +struct LightwalletdConnection { + /// The lightwalletd connected to this Addr + addr: String, + /// The Sender forwarding requests to writing part of underlying stream + rpc_client: Arc>>>>, + /// The Sender used to shutdown the background connection loop when LightwalletdConnection is dropped + shutdown_tx: Option>, + /// Responses are stored here + responses: JsonRpcPendingRequestsShared, + /// Selected protocol version. The value is initialized after the server.version RPC call. + protocol_version: AsyncMutex>, +} + +#[allow(dead_code)] +impl LightwalletdConnection { + async fn is_connected(&self) -> bool { self.rpc_client.lock().await.is_some() } +} + +impl Drop for LightwalletdConnection { + fn drop(&mut self) { + if let Some(shutdown_tx) = self.shutdown_tx.take() { + if shutdown_tx.send(()).is_err() { + warn!("lightwalletd_connection_drop] Warning, shutdown_tx already closed"); + } + } + } +} + +#[allow(dead_code)] +pub struct LightwalletdBuilderArgs { + pub spawn_ping: bool, + pub negotiate_version: bool, +} + +impl Default for LightwalletdBuilderArgs { + fn default() -> Self { + LightwalletdBuilderArgs { + spawn_ping: true, + negotiate_version: true, + } + } +} + +#[allow(dead_code)] +struct LightwalletdImpl { + connections: AsyncMutex>>, + next_id: AtomicU64, + event_handlers: Vec, +} + +#[allow(dead_code)] +impl LightwalletdImpl { + fn new() -> LightwalletdImpl { + LightwalletdImpl { + connections: AsyncMutex::new(vec![]), + next_id: 0.into(), + event_handlers: vec![], + } + } + + // Create an Lightwalletd connection and spawn a green thread actor to handle it. + pub async fn add_server(&self, url: &String) -> Result<(), String> { + let connection = try_s!(spawn_connect(url, self.event_handlers.clone())); + self.connections.lock().await.push(connection); + Ok(()) + } + + /// Remove an Lightwalletd connection and stop corresponding spawned actor. + pub async fn remove_server(&self, server_addr: &str) -> Result<(), String> { + let mut connections = self.connections.lock().await; + // do not use retain, we would have to return an error if we did not find connection by the passed address + let pos = connections + .iter() + .position(|con| con.addr == server_addr) + .ok_or(ERRL!("Unknown lightwalletd address {}", server_addr))?; + // shutdown_tx will be closed immediately on the connection drop + connections.remove(pos); + Ok(()) + } + + /// Moves the lightwalletd servers that fail in a multi request to the end. + pub async fn rotate_servers(&self, no_of_rotations: usize) { + let mut connections = self.connections.lock().await; + connections.rotate_left(no_of_rotations); + } + + /// Check if one of the spawned connections is connected. + pub async fn is_connected(&self) -> bool { + for connection in self.connections.lock().await.iter() { + if connection.is_connected().await { + return true; + } + } + false + } + + pub async fn count_connections(&self) -> usize { self.connections.lock().await.len() } +} + +/// Attempts to process the request (parse url, etc), build up the config and create new lightwalletd connection +#[cfg(not(target_arch = "wasm32"))] +#[allow(dead_code)] +fn spawn_connect( + url: &String, + event_handlers: Vec, +) -> Result, String> { + let uri: Uri = try_s!(url.parse()); + uri.host().ok_or(ERRL!("Couldn't retrieve host from addr {}", url))?; + + Ok(lightwalletd_connect(url.clone(), event_handlers)) +} + +/// Builds up the lightwalletd connection, spawns endless loop that attempts to reconnect to the server +/// in case of connection errors +#[allow(dead_code)] +fn lightwalletd_connect( + addr: String, + event_handlers: Vec, +) -> LightwalletdConnection { + let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); + // todo check working with proto format + let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); + let rpc_client = Arc::new(AsyncMutex::new(None)); + + let connect_loop = connect_loop(addr.clone(), responses.clone(), rpc_client.clone(), event_handlers); + + let connect_loop = select_func(connect_loop.boxed(), shutdown_rx.compat()); + spawn(connect_loop.map(|_| ())); + LightwalletdConnection { + addr, + rpc_client, + shutdown_tx: Some(shutdown_tx), + responses, + protocol_version: AsyncMutex::new(None), + } +} + +#[cfg(not(target_arch = "wasm32"))] +#[allow(dead_code)] +async fn connect_loop( + addr: String, + _responses: JsonRpcPendingRequestsShared, + connection_rpc_client: Arc>>>>, + event_handlers: Vec, +) -> Result<(), ()> { + let delay = Arc::new(AtomicU64::new(0)); + + loop { + let current_delay = delay.load(AtomicOrdering::Relaxed); + if current_delay > 0 { + Timer::sleep(current_delay as f64).await; + }; + let uri = try_loop!(Uri::from_str(addr.as_str()), addr, delay); + + let tonic_channel = try_loop!( + Channel::builder(uri) + .tls_config(ClientTlsConfig::new()) + .unwrap() + .connect() + .await, + addr, + delay + ); + let _client = CompactTxStreamerClient::new(tonic_channel); + info!("Light client connected to {}", addr); + try_loop!(event_handlers.on_connected(addr.clone()), addr, delay); + let last_chunk = Arc::new(AtomicU64::new(now_ms())); + let _last_chunk_f = light_last_chunk_loop(last_chunk.clone()).boxed().fuse(); + + let (tx, _rx) = mpsc::channel(0); + // TODO change event_handlers + *connection_rpc_client.lock().await = Some(tx); + unimplemented!() + } +} + /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -352,6 +554,7 @@ async fn create_wallet_db( } pub(super) async fn init_light_client( + _lightwalletd_urls: Vec, lightwalletd_url: Uri, cache_db_path: PathBuf, wallet_db_path: PathBuf, @@ -725,3 +928,20 @@ pub(super) struct SaplingSyncGuard<'a> { pub(super) _connector_guard: AsyncMutexGuard<'a, SaplingSyncConnector>, pub(super) respawn_guard: SaplingSyncRespawnGuard, } + +const LIGHT_TIMEOUT: u64 = 60; + +#[allow(dead_code)] +async fn light_last_chunk_loop(last_chunk: Arc) { + loop { + Timer::sleep(LIGHT_TIMEOUT as f64).await; + let last = (last_chunk.load(AtomicOrdering::Relaxed) / 1000) as f64; + if now_float() - last > LIGHT_TIMEOUT as f64 { + warn!( + "Didn't receive any data since {}. Shutting down the connection.", + last as i64 + ); + break; + } + } +} From 2663a56994cb31ad3a45a5d5a90345fc61c1fdd0 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 2 Sep 2022 18:36:34 +0700 Subject: [PATCH 43/52] WIP use drop_mutability --- mm2src/coins/z_coin/z_rpc.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 1f494785d2..e94bc2bea8 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -171,8 +171,8 @@ impl ZRpcOps for NativeClient { outputs.push(compact_out); } // Shadowing mut variables as immutable. No longer need to update them. - let spends = spends; - let outputs = outputs; + drop_mutability!(spends); + drop_mutability!(outputs); let mut hash_tx_vec = hash_tx.0.to_vec(); hash_tx_vec.reverse(); @@ -192,9 +192,9 @@ impl ZRpcOps for NativeClient { let mut prev_hash = block.previousblockhash.unwrap_or_default().0.to_vec(); prev_hash.reverse(); // Shadowing mut variables as immutable. - let hash = hash; - let prev_hash = prev_hash; - let compact_txs = compact_txs; + drop_mutability!(hash); + drop_mutability!(prev_hash); + drop_mutability!(compact_txs); let compact_block = TonicCompactBlock { proto_version: 0, @@ -253,7 +253,7 @@ macro_rules! try_loop { #[allow(dead_code)] struct LightwalletdConnection { - /// The lightwalletd connected to this Addr + /// The lightwalletd connected to this addr addr: String, /// The Sender forwarding requests to writing part of underlying stream rpc_client: Arc>>>>, From 2b2b9ca86e2a8c3a198acd915db75696e2aa259e Mon Sep 17 00:00:00 2001 From: laruh Date: Tue, 6 Sep 2022 18:05:32 +0700 Subject: [PATCH 44/52] WIP added RpcClientInitErr, one cache paths, attempts fixed --- mm2src/coins/z_coin.rs | 7 +++---- mm2src/coins/z_coin/z_coin_errors.rs | 4 ++-- mm2src/coins/z_coin/z_rpc.rs | 22 ++++++++++++---------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index c70446345a..e3a4306832 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -766,11 +766,12 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { ); let evk = ExtendedFullViewingKey::from(&self.z_spending_key); + let cache_db_path = self.db_dir_path.join(format!("{}_cache.db", self.ticker)); + let wallet_db_path = self.db_dir_path.join(format!("{}_wallet.db", self.ticker)); let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { ZcoinRpcMode::Native => { + // todo should be the error handle instead of unwrap let native_client: NativeClient = self.native_client().unwrap(); - let cache_db_path = self.db_dir_path.join(format!("{}_native_cache.db", self.ticker)); - let wallet_db_path = self.db_dir_path.join(format!("{}_native_wallet.db", self.ticker)); init_native_client( native_client, cache_db_path, @@ -784,8 +785,6 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { ZcoinRpcMode::Light { light_wallet_d_servers, .. } => { - let cache_db_path = self.db_dir_path.join(format!("{}_light_cache.db", self.ticker)); - let wallet_db_path = self.db_dir_path.join(format!("{}_light_wallet.db", self.ticker)); // TODO multi lightwalletd servers support will be added on the next iteration let uri = Uri::from_str( light_wallet_d_servers diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 314a419d7e..9169a4f668 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -196,7 +196,7 @@ pub enum ZCoinBuildError { Io(std::io::Error), EmptyLightwalletdUris, InvalidLightwalletdUri(InvalidUri), - ClientInitErr(ZcoinClientInitError), + RpcClientInitErr(ZcoinClientInitError), ZCashParamsNotFound, } @@ -221,7 +221,7 @@ impl From for ZCoinBuildError { } impl From for ZCoinBuildError { - fn from(err: ZcoinClientInitError) -> Self { ZCoinBuildError::ClientInitErr(err) } + fn from(err: ZcoinClientInitError) -> Self { ZCoinBuildError::RpcClientInitErr(err) } } pub(super) enum SqlTxHistoryError { diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index e94bc2bea8..d4e7bd4fa4 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -66,7 +66,7 @@ pub trait ZRpcOps { on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), ) -> Result<(), MmError>; - async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option); + async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option); } #[async_trait] @@ -102,7 +102,8 @@ impl ZRpcOps for CompactTxStreamerClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option) { + let mut attempts = 0; loop { let filter = TxFilter { block: None, @@ -115,11 +116,11 @@ impl ZRpcOps for CompactTxStreamerClient { Err(e) => { error!("Error on getting tx {}", tx_id); if e.message().contains(NO_TX_ERROR_CODE) { - if *attempts >= 3 { + if attempts >= 3 { *watch_for_tx = None; return; } - *attempts += 1; + attempts += 1; } Timer::sleep(30.).await; }, @@ -211,18 +212,19 @@ impl ZRpcOps for NativeClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId, attempts: &mut i32, watch_for_tx: &mut Option) { + async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option) { + let mut attempts = 0; loop { match self.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { Ok(_) => break, Err(e) => { error!("Error on getting tx {}", tx_id); if e.to_string().contains(NO_TX_ERROR_CODE) { - if *attempts >= 3 { + if attempts >= 3 { *watch_for_tx = None; return; } - *attempts += 1; + attempts += 1; } Timer::sleep(30.).await; }, @@ -608,6 +610,7 @@ pub(super) async fn init_native_client( let blocks_db = async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; + // todo check async_blocking for wallet_db as it was previously let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); @@ -807,9 +810,8 @@ impl SaplingSyncLoopHandle { async fn check_watch_for_tx_existence(&mut self, rpc: &mut (dyn ZRpcOps + Send)) { if let Some(tx_id) = self.watch_for_tx { - let mut attempts = 0; - rpc.check_watch_for_tx(tx_id, &mut attempts, &mut self.watch_for_tx) - .await; + // todo watch_for_tx must be controlled by the code that calls check_watch_for_tx + rpc.check_watch_for_tx(tx_id, &mut self.watch_for_tx).await; } } } From 7f89dfa2c0c3b43d4fd2bc984a07341954dd94c3 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 7 Sep 2022 13:35:08 +0700 Subject: [PATCH 45/52] WIP propagate the error instead of panic, mutable watch_for_tx removed from check_watch_for_tx() --- mm2src/coins/z_coin.rs | 7 +++---- mm2src/coins/z_coin/z_rpc.rs | 19 ++++++++++--------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index e3a4306832..356fe5792f 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -1,7 +1,7 @@ use crate::my_tx_history_v2::{MyTxHistoryErrorV2, MyTxHistoryRequestV2, MyTxHistoryResponseV2}; use crate::rpc_command::init_withdraw::{InitWithdrawCoin, WithdrawInProgressStatus, WithdrawTaskHandle}; -use crate::utxo::rpc_clients::{ElectrumRpcRequest, NativeClient, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, - UtxoRpcFut, UtxoRpcResult}; +use crate::utxo::rpc_clients::{ElectrumRpcRequest, UnspentInfo, UtxoRpcClientEnum, UtxoRpcError, UtxoRpcFut, + UtxoRpcResult}; use crate::utxo::utxo_builder::{UtxoCoinBuilderCommonOps, UtxoCoinWithIguanaPrivKeyBuilder, UtxoFieldsWithIguanaPrivKeyBuilder}; use crate::utxo::utxo_common::{addresses_from_script, big_decimal_from_sat, big_decimal_from_sat_unsigned, @@ -770,8 +770,7 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { let wallet_db_path = self.db_dir_path.join(format!("{}_wallet.db", self.ticker)); let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { ZcoinRpcMode::Native => { - // todo should be the error handle instead of unwrap - let native_client: NativeClient = self.native_client().unwrap(); + let native_client = self.native_client()?; init_native_client( native_client, cache_db_path, diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index d4e7bd4fa4..39a325b4bf 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -66,7 +66,7 @@ pub trait ZRpcOps { on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), ) -> Result<(), MmError>; - async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option); + async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()>; } #[async_trait] @@ -102,7 +102,7 @@ impl ZRpcOps for CompactTxStreamerClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option) { + async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()> { let mut attempts = 0; loop { let filter = TxFilter { @@ -117,8 +117,7 @@ impl ZRpcOps for CompactTxStreamerClient { error!("Error on getting tx {}", tx_id); if e.message().contains(NO_TX_ERROR_CODE) { if attempts >= 3 { - *watch_for_tx = None; - return; + return Err(()); } attempts += 1; } @@ -126,6 +125,7 @@ impl ZRpcOps for CompactTxStreamerClient { }, } } + Ok(()) } } @@ -212,7 +212,7 @@ impl ZRpcOps for NativeClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId, watch_for_tx: &mut Option) { + async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()> { let mut attempts = 0; loop { match self.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { @@ -221,8 +221,7 @@ impl ZRpcOps for NativeClient { error!("Error on getting tx {}", tx_id); if e.to_string().contains(NO_TX_ERROR_CODE) { if attempts >= 3 { - *watch_for_tx = None; - return; + return Err(()); } attempts += 1; } @@ -230,6 +229,7 @@ impl ZRpcOps for NativeClient { }, } } + Ok(()) } } @@ -810,8 +810,9 @@ impl SaplingSyncLoopHandle { async fn check_watch_for_tx_existence(&mut self, rpc: &mut (dyn ZRpcOps + Send)) { if let Some(tx_id) = self.watch_for_tx { - // todo watch_for_tx must be controlled by the code that calls check_watch_for_tx - rpc.check_watch_for_tx(tx_id, &mut self.watch_for_tx).await; + if rpc.check_watch_for_tx(tx_id).await.is_err() { + self.watch_for_tx = None; + } } } } From ea8f18fa7b7298fc1e38accb7f6a39a9a4c0b546 Mon Sep 17 00:00:00 2001 From: laruh Date: Wed, 7 Sep 2022 17:14:51 +0700 Subject: [PATCH 46/52] WIP async_blocking added in create_wallet_db --- mm2src/coins/z_coin/z_rpc.rs | 38 ++++++++++++++++++++---------------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 39a325b4bf..2210a44dbb 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -536,23 +536,28 @@ async fn create_wallet_db( check_point_block: Option, evk: ExtendedFullViewingKey, ) -> Result, MmError> { - let db = - WalletDb::for_path(wallet_db_path, consensus_params).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; - if db.get_extended_full_viewing_keys()?.is_empty() { - init_accounts_table(&db, &[evk])?; - if let Some(check_point) = check_point_block { - init_blocks_table( - &db, - BlockHeight::from_u32(check_point.height), - BlockHash(check_point.hash.0), - check_point.time, - &check_point.sapling_tree.0, - )?; + async_blocking({ + move || -> Result, MmError> { + let db = WalletDb::for_path(wallet_db_path, consensus_params) + .map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + run_optimization_pragmas(db.sql_conn()).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + init_wallet_db(&db).map_to_mm(ZcoinClientInitError::WalletDbInitFailure)?; + if db.get_extended_full_viewing_keys()?.is_empty() { + init_accounts_table(&db, &[evk])?; + if let Some(check_point) = check_point_block { + init_blocks_table( + &db, + BlockHeight::from_u32(check_point.height), + BlockHash(check_point.hash.0), + check_point.time, + &check_point.sapling_tree.0, + )?; + } + } + Ok(db) } - } - Ok(db) + }) + .await } pub(super) async fn init_light_client( @@ -610,7 +615,6 @@ pub(super) async fn init_native_client( let blocks_db = async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) .await?; - // todo check async_blocking for wallet_db as it was previously let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); From 610a2f4d245dbdd91fd2ed436810e28dcfb0456d Mon Sep 17 00:00:00 2001 From: laruh Date: Thu, 8 Sep 2022 13:54:32 +0700 Subject: [PATCH 47/52] remove some unnecessary code for multi servers --- mm2src/coins/z_coin/z_rpc.rs | 76 +++--------------------------------- 1 file changed, 6 insertions(+), 70 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 2210a44dbb..462d6097f8 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -1,15 +1,13 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; -use crate::utxo::rpc_clients::{JsonRpcPendingRequests, JsonRpcPendingRequestsShared, NativeClient, UtxoRpcClientOps, - NO_TX_ERROR_CODE}; +use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; use async_trait::async_trait; -use common::executor::{spawn, Timer}; +use common::executor::Timer; use common::log::{debug, error, info, warn, LogOnError}; -use common::{async_blocking, now_float, now_ms, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; +use common::{async_blocking, now_float, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; -use futures::future::{select as select_func, FutureExt}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; use futures01::sync::{mpsc, oneshot}; @@ -21,7 +19,6 @@ use prost::Message; use protobuf::Message as ProtobufMessage; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::str::FromStr; use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::Arc; use tokio::task::block_in_place; @@ -41,7 +38,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } -use crate::{RpcTransportEventHandler, RpcTransportEventHandlerShared, ZTransaction}; +use crate::{RpcTransportEventHandlerShared, ZTransaction}; use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, @@ -240,19 +237,6 @@ fn increase_delay(delay: &AtomicU64) { } } -macro_rules! try_loop { - ($e:expr, $addr: ident, $delay: ident) => { - match $e { - Ok(res) => res, - Err(e) => { - error!("{:?} error {:?}", $addr, e); - increase_delay(&$delay); - continue; - }, - } - }; -} - #[allow(dead_code)] struct LightwalletdConnection { /// The lightwalletd connected to this addr @@ -261,8 +245,6 @@ struct LightwalletdConnection { rpc_client: Arc>>>>, /// The Sender used to shutdown the background connection loop when LightwalletdConnection is dropped shutdown_tx: Option>, - /// Responses are stored here - responses: JsonRpcPendingRequestsShared, /// Selected protocol version. The value is initialized after the server.version RPC call. protocol_version: AsyncMutex>, } @@ -371,65 +353,19 @@ fn spawn_connect( #[allow(dead_code)] fn lightwalletd_connect( addr: String, - event_handlers: Vec, + _event_handlers: Vec, ) -> LightwalletdConnection { - let (shutdown_tx, shutdown_rx) = oneshot::channel::<()>(); - // todo check working with proto format - let responses = Arc::new(AsyncMutex::new(JsonRpcPendingRequests::default())); + let (shutdown_tx, _shutdown_rx) = oneshot::channel::<()>(); let rpc_client = Arc::new(AsyncMutex::new(None)); - let connect_loop = connect_loop(addr.clone(), responses.clone(), rpc_client.clone(), event_handlers); - - let connect_loop = select_func(connect_loop.boxed(), shutdown_rx.compat()); - spawn(connect_loop.map(|_| ())); LightwalletdConnection { addr, rpc_client, shutdown_tx: Some(shutdown_tx), - responses, protocol_version: AsyncMutex::new(None), } } -#[cfg(not(target_arch = "wasm32"))] -#[allow(dead_code)] -async fn connect_loop( - addr: String, - _responses: JsonRpcPendingRequestsShared, - connection_rpc_client: Arc>>>>, - event_handlers: Vec, -) -> Result<(), ()> { - let delay = Arc::new(AtomicU64::new(0)); - - loop { - let current_delay = delay.load(AtomicOrdering::Relaxed); - if current_delay > 0 { - Timer::sleep(current_delay as f64).await; - }; - let uri = try_loop!(Uri::from_str(addr.as_str()), addr, delay); - - let tonic_channel = try_loop!( - Channel::builder(uri) - .tls_config(ClientTlsConfig::new()) - .unwrap() - .connect() - .await, - addr, - delay - ); - let _client = CompactTxStreamerClient::new(tonic_channel); - info!("Light client connected to {}", addr); - try_loop!(event_handlers.on_connected(addr.clone()), addr, delay); - let last_chunk = Arc::new(AtomicU64::new(now_ms())); - let _last_chunk_f = light_last_chunk_loop(last_chunk.clone()).boxed().fuse(); - - let (tx, _rx) = mpsc::channel(0); - // TODO change event_handlers - *connection_rpc_client.lock().await = Some(tx); - unimplemented!() - } -} - /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); From e1f0addccf0b823b9a02fa0e830f09413ceed2ea Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Sep 2022 17:19:33 +0700 Subject: [PATCH 48/52] remove unused code --- mm2src/coins/z_coin/z_rpc.rs | 161 +---------------------------------- 1 file changed, 3 insertions(+), 158 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 462d6097f8..8d2e77ce81 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -2,15 +2,14 @@ use super::{z_coin_errors::*, CheckPointBlockInfo, ZcoinConsensusParams}; use crate::utxo::rpc_clients::{NativeClient, UtxoRpcClientOps, NO_TX_ERROR_CODE}; use async_trait::async_trait; use common::executor::Timer; -use common::log::{debug, error, info, warn, LogOnError}; -use common::{async_blocking, now_float, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; +use common::log::{debug, error, info, LogOnError}; +use common::{async_blocking, spawn_abortable, AbortOnDropHandle, Future01CompatExt}; use db_common::sqlite::rusqlite::{params, Connection, Error as SqliteError, NO_PARAMS}; use db_common::sqlite::{query_single_row, run_optimization_pragmas}; use futures::channel::mpsc::{channel, Receiver as AsyncReceiver, Sender as AsyncSender}; use futures::channel::oneshot::{channel as oneshot_channel, Sender as OneshotSender}; use futures::lock::{Mutex as AsyncMutex, MutexGuard as AsyncMutexGuard}; use futures::StreamExt; -use futures01::sync::{mpsc, oneshot}; use group::GroupEncoding; use http::Uri; use mm2_err_handle::prelude::*; @@ -19,7 +18,6 @@ use prost::Message; use protobuf::Message as ProtobufMessage; use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicU64, Ordering as AtomicOrdering}; use std::sync::Arc; use tokio::task::block_in_place; use tonic::transport::{Channel, ClientTlsConfig}; @@ -38,7 +36,7 @@ use zcash_primitives::zip32::ExtendedFullViewingKey; mod z_coin_grpc { tonic::include_proto!("cash.z.wallet.sdk.rpc"); } -use crate::{RpcTransportEventHandlerShared, ZTransaction}; +use crate::ZTransaction; use rpc::v1::types::H256 as H256Json; use z_coin_grpc::compact_tx_streamer_client::CompactTxStreamerClient; use z_coin_grpc::{BlockId, BlockRange, ChainSpec, CompactBlock as TonicCompactBlock, @@ -230,142 +228,6 @@ impl ZRpcOps for NativeClient { } } -#[allow(dead_code)] -fn increase_delay(delay: &AtomicU64) { - if delay.load(AtomicOrdering::Relaxed) < 60 { - delay.fetch_add(5, AtomicOrdering::Relaxed); - } -} - -#[allow(dead_code)] -struct LightwalletdConnection { - /// The lightwalletd connected to this addr - addr: String, - /// The Sender forwarding requests to writing part of underlying stream - rpc_client: Arc>>>>, - /// The Sender used to shutdown the background connection loop when LightwalletdConnection is dropped - shutdown_tx: Option>, - /// Selected protocol version. The value is initialized after the server.version RPC call. - protocol_version: AsyncMutex>, -} - -#[allow(dead_code)] -impl LightwalletdConnection { - async fn is_connected(&self) -> bool { self.rpc_client.lock().await.is_some() } -} - -impl Drop for LightwalletdConnection { - fn drop(&mut self) { - if let Some(shutdown_tx) = self.shutdown_tx.take() { - if shutdown_tx.send(()).is_err() { - warn!("lightwalletd_connection_drop] Warning, shutdown_tx already closed"); - } - } - } -} - -#[allow(dead_code)] -pub struct LightwalletdBuilderArgs { - pub spawn_ping: bool, - pub negotiate_version: bool, -} - -impl Default for LightwalletdBuilderArgs { - fn default() -> Self { - LightwalletdBuilderArgs { - spawn_ping: true, - negotiate_version: true, - } - } -} - -#[allow(dead_code)] -struct LightwalletdImpl { - connections: AsyncMutex>>, - next_id: AtomicU64, - event_handlers: Vec, -} - -#[allow(dead_code)] -impl LightwalletdImpl { - fn new() -> LightwalletdImpl { - LightwalletdImpl { - connections: AsyncMutex::new(vec![]), - next_id: 0.into(), - event_handlers: vec![], - } - } - - // Create an Lightwalletd connection and spawn a green thread actor to handle it. - pub async fn add_server(&self, url: &String) -> Result<(), String> { - let connection = try_s!(spawn_connect(url, self.event_handlers.clone())); - self.connections.lock().await.push(connection); - Ok(()) - } - - /// Remove an Lightwalletd connection and stop corresponding spawned actor. - pub async fn remove_server(&self, server_addr: &str) -> Result<(), String> { - let mut connections = self.connections.lock().await; - // do not use retain, we would have to return an error if we did not find connection by the passed address - let pos = connections - .iter() - .position(|con| con.addr == server_addr) - .ok_or(ERRL!("Unknown lightwalletd address {}", server_addr))?; - // shutdown_tx will be closed immediately on the connection drop - connections.remove(pos); - Ok(()) - } - - /// Moves the lightwalletd servers that fail in a multi request to the end. - pub async fn rotate_servers(&self, no_of_rotations: usize) { - let mut connections = self.connections.lock().await; - connections.rotate_left(no_of_rotations); - } - - /// Check if one of the spawned connections is connected. - pub async fn is_connected(&self) -> bool { - for connection in self.connections.lock().await.iter() { - if connection.is_connected().await { - return true; - } - } - false - } - - pub async fn count_connections(&self) -> usize { self.connections.lock().await.len() } -} - -/// Attempts to process the request (parse url, etc), build up the config and create new lightwalletd connection -#[cfg(not(target_arch = "wasm32"))] -#[allow(dead_code)] -fn spawn_connect( - url: &String, - event_handlers: Vec, -) -> Result, String> { - let uri: Uri = try_s!(url.parse()); - uri.host().ok_or(ERRL!("Couldn't retrieve host from addr {}", url))?; - - Ok(lightwalletd_connect(url.clone(), event_handlers)) -} - -/// Builds up the lightwalletd connection, spawns endless loop that attempts to reconnect to the server -/// in case of connection errors -#[allow(dead_code)] -fn lightwalletd_connect( - addr: String, - _event_handlers: Vec, -) -> LightwalletdConnection { - let (shutdown_tx, _shutdown_rx) = oneshot::channel::<()>(); - let rpc_client = Arc::new(AsyncMutex::new(None)); - - LightwalletdConnection { - addr, - rpc_client, - shutdown_tx: Some(shutdown_tx), - protocol_version: AsyncMutex::new(None), - } -} - /// A wrapper for the SQLite connection to the block cache database. pub struct BlockDb(Connection); @@ -871,20 +733,3 @@ pub(super) struct SaplingSyncGuard<'a> { pub(super) _connector_guard: AsyncMutexGuard<'a, SaplingSyncConnector>, pub(super) respawn_guard: SaplingSyncRespawnGuard, } - -const LIGHT_TIMEOUT: u64 = 60; - -#[allow(dead_code)] -async fn light_last_chunk_loop(last_chunk: Arc) { - loop { - Timer::sleep(LIGHT_TIMEOUT as f64).await; - let last = (last_chunk.load(AtomicOrdering::Relaxed) / 1000) as f64; - if now_float() - last > LIGHT_TIMEOUT as f64 { - warn!( - "Didn't receive any data since {}. Shutting down the connection.", - last as i64 - ); - break; - } - } -} From a3cfe1fa4d5d494a9f19931b53c84ccb65b09a85 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Sep 2022 18:06:52 +0700 Subject: [PATCH 49/52] type OnCompactBlockFn added --- mm2src/coins/z_coin/z_rpc.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 8d2e77ce81..281de1fcea 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -50,6 +50,9 @@ struct CompactBlockRow { data: Vec, } +pub type OnCompactBlockFn<'a> = + dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync + 'a; + #[async_trait] pub trait ZRpcOps { async fn get_block_height(&mut self) -> Result>; @@ -58,7 +61,7 @@ pub trait ZRpcOps { &mut self, start_block: u64, last_block: u64, - on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + on_block: &mut OnCompactBlockFn, ) -> Result<(), MmError>; async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()>; @@ -77,7 +80,7 @@ impl ZRpcOps for CompactTxStreamerClient { &mut self, start_block: u64, last_block: u64, - on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + on_block: &mut OnCompactBlockFn, ) -> Result<(), MmError> { let request = tonic::Request::new(BlockRange { start: Some(BlockId { @@ -134,7 +137,7 @@ impl ZRpcOps for NativeClient { &mut self, start_block: u64, last_block: u64, - on_block: &mut (dyn FnMut(TonicCompactBlock) -> Result<(), MmError> + Send + Sync), + on_block: &mut OnCompactBlockFn, ) -> Result<(), MmError> { for height in start_block..=last_block { let block = self.get_block_by_height(height).await?; From 45ad18f4d9e8e115c7059a400e98aa5e06ccc945 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Sep 2022 19:56:52 +0700 Subject: [PATCH 50/52] rename to check_tx_existence, add return bool, databases init moved to ZCoinBuilder::build --- mm2src/coins/z_coin.rs | 25 ++++++++++++++------- mm2src/coins/z_coin/z_rpc.rs | 42 +++++++++++------------------------- 2 files changed, 30 insertions(+), 37 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 356fe5792f..89dca47579 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -38,6 +38,7 @@ use mm2_core::mm_ctx::MmArc; use mm2_err_handle::prelude::*; use mm2_number::{BigDecimal, MmNumber}; #[cfg(test)] use mocktopus::macros::*; +use parking_lot::Mutex; use primitives::bytes::Bytes; use rpc::v1::types::{Bytes as BytesJson, Transaction as RpcTransaction, H256 as H256Json}; use script::{Builder as ScriptBuilder, Opcode, Script, TransactionInputSigner}; @@ -73,6 +74,7 @@ pub use z_rpc::SyncStatus; use z_rpc::{init_light_client, init_native_client, SaplingSyncConnector, SaplingSyncGuard, WalletDbShared}; mod z_coin_errors; +use crate::z_coin::z_rpc::{create_wallet_db, BlockDb}; pub use z_coin_errors::*; #[cfg(all(test, feature = "zhtlc-native-tests"))] @@ -768,16 +770,25 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { let evk = ExtendedFullViewingKey::from(&self.z_spending_key); let cache_db_path = self.db_dir_path.join(format!("{}_cache.db", self.ticker)); let wallet_db_path = self.db_dir_path.join(format!("{}_wallet.db", self.ticker)); + let blocks_db = + async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) + .await?; + let wallet_db = create_wallet_db( + wallet_db_path, + self.protocol_info.consensus_params.clone(), + self.protocol_info.check_point_block.clone(), + evk, + ) + .await?; + let wallet_db = Arc::new(Mutex::new(wallet_db)); let (sync_state_connector, light_wallet_db) = match &self.z_coin_params.mode { ZcoinRpcMode::Native => { let native_client = self.native_client()?; init_native_client( native_client, - cache_db_path, - wallet_db_path, + blocks_db, + wallet_db, self.protocol_info.consensus_params.clone(), - self.protocol_info.check_point_block, - evk, ) .await? }, @@ -794,11 +805,9 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { init_light_client( light_wallet_d_servers.clone(), uri, - cache_db_path, - wallet_db_path, + blocks_db, + wallet_db, self.protocol_info.consensus_params.clone(), - self.protocol_info.check_point_block, - evk, ) .await? }, diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 281de1fcea..6bc32b04af 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -64,7 +64,7 @@ pub trait ZRpcOps { on_block: &mut OnCompactBlockFn, ) -> Result<(), MmError>; - async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()>; + async fn check_tx_existence(&mut self, tx_id: TxId) -> bool; } #[async_trait] @@ -100,7 +100,7 @@ impl ZRpcOps for CompactTxStreamerClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()> { + async fn check_tx_existence(&mut self, tx_id: TxId) -> bool { let mut attempts = 0; loop { let filter = TxFilter { @@ -115,7 +115,7 @@ impl ZRpcOps for CompactTxStreamerClient { error!("Error on getting tx {}", tx_id); if e.message().contains(NO_TX_ERROR_CODE) { if attempts >= 3 { - return Err(()); + return false; } attempts += 1; } @@ -123,7 +123,7 @@ impl ZRpcOps for CompactTxStreamerClient { }, } } - Ok(()) + true } } @@ -210,7 +210,7 @@ impl ZRpcOps for NativeClient { Ok(()) } - async fn check_watch_for_tx(&mut self, tx_id: TxId) -> Result<(), ()> { + async fn check_tx_existence(&mut self, tx_id: TxId) -> bool { let mut attempts = 0; loop { match self.get_raw_transaction_bytes(&H256Json::from(tx_id.0)).compat().await { @@ -219,7 +219,7 @@ impl ZRpcOps for NativeClient { error!("Error on getting tx {}", tx_id); if e.to_string().contains(NO_TX_ERROR_CODE) { if attempts >= 3 { - return Err(()); + return false; } attempts += 1; } @@ -227,7 +227,7 @@ impl ZRpcOps for NativeClient { }, } } - Ok(()) + true } } @@ -331,7 +331,7 @@ impl BlockSource for BlockDb { } } -async fn create_wallet_db( +pub async fn create_wallet_db( wallet_db_path: PathBuf, consensus_params: ZcoinConsensusParams, check_point_block: Option, @@ -364,22 +364,13 @@ async fn create_wallet_db( pub(super) async fn init_light_client( _lightwalletd_urls: Vec, lightwalletd_url: Uri, - cache_db_path: PathBuf, - wallet_db_path: PathBuf, + blocks_db: BlockDb, + wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, - check_point_block: Option, - evk: ExtendedFullViewingKey, ) -> Result<(AsyncMutex, WalletDbShared), MmError> { - let blocks_db = - async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) - .await?; - - let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; - let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); - let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), blocks_db: Mutex::new(blocks_db), @@ -407,20 +398,13 @@ pub(super) async fn init_light_client( pub(super) async fn init_native_client( native_client: NativeClient, - cache_db_path: PathBuf, - wallet_db_path: PathBuf, + blocks_db: BlockDb, + wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, - check_point_block: Option, - evk: ExtendedFullViewingKey, ) -> Result<(AsyncMutex, WalletDbShared), MmError> { - let blocks_db = - async_blocking(|| BlockDb::for_path(cache_db_path).map_to_mm(ZcoinClientInitError::BlocksDbInitFailure)) - .await?; - let wallet_db = create_wallet_db(wallet_db_path, consensus_params.clone(), check_point_block, evk).await?; let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); - let wallet_db = Arc::new(Mutex::new(wallet_db)); let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), blocks_db: Mutex::new(blocks_db), @@ -615,7 +599,7 @@ impl SaplingSyncLoopHandle { async fn check_watch_for_tx_existence(&mut self, rpc: &mut (dyn ZRpcOps + Send)) { if let Some(tx_id) = self.watch_for_tx { - if rpc.check_watch_for_tx(tx_id).await.is_err() { + if !rpc.check_tx_existence(tx_id).await { self.watch_for_tx = None; } } From da01b73bc816bde6d42db6e29a14bd98fe14d572 Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Sep 2022 20:46:01 +0700 Subject: [PATCH 51/52] leave Vec of urls for init_light_client --- mm2src/coins/z_coin.rs | 11 +---------- mm2src/coins/z_coin/z_coin_errors.rs | 12 ++++++------ mm2src/coins/z_coin/z_rpc.rs | 10 ++++++++-- 3 files changed, 15 insertions(+), 18 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index 89dca47579..a9b6b0bf7c 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -31,7 +31,6 @@ use futures::compat::Future01CompatExt; use futures::lock::Mutex as AsyncMutex; use futures::{FutureExt, TryFutureExt}; use futures01::Future; -use http::Uri; use keys::hash::H256; use keys::{KeyPair, Message, Public}; use mm2_core::mm_ctx::MmArc; @@ -46,7 +45,6 @@ use serde_json::Value as Json; use serialization::CoinVariant; use std::collections::{HashMap, HashSet}; use std::path::PathBuf; -use std::str::FromStr; use std::sync::Arc; use zcash_client_backend::data_api::WalletRead; use zcash_client_backend::encoding::{decode_payment_address, encode_extended_spending_key, encode_payment_address}; @@ -796,15 +794,8 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { light_wallet_d_servers, .. } => { // TODO multi lightwalletd servers support will be added on the next iteration - let uri = Uri::from_str( - light_wallet_d_servers - .first() - .or_mm_err(|| ZCoinBuildError::EmptyLightwalletdUris)?, - )?; - init_light_client( - light_wallet_d_servers.clone(), - uri, + light_wallet_d_servers, blocks_db, wallet_db, self.protocol_info.consensus_params.clone(), diff --git a/mm2src/coins/z_coin/z_coin_errors.rs b/mm2src/coins/z_coin/z_coin_errors.rs index 9169a4f668..f05106cfe8 100644 --- a/mm2src/coins/z_coin/z_coin_errors.rs +++ b/mm2src/coins/z_coin/z_coin_errors.rs @@ -53,12 +53,18 @@ pub enum ZcoinClientInitError { BlocksDbInitFailure(SqliteError), WalletDbInitFailure(SqliteError), ZcashSqliteError(ZcashClientError), + EmptyLightwalletdUris, + InvalidUri(InvalidUri), } impl From for ZcoinClientInitError { fn from(err: ZcashClientError) -> Self { ZcoinClientInitError::ZcashSqliteError(err) } } +impl From for ZcoinClientInitError { + fn from(err: InvalidUri) -> Self { ZcoinClientInitError::InvalidUri(err) } +} + #[derive(Debug, Display)] #[display(fmt = "Blockchain scan process stopped")] pub struct BlockchainScanStopped {} @@ -194,8 +200,6 @@ pub enum ZCoinBuildError { path: String, }, Io(std::io::Error), - EmptyLightwalletdUris, - InvalidLightwalletdUri(InvalidUri), RpcClientInitErr(ZcoinClientInitError), ZCashParamsNotFound, } @@ -216,10 +220,6 @@ impl From for ZCoinBuildError { fn from(err: std::io::Error) -> ZCoinBuildError { ZCoinBuildError::Io(err) } } -impl From for ZCoinBuildError { - fn from(err: InvalidUri) -> Self { ZCoinBuildError::InvalidLightwalletdUri(err) } -} - impl From for ZCoinBuildError { fn from(err: ZcoinClientInitError) -> Self { ZCoinBuildError::RpcClientInitErr(err) } } diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index 6bc32b04af..dd7700a3c9 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -18,6 +18,7 @@ use prost::Message; use protobuf::Message as ProtobufMessage; use std::ops::Deref; use std::path::{Path, PathBuf}; +use std::str::FromStr; use std::sync::Arc; use tokio::task::block_in_place; use tonic::transport::{Channel, ClientTlsConfig}; @@ -362,8 +363,7 @@ pub async fn create_wallet_db( } pub(super) async fn init_light_client( - _lightwalletd_urls: Vec, - lightwalletd_url: Uri, + lightwalletd_urls: &Vec, blocks_db: BlockDb, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams, @@ -371,6 +371,12 @@ pub(super) async fn init_light_client( let (sync_status_notifier, sync_watcher) = channel(1); let (on_tx_gen_notifier, on_tx_gen_watcher) = channel(1); + let lightwalletd_url = Uri::from_str( + lightwalletd_urls + .first() + .or_mm_err(|| ZcoinClientInitError::EmptyLightwalletdUris)?, + )?; + let sync_handle = SaplingSyncLoopHandle { current_block: BlockHeight::from_u32(0), blocks_db: Mutex::new(blocks_db), From e61a4d53545befe4f9058d301c951969beb61c5b Mon Sep 17 00:00:00 2001 From: laruh Date: Fri, 9 Sep 2022 21:06:48 +0700 Subject: [PATCH 52/52] fix clippy --- mm2src/coins/z_coin.rs | 2 +- mm2src/coins/z_coin/z_rpc.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mm2src/coins/z_coin.rs b/mm2src/coins/z_coin.rs index a9b6b0bf7c..853e179c25 100644 --- a/mm2src/coins/z_coin.rs +++ b/mm2src/coins/z_coin.rs @@ -795,7 +795,7 @@ impl<'a> UtxoCoinWithIguanaPrivKeyBuilder for ZCoinBuilder<'a> { } => { // TODO multi lightwalletd servers support will be added on the next iteration init_light_client( - light_wallet_d_servers, + light_wallet_d_servers.clone(), blocks_db, wallet_db, self.protocol_info.consensus_params.clone(), diff --git a/mm2src/coins/z_coin/z_rpc.rs b/mm2src/coins/z_coin/z_rpc.rs index dd7700a3c9..0c8023b728 100644 --- a/mm2src/coins/z_coin/z_rpc.rs +++ b/mm2src/coins/z_coin/z_rpc.rs @@ -363,7 +363,7 @@ pub async fn create_wallet_db( } pub(super) async fn init_light_client( - lightwalletd_urls: &Vec, + lightwalletd_urls: Vec, blocks_db: BlockDb, wallet_db: WalletDbShared, consensus_params: ZcoinConsensusParams,