From e7a7dab685b7a07d74895e137cd76d2231dfc3f3 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Fri, 23 Feb 2024 09:39:19 +0100 Subject: [PATCH 01/38] mempool management in jds --- .../src/lib/job_declarator/message_handler.rs | 26 ++++++-- roles/jd-server/src/lib/mempool/mod.rs | 61 ++++++++++--------- 2 files changed, 54 insertions(+), 33 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 38b68272d1..3e0bc46c1e 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -14,6 +14,8 @@ pub type SendTo = SendTo_, ()>; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; +use crate::mempool::{self, rpc_client::RpcApi, JDsMempool}; + use super::signed_token; use super::JobDeclaratorDownstream; @@ -81,10 +83,26 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); - if let Some(tx_data) = short_id_mempool.get(&sid_) { - txs_in_job.push(tx_data.clone()); - } else { - missing_txs.push(i as u16); + match short_id_mempool.get(&sid_) { + Some(tx_data) => { + match &tx_data.tx { + Some(tx) => txs_in_job.push(tx.clone()), + None => { + let new_tx_data = self + .mempool + .safe_lock(|x| x.get_client()) + .unwrap() + .unwrap() + .get_raw_transaction(&tx_data.id.to_string(), None); + if let Ok(new_tx) = new_tx_data { + mempool::JDsMempool::add_tx_data_to_mempool(self.mempool.clone(), tx_data.clone().id, Some(new_tx.clone())); + txs_in_job.push(new_tx); + } + } + + } + } + None => missing_txs.push(i as u16), } } self.declared_mining_job = Some(( diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index aa990f095e..8ab06457fb 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -5,23 +5,24 @@ use bitcoin::blockdata::transaction::Transaction; use hashbrown::HashMap; use roles_logic_sv2::utils::Mutex; use rpc::mini_rpc_client; -use std::{convert::TryInto, sync::Arc}; +use std::{convert::TryInto, str::FromStr, sync::Arc}; use stratum_common::{bitcoin, bitcoin::hash_types::Txid}; #[derive(Clone, Debug)] -pub struct TransacrtionWithHash { - id: Txid, - tx: Transaction, +pub struct TransactionWithHash { + pub id: Txid, + pub tx: Option, } #[derive(Clone, Debug)] pub struct JDsMempool { - pub mempool: Vec, + pub mempool: HashMap>, auth: mini_rpc_client::Auth, url: String, new_block_receiver: Receiver, } + impl JDsMempool { pub fn get_client(&self) -> Option { let url = self.url.as_str(); @@ -48,7 +49,7 @@ impl JDsMempool { new_block_receiver: Receiver, ) -> Self { let auth = mini_rpc_client::Auth::new(username, password); - let empty_mempool: Vec = Vec::new(); + let empty_mempool: HashMap> = HashMap::new(); JDsMempool { mempool: empty_mempool, auth, @@ -57,14 +58,30 @@ impl JDsMempool { } } - pub async fn update_mempool(self_: Arc>) -> Result<(), JdsMempoolError> { - let mut mempool_ordered: Vec = Vec::new(); + pub async fn get_updated_mempool(self_: Arc>) -> Result>, JdsMempoolError> { + let mut mempool_ordered: HashMap> = HashMap::new(); let client = self_ .safe_lock(|x| x.get_client()) .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? .ok_or(JdsMempoolError::NoClient)?; - let new_mempool: Result, JdsMempoolError> = + let new_mempool: Result>, JdsMempoolError> = tokio::task::spawn(async move { + let mempool: Result, BitcoincoreRpcError> = + client.get_raw_mempool_verbose(); + let mempool = mempool.map_err(JdsMempoolError::BitcoinCoreRpcError)?; + for id in mempool { + let key_id = Txid::from_str(&id).unwrap(); + let tx = self_.safe_lock(|x| { + match x.mempool.get(&key_id) { + Some(entry) => { + //println!("KEY FOUND --> {:?}", entry); + entry.clone() + } + None => None + } + }); + let id = Txid::from_str(&id).unwrap(); + mempool_ordered.insert(id, tx.unwrap()); let mempool: Vec = client .get_raw_mempool_verbose() .await @@ -96,32 +113,18 @@ impl JDsMempool { } } - pub async fn on_submit(self_: Arc>) -> Result<(), JdsMempoolError> { - let new_block_receiver: Receiver = self_ - .safe_lock(|x| x.new_block_receiver.clone()) - .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))?; - let client = self_ - .safe_lock(|x| x.get_client()) - .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? - .ok_or(JdsMempoolError::NoClient)?; - - while let Ok(block_hex) = new_block_receiver.recv().await { - match mini_rpc_client::MiniRpcClient::submit_block(&client, block_hex).await { - Ok(_) => return Ok(()), - Err(e) => JdsMempoolError::Rpc(e), - }; - } - Ok(()) - } - pub fn to_short_ids(&self, nonce: u64) -> Option> { let mut ret = HashMap::new(); for tx in &self.mempool { - let s_id = roles_logic_sv2::utils::get_short_hash(tx.id, nonce) + let s_id = roles_logic_sv2::utils::get_short_hash(*tx.0, nonce) .to_vec() .try_into() .unwrap(); - if ret.insert(s_id, tx.tx.clone()).is_none() { + let tx_data = TransactionWithHash { + id: *tx.0, + tx: tx.1.clone(), + }; + if ret.insert(s_id, tx_data.clone()).is_none() { continue; } else { return None; From 1b664523cc69b119a231c924129675e80009563b Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Fri, 23 Feb 2024 11:26:34 +0100 Subject: [PATCH 02/38] cleanup + fmt --- .../src/lib/job_declarator/message_handler.rs | 29 ++++++++++--------- roles/jd-server/src/lib/mempool/mod.rs | 7 +++-- 2 files changed, 19 insertions(+), 17 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 3e0bc46c1e..4d951be0c7 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -14,7 +14,7 @@ pub type SendTo = SendTo_, ()>; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; -use crate::mempool::{self, rpc_client::RpcApi, JDsMempool}; +use crate::mempool::{self, rpc_client::RpcApi}; use super::signed_token; @@ -84,25 +84,26 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); match short_id_mempool.get(&sid_) { - Some(tx_data) => { - match &tx_data.tx { - Some(tx) => txs_in_job.push(tx.clone()), - None => { - let new_tx_data = self + Some(tx_data) => match &tx_data.tx { + Some(tx) => txs_in_job.push(tx.clone()), + None => { + let new_tx_data = self .mempool - .safe_lock(|x| x.get_client()) + .safe_lock(|x| x.get_client()) .unwrap() .unwrap() .get_raw_transaction(&tx_data.id.to_string(), None); - if let Ok(new_tx) = new_tx_data { - mempool::JDsMempool::add_tx_data_to_mempool(self.mempool.clone(), tx_data.clone().id, Some(new_tx.clone())); - txs_in_job.push(new_tx); - } + if let Ok(new_tx) = new_tx_data { + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + tx_data.clone().id, + Some(new_tx.clone()), + ); + txs_in_job.push(new_tx); } - } - } - None => missing_txs.push(i as u16), + }, + None => missing_txs.push(i as u16), } } self.declared_mining_job = Some(( diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 8ab06457fb..61da874668 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -22,7 +22,6 @@ pub struct JDsMempool { new_block_receiver: Receiver, } - impl JDsMempool { pub fn get_client(&self) -> Option { let url = self.url.as_str(); @@ -58,7 +57,9 @@ impl JDsMempool { } } - pub async fn get_updated_mempool(self_: Arc>) -> Result>, JdsMempoolError> { + pub async fn get_updated_mempool( + self_: Arc>, + ) -> Result>, JdsMempoolError> { let mut mempool_ordered: HashMap> = HashMap::new(); let client = self_ .safe_lock(|x| x.get_client()) @@ -77,7 +78,7 @@ impl JDsMempool { //println!("KEY FOUND --> {:?}", entry); entry.clone() } - None => None + None => None, } }); let id = Txid::from_str(&id).unwrap(); From e736e6a9928a7d87f3def62277a3fbbdb73b6281 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Fri, 23 Feb 2024 12:12:03 +0100 Subject: [PATCH 03/38] mempool txdata insertion after ProvideMissingTx.Success --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 4d951be0c7..07d11a0b68 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -172,7 +172,12 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { ), ), )))? as usize; - transactions.insert(index, tx); + transactions.insert(index, tx.clone()); + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + tx.txid(), + Some(tx), + ); } // TODO check it let tx_hash_list_hash = self.tx_hash_list_hash.clone().unwrap().into_static(); From 42101ae17ff972347cbc9aa8e8edcde4919ee4d8 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Fri, 23 Feb 2024 16:06:46 +0100 Subject: [PATCH 04/38] println! removal --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 3 ++- roles/jd-server/src/lib/mempool/mod.rs | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 07d11a0b68..fa1d6dc5f7 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -13,6 +13,7 @@ use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; +use tracing::{info, warn}; use crate::mempool::{self, rpc_client::RpcApi}; @@ -55,7 +56,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { coinbase_output: self.coinbase_output.clone().try_into().unwrap(), }; let message_enum = JobDeclaration::AllocateMiningJobTokenSuccess(message_success); - println!( + info!( "Sending AllocateMiningJobTokenSuccess to proxy {:?}", message_enum ); diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 61da874668..68a33d7dcd 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -75,7 +75,6 @@ impl JDsMempool { let tx = self_.safe_lock(|x| { match x.mempool.get(&key_id) { Some(entry) => { - //println!("KEY FOUND --> {:?}", entry); entry.clone() } None => None, From 650735997b9a5c153a6a96e5b8137a61d558c613 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Fri, 23 Feb 2024 19:43:22 +0100 Subject: [PATCH 05/38] fmt fix --- roles/jd-server/src/lib/mempool/mod.rs | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 68a33d7dcd..82d2fb0cb8 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -72,13 +72,9 @@ impl JDsMempool { let mempool = mempool.map_err(JdsMempoolError::BitcoinCoreRpcError)?; for id in mempool { let key_id = Txid::from_str(&id).unwrap(); - let tx = self_.safe_lock(|x| { - match x.mempool.get(&key_id) { - Some(entry) => { - entry.clone() - } - None => None, - } + let tx = self_.safe_lock(|x| match x.mempool.get(&key_id) { + Some(entry) => entry.clone(), + None => None, }); let id = Txid::from_str(&id).unwrap(); mempool_ordered.insert(id, tx.unwrap()); From e911536f16745e2a4aa96552ec43e23cbadcceec Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Mon, 26 Feb 2024 14:20:15 +0100 Subject: [PATCH 06/38] last updates after reivews --- .../src/lib/job_declarator/message_handler.rs | 19 ++++++++++++------- roles/jd-server/src/lib/mempool/mod.rs | 11 +++++------ 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index fa1d6dc5f7..47dd9e5538 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -94,13 +94,18 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { .unwrap() .unwrap() .get_raw_transaction(&tx_data.id.to_string(), None); - if let Ok(new_tx) = new_tx_data { - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - tx_data.clone().id, - Some(new_tx.clone()), - ); - txs_in_job.push(new_tx); + match new_tx_data { + Ok(new_tx) => { + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + tx_data.clone().id, + Some(new_tx.clone()), + ); + txs_in_job.push(new_tx); + } + Err(_) => { + missing_txs.push(i as u16); + } } } }, diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 82d2fb0cb8..f9dc9a520c 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -57,22 +57,21 @@ impl JDsMempool { } } - pub async fn get_updated_mempool( - self_: Arc>, - ) -> Result>, JdsMempoolError> { + pub async fn update_mempool(self_: Arc>) -> Result<(), JdsMempoolError> { let mut mempool_ordered: HashMap> = HashMap::new(); let client = self_ .safe_lock(|x| x.get_client()) .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? .ok_or(JdsMempoolError::NoClient)?; + let self_clone = self_.clone(); let new_mempool: Result>, JdsMempoolError> = tokio::task::spawn(async move { let mempool: Result, BitcoincoreRpcError> = client.get_raw_mempool_verbose(); let mempool = mempool.map_err(JdsMempoolError::BitcoinCoreRpcError)?; - for id in mempool { - let key_id = Txid::from_str(&id).unwrap(); - let tx = self_.safe_lock(|x| match x.mempool.get(&key_id) { + for id in &mempool { + let key_id = Txid::from_str(id).unwrap(); + let tx = self_clone.safe_lock(|x| match x.mempool.get(&key_id) { Some(entry) => entry.clone(), None => None, }); From 4ddc9b6a42353955437c6522bc05beb2f9badcba Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Mon, 26 Feb 2024 14:26:51 +0100 Subject: [PATCH 07/38] clippy fix --- roles/jd-server/src/lib/mempool/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index f9dc9a520c..b75f9c7f36 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -75,7 +75,7 @@ impl JDsMempool { Some(entry) => entry.clone(), None => None, }); - let id = Txid::from_str(&id).unwrap(); + let id = Txid::from_str(id).unwrap(); mempool_ordered.insert(id, tx.unwrap()); let mempool: Vec = client .get_raw_mempool_verbose() From b2a808b5f743fac3153e7b3f61bb32c8016cc4d2 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Mon, 26 Feb 2024 20:25:05 +0100 Subject: [PATCH 08/38] self_ cloned as suggested in review --- roles/jd-server/src/lib/mempool/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index b75f9c7f36..1a8b2732e9 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -63,15 +63,15 @@ impl JDsMempool { .safe_lock(|x| x.get_client()) .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? .ok_or(JdsMempoolError::NoClient)?; - let self_clone = self_.clone(); - let new_mempool: Result>, JdsMempoolError> = + let new_mempool: Result>, JdsMempoolError> = { + let self_ = self_.clone(); tokio::task::spawn(async move { let mempool: Result, BitcoincoreRpcError> = client.get_raw_mempool_verbose(); let mempool = mempool.map_err(JdsMempoolError::BitcoinCoreRpcError)?; for id in &mempool { let key_id = Txid::from_str(id).unwrap(); - let tx = self_clone.safe_lock(|x| match x.mempool.get(&key_id) { + let tx = self_.safe_lock(|x| match x.mempool.get(&key_id) { Some(entry) => entry.clone(), None => None, }); From 90d20badd537eaf53296c47bdd2f8f13aa59bca9 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 12:23:46 +0100 Subject: [PATCH 09/38] fix unclosed bracket --- roles/jd-server/src/lib/mempool/mod.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 1a8b2732e9..3aa92c3748 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -38,7 +38,7 @@ impl JDsMempool { #[cfg(debug_assertions)] pub fn _get_transaction_list(self_: Arc>) -> Vec { let tx_list = self_.safe_lock(|x| x.mempool.clone()).unwrap(); - let tx_list_: Vec = tx_list.iter().map(|n| n.id).collect(); + let tx_list_: Vec = tx_list.iter().map(|n| *n.0).collect(); tx_list_ } pub fn new( @@ -95,8 +95,8 @@ impl JDsMempool { } }) .await - .map_err(JdsMempoolError::TokioJoin)?; - + .map_err(JdsMempoolError::TokioJoin)? + }; match new_mempool { Ok(new_mempool_) => { let _ = self_.safe_lock(|x| { From b0788a68efb8152563fbccec36fbe46699cd25d9 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 19:40:20 +0100 Subject: [PATCH 10/38] add_tx_data_to_job_and_mempool to asynchronously update job and jds mempool addition --- .../src/lib/job_declarator/message_handler.rs | 63 +++++++++++++------ 1 file changed, 43 insertions(+), 20 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 47dd9e5538..174b27cf4e 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -80,33 +80,21 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { .unwrap() .unwrap(); let mut txs_in_job = vec![]; + let mut txs_to_retrieve: Vec<(String, usize)> = vec![]; let mut missing_txs = vec![]; for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); match short_id_mempool.get(&sid_) { Some(tx_data) => match &tx_data.tx { - Some(tx) => txs_in_job.push(tx.clone()), - None => { - let new_tx_data = self - .mempool - .safe_lock(|x| x.get_client()) - .unwrap() - .unwrap() - .get_raw_transaction(&tx_data.id.to_string(), None); - match new_tx_data { - Ok(new_tx) => { - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - tx_data.clone().id, - Some(new_tx.clone()), - ); - txs_in_job.push(new_tx); - } - Err(_) => { - missing_txs.push(i as u16); - } + Some(tx) => { + if i >= txs_in_job.len() { + txs_in_job.resize(i + 1, tx.clone()); } + txs_in_job.insert(i, tx.clone()) + }, + None => { + txs_to_retrieve.push(((tx_data.id.to_string()), i)); } }, None => missing_txs.push(i as u16), @@ -118,6 +106,10 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { missing_txs.clone(), )); + if !txs_to_retrieve.is_empty() { + add_tx_data_to_job_and_mempool(txs_to_retrieve, self); + } + if missing_txs.is_empty() { let message_success = DeclareMiningJobSuccess { request_id: message.request_id, @@ -212,3 +204,34 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { Ok(SendTo::None(Some(m))) } } + +fn add_tx_data_to_job_and_mempool( + tx_id_list: Vec<(String, usize)>, + jdd: &mut JobDeclaratorDownstream, +) -> () { + let mempool = jdd.mempool.clone(); + let mut declared_mining_job = jdd.declared_mining_job.clone(); + tokio::task::spawn(async move { + for tx in tx_id_list.iter().enumerate() { + let index = tx.1.1; + let new_tx_data: Result = + mempool + .safe_lock(|x| x.get_client()) + .unwrap() + .unwrap() + .get_raw_transaction(&tx.1.0, None) + .await + .map_err(JdsMempoolError::Rpc); + if let Ok(tx) = new_tx_data { + if let Some((_, transactions, _)) = &mut declared_mining_job { + transactions.insert(index, tx.clone()); + } + mempool::JDsMempool::add_tx_data_to_mempool( + mempool.clone(), + tx.clone().txid(), + Some(tx.clone()), + ); + } + } + }); +} \ No newline at end of file From affb60b205912bc95f1bc49364d99e2f27df7022 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 19:41:43 +0100 Subject: [PATCH 11/38] fmt fix --- .../src/lib/job_declarator/message_handler.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 174b27cf4e..568c41961c 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -11,6 +11,8 @@ use roles_logic_sv2::{ use std::{convert::TryInto, io::Cursor}; use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; +use super::signed_token; +use crate::mempool::{self, error::JdsMempoolError}; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::{info, warn}; @@ -92,7 +94,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { txs_in_job.resize(i + 1, tx.clone()); } txs_in_job.insert(i, tx.clone()) - }, + } None => { txs_to_retrieve.push(((tx_data.id.to_string()), i)); } @@ -206,22 +208,21 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } fn add_tx_data_to_job_and_mempool( - tx_id_list: Vec<(String, usize)>, + tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream, ) -> () { let mempool = jdd.mempool.clone(); let mut declared_mining_job = jdd.declared_mining_job.clone(); tokio::task::spawn(async move { for tx in tx_id_list.iter().enumerate() { - let index = tx.1.1; - let new_tx_data: Result = - mempool - .safe_lock(|x| x.get_client()) - .unwrap() - .unwrap() - .get_raw_transaction(&tx.1.0, None) - .await - .map_err(JdsMempoolError::Rpc); + let index = tx.1 .1; + let new_tx_data: Result = mempool + .safe_lock(|x| x.get_client()) + .unwrap() + .unwrap() + .get_raw_transaction(&tx.1 .0, None) + .await + .map_err(JdsMempoolError::Rpc); if let Ok(tx) = new_tx_data { if let Some((_, transactions, _)) = &mut declared_mining_job { transactions.insert(index, tx.clone()); @@ -232,6 +233,6 @@ fn add_tx_data_to_job_and_mempool( Some(tx.clone()), ); } - } + } }); -} \ No newline at end of file +} From f42cac0e9236564448d2b9fe294ff26e18d01b15 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 19:43:01 +0100 Subject: [PATCH 12/38] cleanup --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 568c41961c..e61fe1ea67 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -210,7 +210,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { fn add_tx_data_to_job_and_mempool( tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream, -) -> () { +) { let mempool = jdd.mempool.clone(); let mut declared_mining_job = jdd.declared_mining_job.clone(); tokio::task::spawn(async move { From c8788728b5ce9fcb2fd8d9651ded6cabe5020ffd Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 20:04:57 +0100 Subject: [PATCH 13/38] check before inserting new tx in declared_mining_job --- .../src/lib/job_declarator/message_handler.rs | 41 +++++++++++-------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index e61fe1ea67..9bc1987387 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -11,8 +11,6 @@ use roles_logic_sv2::{ use std::{convert::TryInto, io::Cursor}; use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; -use super::signed_token; -use crate::mempool::{self, error::JdsMempoolError}; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::{info, warn}; @@ -90,26 +88,30 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { match short_id_mempool.get(&sid_) { Some(tx_data) => match &tx_data.tx { Some(tx) => { + //info!("txs_in_job --> {:?}\nindex --> {:?}", txs_in_job.len(), i); if i >= txs_in_job.len() { txs_in_job.resize(i + 1, tx.clone()); } txs_in_job.insert(i, tx.clone()) - } + }, None => { + //info!("txs to retrieve --> {:?}", txs_to_retrieve); txs_to_retrieve.push(((tx_data.id.to_string()), i)); } }, None => missing_txs.push(i as u16), } } + info!("txs_in_job --> {:?}", txs_in_job.len()); self.declared_mining_job = Some(( message.clone().into_static(), txs_in_job, missing_txs.clone(), )); + //info!("DeclareMiningJob --> {:?}", self.declared_mining_job); if !txs_to_retrieve.is_empty() { - add_tx_data_to_job_and_mempool(txs_to_retrieve, self); + add_tx_data_to_job(txs_to_retrieve, self); } if missing_txs.is_empty() { @@ -207,24 +209,29 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } } -fn add_tx_data_to_job_and_mempool( - tx_id_list: Vec<(String, usize)>, +fn add_tx_data_to_job( + tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream, -) { +) -> () { let mempool = jdd.mempool.clone(); let mut declared_mining_job = jdd.declared_mining_job.clone(); tokio::task::spawn(async move { for tx in tx_id_list.iter().enumerate() { - let index = tx.1 .1; - let new_tx_data: Result = mempool - .safe_lock(|x| x.get_client()) - .unwrap() - .unwrap() - .get_raw_transaction(&tx.1 .0, None) - .await - .map_err(JdsMempoolError::Rpc); + let index = tx.1.1; + let new_tx_data: Result = + mempool + .safe_lock(|x| x.get_client()) + .unwrap() + .unwrap() + .get_raw_transaction(&tx.1.0, None) + .await + .map_err(JdsMempoolError::Rpc); if let Ok(tx) = new_tx_data { if let Some((_, transactions, _)) = &mut declared_mining_job { + info!("TX retrieved -> {:?}", index); + if index >= transactions.len() { + transactions.resize(index + 1, tx.clone()); + } transactions.insert(index, tx.clone()); } mempool::JDsMempool::add_tx_data_to_mempool( @@ -233,6 +240,6 @@ fn add_tx_data_to_job_and_mempool( Some(tx.clone()), ); } - } + } }); -} +} \ No newline at end of file From ce784210f22763ad60b2830458f9d9d43fce1b45 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 20:06:30 +0100 Subject: [PATCH 14/38] info! removal --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 9bc1987387..44a7cfd6ef 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -88,27 +88,23 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { match short_id_mempool.get(&sid_) { Some(tx_data) => match &tx_data.tx { Some(tx) => { - //info!("txs_in_job --> {:?}\nindex --> {:?}", txs_in_job.len(), i); if i >= txs_in_job.len() { txs_in_job.resize(i + 1, tx.clone()); } txs_in_job.insert(i, tx.clone()) }, None => { - //info!("txs to retrieve --> {:?}", txs_to_retrieve); txs_to_retrieve.push(((tx_data.id.to_string()), i)); } }, None => missing_txs.push(i as u16), } } - info!("txs_in_job --> {:?}", txs_in_job.len()); self.declared_mining_job = Some(( message.clone().into_static(), txs_in_job, missing_txs.clone(), )); - //info!("DeclareMiningJob --> {:?}", self.declared_mining_job); if !txs_to_retrieve.is_empty() { add_tx_data_to_job(txs_to_retrieve, self); @@ -228,7 +224,6 @@ fn add_tx_data_to_job( .map_err(JdsMempoolError::Rpc); if let Ok(tx) = new_tx_data { if let Some((_, transactions, _)) = &mut declared_mining_job { - info!("TX retrieved -> {:?}", index); if index >= transactions.len() { transactions.resize(index + 1, tx.clone()); } From d88f59bf87b5d45dd725aec4b2f15dfb73f3a4a7 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 20:13:42 +0100 Subject: [PATCH 15/38] check before adding provided_missing_txs in declared_mining_job txs list --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 44a7cfd6ef..c12b3f24e1 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -170,6 +170,9 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { ), ), )))? as usize; + if index >= transactions.len() { + transactions.resize(index + 1, tx.clone()); + } transactions.insert(index, tx.clone()); mempool::JDsMempool::add_tx_data_to_mempool( self.mempool.clone(), From ea5c94f44afe16768c173e695f92d9812ec3085c Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 20:14:02 +0100 Subject: [PATCH 16/38] fmt --- .../src/lib/job_declarator/message_handler.rs | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index c12b3f24e1..f63a2b2359 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -11,6 +11,8 @@ use roles_logic_sv2::{ use std::{convert::TryInto, io::Cursor}; use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; +use super::signed_token; +use crate::mempool::{self, error::JdsMempoolError}; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::{info, warn}; @@ -92,7 +94,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { txs_in_job.resize(i + 1, tx.clone()); } txs_in_job.insert(i, tx.clone()) - }, + } None => { txs_to_retrieve.push(((tx_data.id.to_string()), i)); } @@ -208,23 +210,19 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } } -fn add_tx_data_to_job( - tx_id_list: Vec<(String, usize)>, - jdd: &mut JobDeclaratorDownstream, -) -> () { +fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream) -> () { let mempool = jdd.mempool.clone(); let mut declared_mining_job = jdd.declared_mining_job.clone(); tokio::task::spawn(async move { for tx in tx_id_list.iter().enumerate() { - let index = tx.1.1; - let new_tx_data: Result = - mempool - .safe_lock(|x| x.get_client()) - .unwrap() - .unwrap() - .get_raw_transaction(&tx.1.0, None) - .await - .map_err(JdsMempoolError::Rpc); + let index = tx.1 .1; + let new_tx_data: Result = mempool + .safe_lock(|x| x.get_client()) + .unwrap() + .unwrap() + .get_raw_transaction(&tx.1 .0, None) + .await + .map_err(JdsMempoolError::Rpc); if let Ok(tx) = new_tx_data { if let Some((_, transactions, _)) = &mut declared_mining_job { if index >= transactions.len() { @@ -238,6 +236,6 @@ fn add_tx_data_to_job( Some(tx.clone()), ); } - } + } }); -} \ No newline at end of file +} From 7d03248bbdbee7cb3ef0c7579ff3dcc17523077a Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Tue, 5 Mar 2024 20:15:59 +0100 Subject: [PATCH 17/38] clippy fix --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index f63a2b2359..cc8349a0cf 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -210,7 +210,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } } -fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream) -> () { +fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream) { let mempool = jdd.mempool.clone(); let mut declared_mining_job = jdd.declared_mining_job.clone(); tokio::task::spawn(async move { From f490fe53adfa4525570a1997e99c05c2e3c5e87c Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Wed, 6 Mar 2024 17:35:24 +0100 Subject: [PATCH 18/38] rebased on main --- .../src/lib/job_declarator/message_handler.rs | 6 +-- roles/jd-server/src/lib/mempool/mod.rs | 48 +++++++++++++------ 2 files changed, 35 insertions(+), 19 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index cc8349a0cf..06ae18cb46 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -15,11 +15,7 @@ use super::signed_token; use crate::mempool::{self, error::JdsMempoolError}; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; -use tracing::{info, warn}; - -use crate::mempool::{self, rpc_client::RpcApi}; - -use super::signed_token; +use tracing::info; use super::JobDeclaratorDownstream; diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 3aa92c3748..a0ab1d3e91 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -41,6 +41,7 @@ impl JDsMempool { let tx_list_: Vec = tx_list.iter().map(|n| *n.0).collect(); tx_list_ } + pub fn new( url: String, username: String, @@ -57,6 +58,16 @@ impl JDsMempool { } } + pub fn add_tx_data_to_mempool( + self_: Arc>, + txid: Txid, + transaction: Option, + ) { + let _ = self_.safe_lock(|x| { + x.mempool.insert(txid, transaction); + }); + } + pub async fn update_mempool(self_: Arc>) -> Result<(), JdsMempoolError> { let mut mempool_ordered: HashMap> = HashMap::new(); let client = self_ @@ -66,9 +77,10 @@ impl JDsMempool { let new_mempool: Result>, JdsMempoolError> = { let self_ = self_.clone(); tokio::task::spawn(async move { - let mempool: Result, BitcoincoreRpcError> = - client.get_raw_mempool_verbose(); - let mempool = mempool.map_err(JdsMempoolError::BitcoinCoreRpcError)?; + let mempool: Vec = client + .get_raw_mempool_verbose() + .await + .map_err(JdsMempoolError::Rpc)?; for id in &mempool { let key_id = Txid::from_str(id).unwrap(); let tx = self_.safe_lock(|x| match x.mempool.get(&key_id) { @@ -77,16 +89,6 @@ impl JDsMempool { }); let id = Txid::from_str(id).unwrap(); mempool_ordered.insert(id, tx.unwrap()); - let mempool: Vec = client - .get_raw_mempool_verbose() - .await - .map_err(JdsMempoolError::Rpc)?; - for id in &mempool { - let tx: Result = client.get_raw_transaction(id, None).await; - if let Ok(tx) = tx { - let id = tx.txid(); - mempool_ordered.push(TransacrtionWithHash { id, tx }); - } } if mempool_ordered.is_empty() { Err(JdsMempoolError::EmptyMempool) @@ -108,7 +110,25 @@ impl JDsMempool { } } - pub fn to_short_ids(&self, nonce: u64) -> Option> { + pub async fn on_submit(self_: Arc>) -> Result<(), JdsMempoolError> { + let new_block_receiver: Receiver = self_ + .safe_lock(|x| x.new_block_receiver.clone()) + .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))?; + let client = self_ + .safe_lock(|x| x.get_client()) + .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? + .ok_or(JdsMempoolError::NoClient)?; + + while let Ok(block_hex) = new_block_receiver.recv().await { + match mini_rpc_client::MiniRpcClient::submit_block(&client, block_hex).await { + Ok(_) => return Ok(()), + Err(e) => JdsMempoolError::Rpc(e), + }; + } + Ok(()) + } + + pub fn to_short_ids(&self, nonce: u64) -> Option> { let mut ret = HashMap::new(); for tx in &self.mempool { let s_id = roles_logic_sv2::utils::get_short_hash(*tx.0, nonce) From c11b5a08fbe68dfe647104c293218f2f1fffa545 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Wed, 6 Mar 2024 19:29:47 +0100 Subject: [PATCH 19/38] error propagation --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 06ae18cb46..a23a933df3 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -214,8 +214,8 @@ fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorD let index = tx.1 .1; let new_tx_data: Result = mempool .safe_lock(|x| x.get_client()) - .unwrap() - .unwrap() + .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? + .ok_or(JdsMempoolError::NoClient)? .get_raw_transaction(&tx.1 .0, None) .await .map_err(JdsMempoolError::Rpc); @@ -233,5 +233,6 @@ fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorD ); } } + Ok::<(), JdsMempoolError>(()) }); } From 4244cb662a1d65ff24204922eac0bdfa2ebe8cf2 Mon Sep 17 00:00:00 2001 From: lorban Date: Wed, 6 Mar 2024 23:00:28 +0100 Subject: [PATCH 20/38] Renamed rpc method the method get_raw_mempool_verbose perform the rpc call `getrawmempool`, whichout verbose. This commit remove this inconsistency --- roles/jd-server/src/lib/mempool/mod.rs | 2 +- utils/rpc/src/mini_rpc_client.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index a0ab1d3e91..69fca884fe 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -78,7 +78,7 @@ impl JDsMempool { let self_ = self_.clone(); tokio::task::spawn(async move { let mempool: Vec = client - .get_raw_mempool_verbose() + .get_raw_mempool() .await .map_err(JdsMempoolError::Rpc)?; for id in &mempool { diff --git a/utils/rpc/src/mini_rpc_client.rs b/utils/rpc/src/mini_rpc_client.rs index 12769ac19f..d0ce084351 100644 --- a/utils/rpc/src/mini_rpc_client.rs +++ b/utils/rpc/src/mini_rpc_client.rs @@ -59,7 +59,7 @@ impl MiniRpcClient { } } - pub async fn get_raw_mempool_verbose(&self) -> Result, RpcError> { + pub async fn get_raw_mempool(&self) -> Result, RpcError> { let response = self.send_json_rpc_request("getrawmempool", json!([])).await; match response { Ok(result_hex) => { From 9f635058f9057342766ed07c5f7fb83a38a43e15 Mon Sep 17 00:00:00 2001 From: lorban Date: Sat, 9 Mar 2024 12:41:41 +0100 Subject: [PATCH 21/38] JobDeclaratordownstream modified Now the DeclaratorDownstream has a method that retrieves the the recognized transactions in the mempool. This method is called inside the JobdeclaratorDownstream::accept_incomin_connections(). --- protocols/v2/roles-logic-sv2/src/errors.rs | 2 + roles/jd-server/src/lib/error.rs | 2 + .../src/lib/job_declarator/message_handler.rs | 172 +++++++++++------- roles/jd-server/src/lib/job_declarator/mod.rs | 112 ++++++++++-- roles/jd-server/src/lib/status.rs | 4 + 5 files changed, 208 insertions(+), 84 deletions(-) diff --git a/protocols/v2/roles-logic-sv2/src/errors.rs b/protocols/v2/roles-logic-sv2/src/errors.rs index 1d8625124b..198b94e011 100644 --- a/protocols/v2/roles-logic-sv2/src/errors.rs +++ b/protocols/v2/roles-logic-sv2/src/errors.rs @@ -59,6 +59,7 @@ pub enum Error { TargetError(InputError), HashrateError(InputError), LogicErrorMessage(std::boxed::Box>), + JDSMissingTransactions, } impl From for Error { @@ -149,6 +150,7 @@ impl Display for Error { TargetError(e) => write!(f, "Impossible to get Target: {:?}", e), HashrateError(e) => write!(f, "Impossible to get Hashrate: {:?}", e), LogicErrorMessage(e) => write!(f, "Message is well formatted but can not be handled: {:?}", e), + JDSMissingTransactions => write!(f, "JD server cannot propagate the block: missing transactions"), } } } diff --git a/roles/jd-server/src/lib/error.rs b/roles/jd-server/src/lib/error.rs index e94a2395cb..235c8f19d7 100644 --- a/roles/jd-server/src/lib/error.rs +++ b/roles/jd-server/src/lib/error.rs @@ -22,6 +22,7 @@ pub enum JdsError { Custom(String), Sv2ProtocolError((u32, Mining<'static>)), MempoolError(JdsMempoolError), + ImpossibleToReconstructBlock(String), } impl std::fmt::Display for JdsError { @@ -42,6 +43,7 @@ impl std::fmt::Display for JdsError { write!(f, "Received Sv2 Protocol Error from upstream: `{:?}`", e) } MempoolError(ref e) => write!(f, "Mempool error: `{:?}`", e), + ImpossibleToReconstructBlock(e) => write!(f, "Error in reconstructing the block: {:?}", e), } } } diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index a23a933df3..6e9caf32f9 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -6,13 +6,13 @@ use roles_logic_sv2::{ DeclareMiningJobError, DeclareMiningJobSuccess, IdentifyTransactionsSuccess, ProvideMissingTransactions, ProvideMissingTransactionsSuccess, SubmitSolutionJd, }, - parsers::JobDeclaration, + parsers::JobDeclaration, utils::Mutex, }; -use std::{convert::TryInto, io::Cursor}; -use stratum_common::bitcoin::Transaction; +use std::{convert::TryInto, io::Cursor, sync::Arc}; +use stratum_common::bitcoin::{Transaction, Txid}; pub type SendTo = SendTo_, ()>; -use super::signed_token; -use crate::mempool::{self, error::JdsMempoolError}; +use super::{signed_token, TransactionState}; +use crate::mempool::{self, error::JdsMempoolError, JDsMempool}; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::info; @@ -77,36 +77,35 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { .safe_lock(|x| x.to_short_ids(nonce)) .unwrap() .unwrap(); - let mut txs_in_job = vec![]; + let mut transactions_with_state = vec![TransactionState::Missing; short_hash_list.len()]; let mut txs_to_retrieve: Vec<(String, usize)> = vec![]; - let mut missing_txs = vec![]; + let mut missing_txs: Vec = Vec::new(); for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); match short_id_mempool.get(&sid_) { Some(tx_data) => match &tx_data.tx { Some(tx) => { - if i >= txs_in_job.len() { - txs_in_job.resize(i + 1, tx.clone()); - } - txs_in_job.insert(i, tx.clone()) + transactions_with_state[i] = TransactionState::Present(tx.clone()); } None => { - txs_to_retrieve.push(((tx_data.id.to_string()), i)); + transactions_with_state[i] = TransactionState::ToBeRetrievedFromMempool(tx_data.id); } }, - None => missing_txs.push(i as u16), + None => { + transactions_with_state[i] = TransactionState::Missing; + // TODO remove this, the the ids of missing transactions from the vector + missing_txs.push(i as u16); + }, } } self.declared_mining_job = Some(( message.clone().into_static(), - txs_in_job, - missing_txs.clone(), + transactions_with_state )); - if !txs_to_retrieve.is_empty() { - add_tx_data_to_job(txs_to_retrieve, self); - } + //let self_mutex = Arc::new(Mutex::new(self)); + //add_tx_data_to_job(self_mutex); if missing_txs.is_empty() { let message_success = DeclareMiningJobSuccess { @@ -122,6 +121,8 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } else { let message_provide_missing_transactions = ProvideMissingTransactions { request_id: message.request_id, + // TODO here get the missing IDS from the entries of txs_in_job which are + // TransactionState::Missing unknown_tx_position_list: missing_txs.into(), }; let message_enum_provide_missing_transactions = @@ -153,30 +154,38 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message: ProvideMissingTransactionsSuccess, ) -> Result { match &mut self.declared_mining_job { - Some((_, ref mut transactions, missing_indexes)) => { + Some((_, ref mut transactions_with_state)) => { for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { - let mut cursor = Cursor::new(tx); - let tx = Transaction::consensus_decode_from_finite_reader(&mut cursor) - .expect("Invalid tx data from downstream"); - let index = - *missing_indexes - .get(i) - .ok_or(Error::LogicErrorMessage(Box::new( - AllMessages::JobDeclaration( - JobDeclaration::ProvideMissingTransactionsSuccess( - message.clone().into_static(), - ), - ), - )))? as usize; - if index >= transactions.len() { - transactions.resize(index + 1, tx.clone()); + for tx_with_state in transactions_with_state.clone() { + match tx_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => continue, + TransactionState::Missing => { + let mut cursor = Cursor::new(tx); + // TODO remove this unwrap + let transaction = Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); + transactions_with_state[i] = TransactionState::Present(transaction.clone()); + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + transaction.txid(), + Some(transaction), + ); + break; + } + + } + } + } + // if there still a missing transaction return an error + for tx_with_state in transactions_with_state { + match tx_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => continue, + TransactionState::Missing => { + return Err(Error::JDSMissingTransactions); + + } } - transactions.insert(index, tx.clone()); - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - tx.txid(), - Some(tx), - ); } // TODO check it let tx_hash_list_hash = self.tx_hash_list_hash.clone().unwrap().into_static(); @@ -206,33 +215,56 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } } -fn add_tx_data_to_job(tx_id_list: Vec<(String, usize)>, jdd: &mut JobDeclaratorDownstream) { - let mempool = jdd.mempool.clone(); - let mut declared_mining_job = jdd.declared_mining_job.clone(); - tokio::task::spawn(async move { - for tx in tx_id_list.iter().enumerate() { - let index = tx.1 .1; - let new_tx_data: Result = mempool - .safe_lock(|x| x.get_client()) - .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? - .ok_or(JdsMempoolError::NoClient)? - .get_raw_transaction(&tx.1 .0, None) - .await - .map_err(JdsMempoolError::Rpc); - if let Ok(tx) = new_tx_data { - if let Some((_, transactions, _)) = &mut declared_mining_job { - if index >= transactions.len() { - transactions.resize(index + 1, tx.clone()); - } - transactions.insert(index, tx.clone()); - } - mempool::JDsMempool::add_tx_data_to_mempool( - mempool.clone(), - tx.clone().txid(), - Some(tx.clone()), - ); - } - } - Ok::<(), JdsMempoolError>(()) - }); -} +//fn add_tx_data_to_job(jdd: Arc>) { +// tokio::task::spawn(async move { +// let mut tx_list: Vec = Vec::new(); +// let mut new_transactions: Vec = Vec::new(); +// let mempool = jdd.safe_lock(|a| a.mempool.clone()).unwrap(); +// jdd.safe_lock(|a| for tx in a.declared_mining_job.clone().unwrap().1 { +// match tx { +// TransactionState::Present(_) => continue, +// TransactionState::Missing => continue, +// TransactionState::ToBeRetrievedFromMempool(m) => tx_list.push(m), +// } +// }); +// for txid in tx_list { +// let new_tx_data: Result = mempool +// .safe_lock(|x| x.get_client()) +// .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? +// .ok_or(JdsMempoolError::NoClient)? +// .get_raw_transaction(&txid.to_string(), None) +// .await +// .map_err(JdsMempoolError::Rpc); +// if let Ok(transaction) = new_tx_data { +// new_transactions.push(transaction); +// //this unwrap is safe +// } else { +// // TODO propagate error +// todo!() +// }; +// }; +// +// //for tx in tx_list.iter().enumerate() { +// // match tx.1 { +// // &TransactionState::Missing | &TransactionState::Present(_) => continue, +// // &TransactionState::ToBeRetrievedFromMempool(txid) => { +// // let new_tx_data: Result = mempool +// // .safe_lock(|x| x.get_client()) +// // .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? +// // .ok_or(JdsMempoolError::NoClient)? +// // .get_raw_transaction(&txid.to_string(), None) +// // .await +// // .map_err(JdsMempoolError::Rpc); +// // if let Ok(transaction) = new_tx_data { +// // new_transactions_mutex.safe_lock(|a| a.push(transaction)); +// // //this unwrap is safe +// // } else { +// // // TODO propagate error +// // todo!() +// // }; +// // } +// // }; +// //} +// Ok::<(), JdsMempoolError>(()) +// }); +//} diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 08a9f62d93..14bfa1a3f2 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -21,9 +21,27 @@ use tracing::{error, info}; use stratum_common::bitcoin::{ consensus::{encode::serialize, Encodable}, - Block, Transaction, + Block, Transaction, Txid, }; + +// this structure wraps each transaction with the index of the position in the job declared by the +// JD-Client. the tx field can be None if the transaction is missing. In this case, the index is +// used to retrieve the transaction from the message ProvideMissingTransactionsSuccess +//#[derive(Debug)] +//struct TransactionWithIndex { +// tx: Option, +// index: u16, +//} + +#[derive(Clone, Debug)] +enum TransactionState { + Present(Transaction), + ToBeRetrievedFromMempool(Txid), + Missing, +} + + #[derive(Debug)] pub struct JobDeclaratorDownstream { sender: Sender, @@ -37,7 +55,11 @@ pub struct JobDeclaratorDownstream { public_key: Secp256k1PublicKey, private_key: Secp256k1SecretKey, mempool: Arc>, - declared_mining_job: Option<(DeclareMiningJob<'static>, Vec, Vec)>, + // Vec is the vector of missing transactions + // this should be (Option>, Vec) + // TODO call the vector with TransactionState in with the same name everywhere, also in the + // block creator in utils + declared_mining_job: Option<(DeclareMiningJob<'static>, Vec)>, tx_hash_list_hash: Option>, } @@ -70,18 +92,78 @@ impl JobDeclaratorDownstream { } } - fn get_block_hex(self_mutex: Arc>, message: SubmitSolutionJd) -> Option { + async fn get_job_transactions_from_mempool(self_mutex: Arc>) { + let mut transactions_to_be_retrieved: Vec = Vec::new(); + let mut new_transactions: Vec = Vec::new(); + self_mutex.clone().safe_lock(|a| for tx_with_state in a.declared_mining_job.clone().unwrap().1 { + match tx_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(tx) => transactions_to_be_retrieved.push(tx), + TransactionState::Missing => continue, + } + }); + let mempool_ = self_mutex.safe_lock(|a| a.mempool.clone()).unwrap(); + let client = mempool_.safe_lock(|a| a.get_client()).unwrap().unwrap(); + for txid in transactions_to_be_retrieved { + let transaction = client.get_raw_transaction(&txid.to_string(), None).await.unwrap(); + new_transactions.push(transaction); +// let new_tx_data: Result = mempool +// .safe_lock(|x| x.get_client()) +// .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? +// .ok_or(JdsMempoolError::NoClient)? +// .get_raw_transaction(&txid.to_string(), None) +// .await +// .map_err(JdsMempoolError::Rpc); +// if let Ok(transaction) = new_tx_data { +// new_transactions.push(transaction); +// //this unwrap is safe +// } else { +// // TODO propagate error +// todo!() +// }; + + }; + for transaction in new_transactions { + self_mutex.clone().safe_lock(|a| for transaction_with_state in &mut a.declared_mining_job.as_mut().unwrap().1 { + match transaction_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => { + *transaction_with_state = TransactionState::Present(transaction); + break; + }, + TransactionState::Missing => continue, + } + }).unwrap() + }; + //self_mutex.clone().safe_lock(|a| for transaction_with_state in a.declared_mining_job.unwrap().1 { + // match transaction_with_state { + // TransactionState::Present(_) => continue, + // TransactionState::ToBeRetrievedFromMempool(_) => {transaction_with_state = TransactionState::Present(new_transactions.clone()[1])}, + // TransactionState::Missing => continue, + // } + //}); + + todo!() + + } + + fn get_block_hex(self_mutex: Arc>, message: SubmitSolutionJd) -> Result { //TODO: implement logic for success or error - let (last_declare, tx_list, _) = match self_mutex + let (last_declare, transactions_with_state) = self_mutex .safe_lock(|x| x.declared_mining_job.take()) - .unwrap() - { - Some((last_declare, tx_list, _x)) => (last_declare, tx_list, _x), - None => return None, - }; + // TODO manage these errors + .unwrap().unwrap(); + let mut transactions_list: Vec = Vec::new(); + for tx_with_state in transactions_with_state.iter().enumerate() { + if let TransactionState::Present(tx) = tx_with_state.1 { + transactions_list.push(tx.clone()); + } else { + return Err(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string())); + }; + } let block: Block = - roles_logic_sv2::utils::BlockCreator::new(last_declare, tx_list, message).into(); - Some(hex::encode(serialize(&block))) + roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message).into(); + Ok(hex::encode(serialize(&block))) } pub async fn send( @@ -138,10 +220,11 @@ impl JobDeclaratorDownstream { self_mutex.clone(), message, ) { - Some(inner) => inner, - None => { - error!("Received solution but no job available"); + Ok(inner) => inner, + Err(e) => { + error!("Received solution but encountered error: {:?}", e); recv.close(); + //TODO should we brake it? break; } }; @@ -193,6 +276,7 @@ impl JobDeclaratorDownstream { break; } } + JobDeclaratorDownstream::get_job_transactions_from_mempool(self_mutex.clone()).await; } }); } diff --git a/roles/jd-server/src/lib/status.rs b/roles/jd-server/src/lib/status.rs index b05294e47b..010cceef6f 100644 --- a/roles/jd-server/src/lib/status.rs +++ b/roles/jd-server/src/lib/status.rs @@ -94,6 +94,7 @@ async fn send_status( outcome } +// TODO do we really want to brake at MempoolError and ImpossibleToReconstructBlock? // this is called by `error_handling::handle_result!` pub async fn handle_error(sender: &Sender, e: JdsError) -> error_handling::ErrorBranch { tracing::debug!("Error: {:?}", &e); @@ -121,5 +122,8 @@ pub async fn handle_error(sender: &Sender, e: JdsError) -> error_handling::Error JdsError::MempoolError(_) => { send_status(sender, e, error_handling::ErrorBranch::Break).await } + JdsError::ImpossibleToReconstructBlock(_) => { + send_status(sender, e, error_handling::ErrorBranch::Break).await + } } } From 970cd43dc701bf713b0f2aa6279e4e858f6f3343 Mon Sep 17 00:00:00 2001 From: lorban Date: Mon, 11 Mar 2024 20:49:43 +0100 Subject: [PATCH 22/38] error management --- roles/jd-server/src/lib/error.rs | 6 +- .../src/lib/job_declarator/message_handler.rs | 170 ++++++------------ roles/jd-server/src/lib/job_declarator/mod.rs | 146 +++++++++------ roles/jd-server/src/lib/status.rs | 6 +- 4 files changed, 149 insertions(+), 179 deletions(-) diff --git a/roles/jd-server/src/lib/error.rs b/roles/jd-server/src/lib/error.rs index 235c8f19d7..7b58c75939 100644 --- a/roles/jd-server/src/lib/error.rs +++ b/roles/jd-server/src/lib/error.rs @@ -23,6 +23,7 @@ pub enum JdsError { Sv2ProtocolError((u32, Mining<'static>)), MempoolError(JdsMempoolError), ImpossibleToReconstructBlock(String), + NoLastDeclaredJob, } impl std::fmt::Display for JdsError { @@ -43,7 +44,10 @@ impl std::fmt::Display for JdsError { write!(f, "Received Sv2 Protocol Error from upstream: `{:?}`", e) } MempoolError(ref e) => write!(f, "Mempool error: `{:?}`", e), - ImpossibleToReconstructBlock(e) => write!(f, "Error in reconstructing the block: {:?}", e), + ImpossibleToReconstructBlock(e) => { + write!(f, "Error in reconstructing the block: {:?}", e) + } + NoLastDeclaredJob => write!(f, "Last declared job not found"), } } } diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 6e9caf32f9..7609f3a2f2 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -6,14 +6,14 @@ use roles_logic_sv2::{ DeclareMiningJobError, DeclareMiningJobSuccess, IdentifyTransactionsSuccess, ProvideMissingTransactions, ProvideMissingTransactionsSuccess, SubmitSolutionJd, }, - parsers::JobDeclaration, utils::Mutex, + parsers::JobDeclaration, }; -use std::{convert::TryInto, io::Cursor, sync::Arc}; -use stratum_common::bitcoin::{Transaction, Txid}; +use std::{convert::TryInto, io::Cursor}; +use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; use super::{signed_token, TransactionState}; -use crate::mempool::{self, error::JdsMempoolError, JDsMempool}; -use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; +use crate::mempool; +use roles_logic_sv2::errors::Error; use stratum_common::bitcoin::consensus::Decodable; use tracing::info; @@ -77,9 +77,9 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { .safe_lock(|x| x.to_short_ids(nonce)) .unwrap() .unwrap(); - let mut transactions_with_state = vec![TransactionState::Missing; short_hash_list.len()]; - let mut txs_to_retrieve: Vec<(String, usize)> = vec![]; - let mut missing_txs: Vec = Vec::new(); + let mut transactions_with_state = + vec![TransactionState::Missing; short_hash_list.len()]; + let mut missing_txs: Vec = Vec::new(); for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); @@ -89,20 +89,19 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { transactions_with_state[i] = TransactionState::Present(tx.clone()); } None => { - transactions_with_state[i] = TransactionState::ToBeRetrievedFromMempool(tx_data.id); + transactions_with_state[i] = + TransactionState::ToBeRetrievedFromMempool(tx_data.id); } }, None => { transactions_with_state[i] = TransactionState::Missing; // TODO remove this, the the ids of missing transactions from the vector missing_txs.push(i as u16); - }, + } } } - self.declared_mining_job = Some(( - message.clone().into_static(), - transactions_with_state - )); + self.declared_mining_job = + (Some(message.clone().into_static()), transactions_with_state); //let self_mutex = Arc::new(Mutex::new(self)); //add_tx_data_to_job(self_mutex); @@ -122,7 +121,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { let message_provide_missing_transactions = ProvideMissingTransactions { request_id: message.request_id, // TODO here get the missing IDS from the entries of txs_in_job which are - // TransactionState::Missing + // TransactionState::Missing unknown_tx_position_list: missing_txs.into(), }; let message_enum_provide_missing_transactions = @@ -153,59 +152,48 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { &mut self, message: ProvideMissingTransactionsSuccess, ) -> Result { - match &mut self.declared_mining_job { - Some((_, ref mut transactions_with_state)) => { - for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { - for tx_with_state in transactions_with_state.clone() { - match tx_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => continue, - TransactionState::Missing => { - let mut cursor = Cursor::new(tx); - // TODO remove this unwrap - let transaction = Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); - transactions_with_state[i] = TransactionState::Present(transaction.clone()); - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - transaction.txid(), - Some(transaction), - ); - break; - } - - } + let (_, ref mut transactions_with_state) = &mut self.declared_mining_job; + for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { + for tx_with_state in transactions_with_state.clone() { + match tx_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => continue, + TransactionState::Missing => { + let mut cursor = Cursor::new(tx); + // TODO remove this unwrap + let transaction = + Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); + transactions_with_state[i] = TransactionState::Present(transaction.clone()); + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + transaction.txid(), + Some(transaction), + ); + break; } } - // if there still a missing transaction return an error - for tx_with_state in transactions_with_state { - match tx_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => continue, - TransactionState::Missing => { - return Err(Error::JDSMissingTransactions); - - } - } - } - // TODO check it - let tx_hash_list_hash = self.tx_hash_list_hash.clone().unwrap().into_static(); - let message_success = DeclareMiningJobSuccess { - request_id: message.request_id, - new_mining_job_token: signed_token( - tx_hash_list_hash, - &self.public_key.clone(), - &self.private_key.clone(), - ), - }; - let message_enum_success = JobDeclaration::DeclareMiningJobSuccess(message_success); - Ok(SendTo::Respond(message_enum_success)) } - None => Err(Error::LogicErrorMessage(Box::new( - AllMessages::JobDeclaration(JobDeclaration::ProvideMissingTransactionsSuccess( - message.clone().into_static(), - )), - ))), } + // if there still a missing transaction return an error + for tx_with_state in transactions_with_state { + match tx_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => continue, + TransactionState::Missing => return Err(Error::JDSMissingTransactions), + } + } + // TODO check it + let tx_hash_list_hash = self.tx_hash_list_hash.clone().unwrap().into_static(); + let message_success = DeclareMiningJobSuccess { + request_id: message.request_id, + new_mining_job_token: signed_token( + tx_hash_list_hash, + &self.public_key.clone(), + &self.private_key.clone(), + ), + }; + let message_enum_success = JobDeclaration::DeclareMiningJobSuccess(message_success); + Ok(SendTo::Respond(message_enum_success)) } fn handle_submit_solution(&mut self, message: SubmitSolutionJd<'_>) -> Result { @@ -214,57 +202,3 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { Ok(SendTo::None(Some(m))) } } - -//fn add_tx_data_to_job(jdd: Arc>) { -// tokio::task::spawn(async move { -// let mut tx_list: Vec = Vec::new(); -// let mut new_transactions: Vec = Vec::new(); -// let mempool = jdd.safe_lock(|a| a.mempool.clone()).unwrap(); -// jdd.safe_lock(|a| for tx in a.declared_mining_job.clone().unwrap().1 { -// match tx { -// TransactionState::Present(_) => continue, -// TransactionState::Missing => continue, -// TransactionState::ToBeRetrievedFromMempool(m) => tx_list.push(m), -// } -// }); -// for txid in tx_list { -// let new_tx_data: Result = mempool -// .safe_lock(|x| x.get_client()) -// .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? -// .ok_or(JdsMempoolError::NoClient)? -// .get_raw_transaction(&txid.to_string(), None) -// .await -// .map_err(JdsMempoolError::Rpc); -// if let Ok(transaction) = new_tx_data { -// new_transactions.push(transaction); -// //this unwrap is safe -// } else { -// // TODO propagate error -// todo!() -// }; -// }; -// -// //for tx in tx_list.iter().enumerate() { -// // match tx.1 { -// // &TransactionState::Missing | &TransactionState::Present(_) => continue, -// // &TransactionState::ToBeRetrievedFromMempool(txid) => { -// // let new_tx_data: Result = mempool -// // .safe_lock(|x| x.get_client()) -// // .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? -// // .ok_or(JdsMempoolError::NoClient)? -// // .get_raw_transaction(&txid.to_string(), None) -// // .await -// // .map_err(JdsMempoolError::Rpc); -// // if let Ok(transaction) = new_tx_data { -// // new_transactions_mutex.safe_lock(|a| a.push(transaction)); -// // //this unwrap is safe -// // } else { -// // // TODO propagate error -// // todo!() -// // }; -// // } -// // }; -// //} -// Ok::<(), JdsMempoolError>(()) -// }); -//} diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 14bfa1a3f2..22e080115b 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -1,4 +1,6 @@ pub mod message_handler; +use crate::mempool; + use super::{error::JdsError, mempool::JDsMempool, status, Configuration, EitherFrame, StdFrame}; use async_channel::{Receiver, Sender}; use binary_sv2::{B0255, U256}; @@ -24,7 +26,6 @@ use stratum_common::bitcoin::{ Block, Transaction, Txid, }; - // this structure wraps each transaction with the index of the position in the job declared by the // JD-Client. the tx field can be None if the transaction is missing. In this case, the index is // used to retrieve the transaction from the message ProvideMissingTransactionsSuccess @@ -41,7 +42,6 @@ enum TransactionState { Missing, } - #[derive(Debug)] pub struct JobDeclaratorDownstream { sender: Sender, @@ -59,7 +59,7 @@ pub struct JobDeclaratorDownstream { // this should be (Option>, Vec) // TODO call the vector with TransactionState in with the same name everywhere, also in the // block creator in utils - declared_mining_job: Option<(DeclareMiningJob<'static>, Vec)>, + declared_mining_job: (Option>, Vec), tx_hash_list_hash: Option>, } @@ -87,82 +87,100 @@ impl JobDeclaratorDownstream { public_key: config.authority_public_key, private_key: config.authority_secret_key, mempool, - declared_mining_job: None, + declared_mining_job: (None, Vec::new()), tx_hash_list_hash: None, } } - async fn get_job_transactions_from_mempool(self_mutex: Arc>) { + // This only errors that are returned are PoisonLock, Custom, MempoolError + // this function is called in JobDeclaratorDowenstream::start(), if different errors are + // returned, change also the error management there + async fn retrieve_transactions_via_rpc( + self_mutex: Arc>, + ) -> Result<(), JdsError> { let mut transactions_to_be_retrieved: Vec = Vec::new(); let mut new_transactions: Vec = Vec::new(); - self_mutex.clone().safe_lock(|a| for tx_with_state in a.declared_mining_job.clone().unwrap().1 { + let transactions_with_state = self_mutex + .clone() + .safe_lock(|a| a.declared_mining_job.clone()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))? + .1; + for tx_with_state in transactions_with_state { match tx_with_state { TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(tx) => transactions_to_be_retrieved.push(tx), + TransactionState::ToBeRetrievedFromMempool(tx) => { + transactions_to_be_retrieved.push(tx) + } TransactionState::Missing => continue, } - }); - let mempool_ = self_mutex.safe_lock(|a| a.mempool.clone()).unwrap(); - let client = mempool_.safe_lock(|a| a.get_client()).unwrap().unwrap(); + } + let mempool_ = self_mutex + .safe_lock(|a| a.mempool.clone()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))?; + let client = mempool_ + .clone() + .safe_lock(|a| a.get_client()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))? + .ok_or(JdsError::MempoolError( + mempool::error::JdsMempoolError::NoClient, + ))?; for txid in transactions_to_be_retrieved { - let transaction = client.get_raw_transaction(&txid.to_string(), None).await.unwrap(); + let transaction = client + .get_raw_transaction(&txid.to_string(), None) + .await + .map_err(|e| JdsError::MempoolError(mempool::error::JdsMempoolError::Rpc(e)))?; + let txid = transaction.txid(); + mempool::JDsMempool::add_tx_data_to_mempool( + mempool_.clone(), + txid, + Some(transaction.clone()), + ); new_transactions.push(transaction); -// let new_tx_data: Result = mempool -// .safe_lock(|x| x.get_client()) -// .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? -// .ok_or(JdsMempoolError::NoClient)? -// .get_raw_transaction(&txid.to_string(), None) -// .await -// .map_err(JdsMempoolError::Rpc); -// if let Ok(transaction) = new_tx_data { -// new_transactions.push(transaction); -// //this unwrap is safe -// } else { -// // TODO propagate error -// todo!() -// }; - - }; + } + //TODO remove this unwrap for transaction in new_transactions { - self_mutex.clone().safe_lock(|a| for transaction_with_state in &mut a.declared_mining_job.as_mut().unwrap().1 { - match transaction_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => { - *transaction_with_state = TransactionState::Present(transaction); - break; - }, - TransactionState::Missing => continue, - } - }).unwrap() - }; - //self_mutex.clone().safe_lock(|a| for transaction_with_state in a.declared_mining_job.unwrap().1 { - // match transaction_with_state { - // TransactionState::Present(_) => continue, - // TransactionState::ToBeRetrievedFromMempool(_) => {transaction_with_state = TransactionState::Present(new_transactions.clone()[1])}, - // TransactionState::Missing => continue, - // } - //}); - - todo!() - + self_mutex + .clone() + .safe_lock(|a| { + for transaction_with_state in &mut a.declared_mining_job.1 { + match transaction_with_state { + TransactionState::Present(_) => continue, + TransactionState::ToBeRetrievedFromMempool(_) => { + *transaction_with_state = TransactionState::Present(transaction); + break; + } + TransactionState::Missing => continue, + } + } + }) + .map_err(|e| JdsError::PoisonLock(e.to_string()))? + } + Ok(()) } - fn get_block_hex(self_mutex: Arc>, message: SubmitSolutionJd) -> Result { - //TODO: implement logic for success or error - let (last_declare, transactions_with_state) = self_mutex - .safe_lock(|x| x.declared_mining_job.take()) - // TODO manage these errors - .unwrap().unwrap(); + // TODO try to remove this configuration + #[allow(clippy::result_large_err)] + fn get_block_hex( + self_mutex: Arc>, + message: SubmitSolutionJd, + ) -> Result { + let (last_declare_, transactions_with_state) = self_mutex + .safe_lock(|x| x.declared_mining_job.clone()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))?; + let last_declare = last_declare_.ok_or(JdsError::NoLastDeclaredJob)?; let mut transactions_list: Vec = Vec::new(); for tx_with_state in transactions_with_state.iter().enumerate() { if let TransactionState::Present(tx) = tx_with_state.1 { transactions_list.push(tx.clone()); } else { - return Err(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string())); + return Err(JdsError::ImpossibleToReconstructBlock( + "Missing transactions".to_string(), + )); }; } let block: Block = - roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message).into(); + roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message) + .into(); Ok(hex::encode(serialize(&block))) } @@ -222,7 +240,10 @@ impl JobDeclaratorDownstream { ) { Ok(inner) => inner, Err(e) => { - error!("Received solution but encountered error: {:?}", e); + error!( + "Received solution but encountered error: {:?}", + e + ); recv.close(); //TODO should we brake it? break; @@ -276,7 +297,16 @@ impl JobDeclaratorDownstream { break; } } - JobDeclaratorDownstream::get_job_transactions_from_mempool(self_mutex.clone()).await; + let retrive_transactions = + JobDeclaratorDownstream::retrieve_transactions_via_rpc(self_mutex.clone()) + .await; + match retrive_transactions { + Ok(_) => (), + Err(error) => { + handle_result!(tx_status, Err(error)); + break; + } + } } }); } diff --git a/roles/jd-server/src/lib/status.rs b/roles/jd-server/src/lib/status.rs index 010cceef6f..83a50026ff 100644 --- a/roles/jd-server/src/lib/status.rs +++ b/roles/jd-server/src/lib/status.rs @@ -94,7 +94,6 @@ async fn send_status( outcome } -// TODO do we really want to brake at MempoolError and ImpossibleToReconstructBlock? // this is called by `error_handling::handle_result!` pub async fn handle_error(sender: &Sender, e: JdsError) -> error_handling::ErrorBranch { tracing::debug!("Error: {:?}", &e); @@ -123,7 +122,10 @@ pub async fn handle_error(sender: &Sender, e: JdsError) -> error_handling::Error send_status(sender, e, error_handling::ErrorBranch::Break).await } JdsError::ImpossibleToReconstructBlock(_) => { - send_status(sender, e, error_handling::ErrorBranch::Break).await + send_status(sender, e, error_handling::ErrorBranch::Continue).await + } + JdsError::NoLastDeclaredJob => { + send_status(sender, e, error_handling::ErrorBranch::Continue).await } } } From 371d4d40ddb19337717ecf7f7374ccaa9f3b1258 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Tue, 12 Mar 2024 20:05:30 +0100 Subject: [PATCH 23/38] index used in transactions_with_state fix --- roles/jd-server/src/lib/job_declarator/message_handler.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 7609f3a2f2..c758013ef5 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -153,8 +153,8 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message: ProvideMissingTransactionsSuccess, ) -> Result { let (_, ref mut transactions_with_state) = &mut self.declared_mining_job; - for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { - for tx_with_state in transactions_with_state.clone() { + for (_index, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { + for (i, tx_with_state) in transactions_with_state.clone().iter().enumerate() { match tx_with_state { TransactionState::Present(_) => continue, TransactionState::ToBeRetrievedFromMempool(_) => continue, @@ -201,4 +201,4 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { Ok(SendTo::None(Some(m))) } -} +} \ No newline at end of file From 3207c7afe232ca744989875cec632d5e61fec3e3 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Wed, 13 Mar 2024 16:49:21 +0100 Subject: [PATCH 24/38] double for cycle removal, missing_indexes reintroduction --- .../src/lib/job_declarator/message_handler.rs | 46 +++++++++---------- roles/jd-server/src/lib/job_declarator/mod.rs | 9 ++-- 2 files changed, 26 insertions(+), 29 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index c758013ef5..48429fdc37 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -11,9 +11,9 @@ use roles_logic_sv2::{ use std::{convert::TryInto, io::Cursor}; use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; +use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use super::{signed_token, TransactionState}; use crate::mempool; -use roles_logic_sv2::errors::Error; use stratum_common::bitcoin::consensus::Decodable; use tracing::info; @@ -101,7 +101,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } } self.declared_mining_job = - (Some(message.clone().into_static()), transactions_with_state); + (Some(message.clone().into_static()), transactions_with_state, missing_txs.clone()); //let self_mutex = Arc::new(Mutex::new(self)); //add_tx_data_to_job(self_mutex); @@ -152,27 +152,27 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { &mut self, message: ProvideMissingTransactionsSuccess, ) -> Result { - let (_, ref mut transactions_with_state) = &mut self.declared_mining_job; - for (_index, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { - for (i, tx_with_state) in transactions_with_state.clone().iter().enumerate() { - match tx_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => continue, - TransactionState::Missing => { - let mut cursor = Cursor::new(tx); - // TODO remove this unwrap - let transaction = - Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); - transactions_with_state[i] = TransactionState::Present(transaction.clone()); - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - transaction.txid(), - Some(transaction), - ); - break; - } - } - } + let (_, ref mut transactions_with_state, missing_indexes) = &mut self.declared_mining_job; + for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { + let mut cursor = Cursor::new(tx); + // TODO remove this unwrap + let transaction= Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); + let index = + *missing_indexes + .get(i) + .ok_or(Error::LogicErrorMessage(Box::new( + AllMessages::JobDeclaration( + JobDeclaration::ProvideMissingTransactionsSuccess( + message.clone().into_static(), + ), + ), + )))? as usize; + transactions_with_state[index] = TransactionState::Present(transaction.clone()); + mempool::JDsMempool::add_tx_data_to_mempool( + self.mempool.clone(), + transaction.txid(), + Some(transaction), + ); } // if there still a missing transaction return an error for tx_with_state in transactions_with_state { diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 22e080115b..52eeca26f7 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -56,10 +56,7 @@ pub struct JobDeclaratorDownstream { private_key: Secp256k1SecretKey, mempool: Arc>, // Vec is the vector of missing transactions - // this should be (Option>, Vec) - // TODO call the vector with TransactionState in with the same name everywhere, also in the - // block creator in utils - declared_mining_job: (Option>, Vec), + declared_mining_job: (Option>, Vec, Vec), tx_hash_list_hash: Option>, } @@ -87,7 +84,7 @@ impl JobDeclaratorDownstream { public_key: config.authority_public_key, private_key: config.authority_secret_key, mempool, - declared_mining_job: (None, Vec::new()), + declared_mining_job: (None, Vec::new(), Vec::new()), tx_hash_list_hash: None, } } @@ -164,7 +161,7 @@ impl JobDeclaratorDownstream { self_mutex: Arc>, message: SubmitSolutionJd, ) -> Result { - let (last_declare_, transactions_with_state) = self_mutex + let (last_declare_, transactions_with_state, _) = self_mutex .safe_lock(|x| x.declared_mining_job.clone()) .map_err(|e| JdsError::PoisonLock(e.to_string()))?; let last_declare = last_declare_.ok_or(JdsError::NoLastDeclaredJob)?; From 68887cb3b2ca4ede4a70590394931d1fe88b17c4 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Wed, 13 Mar 2024 16:51:18 +0100 Subject: [PATCH 25/38] fmt and cleanup --- .../src/lib/job_declarator/message_handler.rs | 37 ++++++++----------- roles/jd-server/src/lib/job_declarator/mod.rs | 6 ++- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 48429fdc37..36bc7a3c2d 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -11,9 +11,9 @@ use roles_logic_sv2::{ use std::{convert::TryInto, io::Cursor}; use stratum_common::bitcoin::Transaction; pub type SendTo = SendTo_, ()>; -use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use super::{signed_token, TransactionState}; use crate::mempool; +use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::info; @@ -95,16 +95,15 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { }, None => { transactions_with_state[i] = TransactionState::Missing; - // TODO remove this, the the ids of missing transactions from the vector missing_txs.push(i as u16); } } } - self.declared_mining_job = - (Some(message.clone().into_static()), transactions_with_state, missing_txs.clone()); - - //let self_mutex = Arc::new(Mutex::new(self)); - //add_tx_data_to_job(self_mutex); + self.declared_mining_job = ( + Some(message.clone().into_static()), + transactions_with_state, + missing_txs.clone(), + ); if missing_txs.is_empty() { let message_success = DeclareMiningJobSuccess { @@ -120,8 +119,6 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } else { let message_provide_missing_transactions = ProvideMissingTransactions { request_id: message.request_id, - // TODO here get the missing IDS from the entries of txs_in_job which are - // TransactionState::Missing unknown_tx_position_list: missing_txs.into(), }; let message_enum_provide_missing_transactions = @@ -156,17 +153,15 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { let mut cursor = Cursor::new(tx); // TODO remove this unwrap - let transaction= Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); - let index = - *missing_indexes - .get(i) - .ok_or(Error::LogicErrorMessage(Box::new( - AllMessages::JobDeclaration( - JobDeclaration::ProvideMissingTransactionsSuccess( - message.clone().into_static(), - ), - ), - )))? as usize; + let transaction = + Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); + let index = *missing_indexes + .get(i) + .ok_or(Error::LogicErrorMessage(Box::new( + AllMessages::JobDeclaration(JobDeclaration::ProvideMissingTransactionsSuccess( + message.clone().into_static(), + )), + )))? as usize; transactions_with_state[index] = TransactionState::Present(transaction.clone()); mempool::JDsMempool::add_tx_data_to_mempool( self.mempool.clone(), @@ -201,4 +196,4 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { Ok(SendTo::None(Some(m))) } -} \ No newline at end of file +} diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 52eeca26f7..db0ce10f81 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -56,7 +56,11 @@ pub struct JobDeclaratorDownstream { private_key: Secp256k1SecretKey, mempool: Arc>, // Vec is the vector of missing transactions - declared_mining_job: (Option>, Vec, Vec), + declared_mining_job: ( + Option>, + Vec, + Vec, + ), tx_hash_list_hash: Option>, } From 096b30bd278d1c2995d9369c9a356041e9a2ab79 Mon Sep 17 00:00:00 2001 From: Gabriele Vernetti Date: Wed, 13 Mar 2024 23:01:40 +0100 Subject: [PATCH 26/38] error management for transactions decoding --- protocols/v2/roles-logic-sv2/src/errors.rs | 2 ++ .../src/lib/job_declarator/message_handler.rs | 5 ++--- roles/jd-server/src/lib/job_declarator/mod.rs | 12 +++++------- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/protocols/v2/roles-logic-sv2/src/errors.rs b/protocols/v2/roles-logic-sv2/src/errors.rs index 198b94e011..20c4bcd554 100644 --- a/protocols/v2/roles-logic-sv2/src/errors.rs +++ b/protocols/v2/roles-logic-sv2/src/errors.rs @@ -46,6 +46,7 @@ pub enum Error { VersionTooBig, TxVersionTooBig, TxVersionTooLow, + TxDecodingError(String), NotFoundChannelId, NoValidJob, NoValidTranslatorJob, @@ -139,6 +140,7 @@ impl Display for Error { VersionTooBig => write!(f, "We are trying to construct a block header with version bigger than i32::MAX"), TxVersionTooBig => write!(f, "Tx version can not be greater than i32::MAX"), TxVersionTooLow => write!(f, "Tx version can not be lower than 1"), + TxDecodingError(e) => write!(f, "Impossible to decode tx: {:?}", e), NotFoundChannelId => write!(f, "No downstream has been registred for this channel id"), NoValidJob => write!(f, "Impossible to create a standard job for channelA cause no valid job has been received from upstream yet"), NoValidTranslatorJob => write!(f, "Impossible to create a extended job for channel cause no valid job has been received from upstream yet"), diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 36bc7a3c2d..3774ecb240 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -152,9 +152,8 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { let (_, ref mut transactions_with_state, missing_indexes) = &mut self.declared_mining_job; for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { let mut cursor = Cursor::new(tx); - // TODO remove this unwrap - let transaction = - Transaction::consensus_decode_from_finite_reader(&mut cursor).unwrap(); + let transaction = Transaction::consensus_decode_from_finite_reader(&mut cursor) + .map_err(|e| Error::TxDecodingError(e.to_string()))?; let index = *missing_indexes .get(i) .ok_or(Error::LogicErrorMessage(Box::new( diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index db0ce10f81..18c4c3f19a 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -159,12 +159,10 @@ impl JobDeclaratorDownstream { Ok(()) } - // TODO try to remove this configuration - #[allow(clippy::result_large_err)] fn get_block_hex( self_mutex: Arc>, message: SubmitSolutionJd, - ) -> Result { + ) -> Result> { let (last_declare_, transactions_with_state, _) = self_mutex .safe_lock(|x| x.declared_mining_job.clone()) .map_err(|e| JdsError::PoisonLock(e.to_string()))?; @@ -174,9 +172,9 @@ impl JobDeclaratorDownstream { if let TransactionState::Present(tx) = tx_with_state.1 { transactions_list.push(tx.clone()); } else { - return Err(JdsError::ImpossibleToReconstructBlock( + return Err(Box::new(JdsError::ImpossibleToReconstructBlock( "Missing transactions".to_string(), - )); + ))); }; } let block: Block = @@ -298,10 +296,10 @@ impl JobDeclaratorDownstream { break; } } - let retrive_transactions = + let retrieve_transactions = JobDeclaratorDownstream::retrieve_transactions_via_rpc(self_mutex.clone()) .await; - match retrive_transactions { + match retrieve_transactions { Ok(_) => (), Err(error) => { handle_result!(tx_status, Err(error)); From f5bd5e0b87697aa1c41c09d2457f214420548427 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Fri, 15 Mar 2024 22:45:38 +0100 Subject: [PATCH 27/38] transactions data removed from JobDeclaratorDownstream data structure, get them from jds mempool when a block is found --- .../src/lib/job_declarator/message_handler.rs | 8 +++--- roles/jd-server/src/lib/job_declarator/mod.rs | 27 +++++++++++++------ 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 3774ecb240..955aeb28d5 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -86,11 +86,11 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { match short_id_mempool.get(&sid_) { Some(tx_data) => match &tx_data.tx { Some(tx) => { - transactions_with_state[i] = TransactionState::Present(tx.clone()); + transactions_with_state[i] = TransactionState::Present(tx.txid()); } None => { transactions_with_state[i] = - TransactionState::ToBeRetrievedFromMempool(tx_data.id); + TransactionState::ToBeRetrievedFromNodeMempool(tx_data.id); } }, None => { @@ -161,7 +161,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message.clone().into_static(), )), )))? as usize; - transactions_with_state[index] = TransactionState::Present(transaction.clone()); + transactions_with_state[index] = TransactionState::Present(transaction.txid()); mempool::JDsMempool::add_tx_data_to_mempool( self.mempool.clone(), transaction.txid(), @@ -172,7 +172,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { for tx_with_state in transactions_with_state { match tx_with_state { TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => continue, + TransactionState::ToBeRetrievedFromNodeMempool(_) => continue, TransactionState::Missing => return Err(Error::JDSMissingTransactions), } } diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 18c4c3f19a..a428f3e0fd 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -37,8 +37,8 @@ use stratum_common::bitcoin::{ #[derive(Clone, Debug)] enum TransactionState { - Present(Transaction), - ToBeRetrievedFromMempool(Txid), + Present(Txid), + ToBeRetrievedFromNodeMempool(Txid), Missing, } @@ -109,7 +109,7 @@ impl JobDeclaratorDownstream { for tx_with_state in transactions_with_state { match tx_with_state { TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(tx) => { + TransactionState::ToBeRetrievedFromNodeMempool(tx) => { transactions_to_be_retrieved.push(tx) } TransactionState::Missing => continue, @@ -138,7 +138,6 @@ impl JobDeclaratorDownstream { ); new_transactions.push(transaction); } - //TODO remove this unwrap for transaction in new_transactions { self_mutex .clone() @@ -146,8 +145,8 @@ impl JobDeclaratorDownstream { for transaction_with_state in &mut a.declared_mining_job.1 { match transaction_with_state { TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromMempool(_) => { - *transaction_with_state = TransactionState::Present(transaction); + TransactionState::ToBeRetrievedFromNodeMempool(_) => { + *transaction_with_state = TransactionState::Present(transaction.txid()); break; } TransactionState::Missing => continue, @@ -166,11 +165,23 @@ impl JobDeclaratorDownstream { let (last_declare_, transactions_with_state, _) = self_mutex .safe_lock(|x| x.declared_mining_job.clone()) .map_err(|e| JdsError::PoisonLock(e.to_string()))?; + let mempool_ = self_mutex + .safe_lock(|x| x.mempool.clone()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))?; let last_declare = last_declare_.ok_or(JdsError::NoLastDeclaredJob)?; let mut transactions_list: Vec = Vec::new(); for tx_with_state in transactions_with_state.iter().enumerate() { - if let TransactionState::Present(tx) = tx_with_state.1 { - transactions_list.push(tx.clone()); + if let TransactionState::Present(txid) = tx_with_state.1 { + let tx_ = match mempool_.safe_lock(|x| x.mempool.get(txid).cloned()) { + Ok(tx) => tx, + Err(e) => return Err(Box::new(JdsError::PoisonLock(e.to_string()))), + }; + let tx = tx_.ok_or(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string())); + if let Ok(Some(tx)) = tx { + transactions_list.push(tx); + } else { + return Err(Box::new(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string()))); + } } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( "Missing transactions".to_string(), From 57685894132a24e13925e6f4d22b2100e17f75fd Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Fri, 15 Mar 2024 22:47:05 +0100 Subject: [PATCH 28/38] fmt --- roles/jd-server/src/lib/job_declarator/mod.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index a428f3e0fd..a2cb2ca3d2 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -146,7 +146,8 @@ impl JobDeclaratorDownstream { match transaction_with_state { TransactionState::Present(_) => continue, TransactionState::ToBeRetrievedFromNodeMempool(_) => { - *transaction_with_state = TransactionState::Present(transaction.txid()); + *transaction_with_state = + TransactionState::Present(transaction.txid()); break; } TransactionState::Missing => continue, @@ -176,11 +177,15 @@ impl JobDeclaratorDownstream { Ok(tx) => tx, Err(e) => return Err(Box::new(JdsError::PoisonLock(e.to_string()))), }; - let tx = tx_.ok_or(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string())); + let tx = tx_.ok_or(JdsError::ImpossibleToReconstructBlock( + "Missing transactions".to_string(), + )); if let Ok(Some(tx)) = tx { transactions_list.push(tx); } else { - return Err(Box::new(JdsError::ImpossibleToReconstructBlock("Missing transactions".to_string()))); + return Err(Box::new(JdsError::ImpossibleToReconstructBlock( + "Missing transactions".to_string(), + ))); } } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( From 9fdd74a679e008406d39771b8b821a828615ca2a Mon Sep 17 00:00:00 2001 From: lorban Date: Sun, 17 Mar 2024 22:24:07 +0100 Subject: [PATCH 29/38] Move task on main As suggested in https://github.com/stratum-mining/stratum/pull/772#issuecomment-1999112198 A task that before was blocking now is moved as a separate task on main --- .../src/lib/job_declarator/message_handler.rs | 37 ++-- roles/jd-server/src/lib/job_declarator/mod.rs | 190 +++++++++--------- roles/jd-server/src/lib/mempool/mod.rs | 34 +++- roles/jd-server/src/main.rs | 21 +- 4 files changed, 165 insertions(+), 117 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 955aeb28d5..8e038f1818 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -9,10 +9,9 @@ use roles_logic_sv2::{ parsers::JobDeclaration, }; use std::{convert::TryInto, io::Cursor}; -use stratum_common::bitcoin::Transaction; +use stratum_common::bitcoin::{Transaction, Txid}; pub type SendTo = SendTo_, ()>; use super::{signed_token, TransactionState}; -use crate::mempool; use roles_logic_sv2::{errors::Error, parsers::PoolMessages as AllMessages}; use stratum_common::bitcoin::consensus::Decodable; use tracing::info; @@ -62,6 +61,12 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { } fn handle_declare_mining_job(&mut self, message: DeclareMiningJob) -> Result { + // the transactions that are present in the mempool are stored here, that is sent to the + // mempool which use the rpc client to retrieve the whole data for each transactions. + // The unknown transactions is a vector that contains the transactions that are not in the + // jds mempool, and will be non-empty in the ProvideMissingTransactionsSuccess message + let mut known_transactions: Vec = Vec::new(); + let unknown_transactions: Vec = Vec::new(); self.tx_hash_list_hash = Some(message.tx_hash_list_hash.clone().into_static()); if self.verify_job(&message) { let short_hash_list: Vec = message @@ -84,14 +89,9 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { for (i, sid) in short_hash_list.iter().enumerate() { let sid_: [u8; 6] = sid.to_vec().try_into().unwrap(); match short_id_mempool.get(&sid_) { - Some(tx_data) => match &tx_data.tx { - Some(tx) => { - transactions_with_state[i] = TransactionState::Present(tx.txid()); - } - None => { - transactions_with_state[i] = - TransactionState::ToBeRetrievedFromNodeMempool(tx_data.id); - } + Some(tx_data) => { + transactions_with_state[i] = TransactionState::PresentInMempool(tx_data.id); + known_transactions.push(tx_data.id); }, None => { transactions_with_state[i] = TransactionState::Missing; @@ -104,6 +104,8 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { transactions_with_state, missing_txs.clone(), ); + // here we send the transactions that we want to be stored in jds mempool with full data + self.sender_add_txs_to_mempool.send(super::AddTrasactionsToMempool { known_transactions, unknown_transactions}); if missing_txs.is_empty() { let message_success = DeclareMiningJobSuccess { @@ -150,10 +152,13 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message: ProvideMissingTransactionsSuccess, ) -> Result { let (_, ref mut transactions_with_state, missing_indexes) = &mut self.declared_mining_job; + let mut unknown_transactions: Vec = Vec::new(); + let known_transactions: Vec = Vec::new(); for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { let mut cursor = Cursor::new(tx); let transaction = Transaction::consensus_decode_from_finite_reader(&mut cursor) .map_err(|e| Error::TxDecodingError(e.to_string()))?; + &unknown_transactions.push(transaction.clone()); let index = *missing_indexes .get(i) .ok_or(Error::LogicErrorMessage(Box::new( @@ -161,18 +166,14 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message.clone().into_static(), )), )))? as usize; - transactions_with_state[index] = TransactionState::Present(transaction.txid()); - mempool::JDsMempool::add_tx_data_to_mempool( - self.mempool.clone(), - transaction.txid(), - Some(transaction), - ); + // insert the missing transactions in the mempool + transactions_with_state[index] = TransactionState::PresentInMempool(transaction.txid()); } + self.sender_add_txs_to_mempool.send(super::AddTrasactionsToMempool { known_transactions, unknown_transactions: unknown_transactions.clone()}); // if there still a missing transaction return an error for tx_with_state in transactions_with_state { match tx_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromNodeMempool(_) => continue, + TransactionState::PresentInMempool(_) => continue, TransactionState::Missing => return Err(Error::JDSMissingTransactions), } } diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index a2cb2ca3d2..bf0976575c 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -1,6 +1,4 @@ pub mod message_handler; -use crate::mempool; - use super::{error::JdsError, mempool::JDsMempool, status, Configuration, EitherFrame, StdFrame}; use async_channel::{Receiver, Sender}; use binary_sv2::{B0255, U256}; @@ -26,22 +24,29 @@ use stratum_common::bitcoin::{ Block, Transaction, Txid, }; -// this structure wraps each transaction with the index of the position in the job declared by the -// JD-Client. the tx field can be None if the transaction is missing. In this case, the index is -// used to retrieve the transaction from the message ProvideMissingTransactionsSuccess -//#[derive(Debug)] -//struct TransactionWithIndex { -// tx: Option, -// index: u16, -//} - #[derive(Clone, Debug)] enum TransactionState { - Present(Txid), - ToBeRetrievedFromNodeMempool(Txid), + PresentInMempool(Txid), Missing, } +pub fn get_ids_of_known_transactions(transactions: &Vec) -> Vec { + let mut known_transactions: Vec = Vec::new(); + for transaction in transactions { + match *transaction { + TransactionState::PresentInMempool(txid) => known_transactions.push(txid.clone()), + TransactionState::Missing => continue, + } + }; + known_transactions +} + +#[derive(Clone, Debug)] +pub struct AddTrasactionsToMempool { + pub known_transactions: Vec, + pub unknown_transactions: Vec, +} + #[derive(Debug)] pub struct JobDeclaratorDownstream { sender: Sender, @@ -62,6 +67,7 @@ pub struct JobDeclaratorDownstream { Vec, ), tx_hash_list_hash: Option>, + sender_add_txs_to_mempool: Sender, } impl JobDeclaratorDownstream { @@ -70,6 +76,7 @@ impl JobDeclaratorDownstream { sender: Sender, config: &Configuration, mempool: Arc>, + sender_add_txs_to_mempool: Sender ) -> Self { let mut coinbase_output = vec![]; // TODO: use next variables @@ -90,74 +97,73 @@ impl JobDeclaratorDownstream { mempool, declared_mining_job: (None, Vec::new(), Vec::new()), tx_hash_list_hash: None, + sender_add_txs_to_mempool, } } - // This only errors that are returned are PoisonLock, Custom, MempoolError - // this function is called in JobDeclaratorDowenstream::start(), if different errors are - // returned, change also the error management there - async fn retrieve_transactions_via_rpc( - self_mutex: Arc>, - ) -> Result<(), JdsError> { - let mut transactions_to_be_retrieved: Vec = Vec::new(); - let mut new_transactions: Vec = Vec::new(); - let transactions_with_state = self_mutex - .clone() - .safe_lock(|a| a.declared_mining_job.clone()) - .map_err(|e| JdsError::PoisonLock(e.to_string()))? - .1; - for tx_with_state in transactions_with_state { - match tx_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromNodeMempool(tx) => { - transactions_to_be_retrieved.push(tx) - } - TransactionState::Missing => continue, - } - } - let mempool_ = self_mutex - .safe_lock(|a| a.mempool.clone()) - .map_err(|e| JdsError::PoisonLock(e.to_string()))?; - let client = mempool_ - .clone() - .safe_lock(|a| a.get_client()) - .map_err(|e| JdsError::PoisonLock(e.to_string()))? - .ok_or(JdsError::MempoolError( - mempool::error::JdsMempoolError::NoClient, - ))?; - for txid in transactions_to_be_retrieved { - let transaction = client - .get_raw_transaction(&txid.to_string(), None) - .await - .map_err(|e| JdsError::MempoolError(mempool::error::JdsMempoolError::Rpc(e)))?; - let txid = transaction.txid(); - mempool::JDsMempool::add_tx_data_to_mempool( - mempool_.clone(), - txid, - Some(transaction.clone()), - ); - new_transactions.push(transaction); - } - for transaction in new_transactions { - self_mutex - .clone() - .safe_lock(|a| { - for transaction_with_state in &mut a.declared_mining_job.1 { - match transaction_with_state { - TransactionState::Present(_) => continue, - TransactionState::ToBeRetrievedFromNodeMempool(_) => { - *transaction_with_state = - TransactionState::Present(transaction.txid()); - break; - } - TransactionState::Missing => continue, - } - } - }) - .map_err(|e| JdsError::PoisonLock(e.to_string()))? - } - Ok(()) - } + //// This only errors that are returned are PoisonLock, Custom, MempoolError + //// this function is called in JobDeclaratorDowenstream::start(), if different errors are + //// returned, change also the error management there + //async fn send_transactions_needed_in_mempool( + // self_mutex: Arc>, + //) -> Result<(), JdsError> { + // let sender_add_txs_to_mempool = self_mutex.safe_lock(|a| a.sender_add_txs_to_mempool.clone()).unwrap(); + // let mut transactions_to_be_retrieved: Vec = Vec::new(); + // let transactions_with_state = self_mutex + // .clone() + // .safe_lock(|a| a.declared_mining_job.clone()) + // .map_err(|e| JdsError::PoisonLock(e.to_string()))? + // .1; + // for tx_with_state in transactions_with_state { + // match tx_with_state { + // TransactionState::PresentInMempool(txid) => transactions_to_be_retrieved.push(txid), + // TransactionState::Missing => continue, + // } + // }; + // let _ = sender_add_txs_to_mempool.send(transactions_to_be_retrieved); + // //let mempool_ = self_mutex + // // .safe_lock(|a| a.mempool.clone()) + // // .map_err(|e| JdsError::PoisonLock(e.to_string()))?; + // //let client = mempool_ + // // .clone() + // // .safe_lock(|a| a.get_client()) + // // .map_err(|e| JdsError::PoisonLock(e.to_string()))? + // // .ok_or(JdsError::MempoolError( + // // mempool::error::JdsMempoolError::NoClient, + // // ))?; + // //for txid in transactions_to_be_retrieved { + // // let transaction = client + // // .get_raw_transaction(&txid.to_string(), None) + // // .await + // // .map_err(|e| JdsError::MempoolError(mempool::error::JdsMempoolError::Rpc(e)))?; + // // let txid = transaction.txid(); + // // mempool::JDsMempool::add_tx_data_to_mempool( + // // mempool_.clone(), + // // txid, + // // Some(transaction.clone()), + // // ); + // // new_transactions.push(transaction); + // //} + // //for transaction in new_transactions { + // // self_mutex + // // .clone() + // // .safe_lock(|a| { + // // for transaction_with_state in &mut a.declared_mining_job.1 { + // // match transaction_with_state { + // // TransactionState::Present(_) => continue, + // // TransactionState::ToBeRetrievedFromNodeMempool(_) => { + // // *transaction_with_state = + // // TransactionState::Present(transaction.txid()); + // // break; + // // } + // // TransactionState::Missing => continue, + // // } + // // } + // // }) + // // .map_err(|e| JdsError::PoisonLock(e.to_string()))? + // //} + // Ok(()) + //} fn get_block_hex( self_mutex: Arc>, @@ -172,7 +178,7 @@ impl JobDeclaratorDownstream { let last_declare = last_declare_.ok_or(JdsError::NoLastDeclaredJob)?; let mut transactions_list: Vec = Vec::new(); for tx_with_state in transactions_with_state.iter().enumerate() { - if let TransactionState::Present(txid) = tx_with_state.1 { + if let TransactionState::PresentInMempool(txid) = tx_with_state.1 { let tx_ = match mempool_.safe_lock(|x| x.mempool.get(txid).cloned()) { Ok(tx) => tx, Err(e) => return Err(Box::new(JdsError::PoisonLock(e.to_string()))), @@ -312,16 +318,16 @@ impl JobDeclaratorDownstream { break; } } - let retrieve_transactions = - JobDeclaratorDownstream::retrieve_transactions_via_rpc(self_mutex.clone()) - .await; - match retrieve_transactions { - Ok(_) => (), - Err(error) => { - handle_result!(tx_status, Err(error)); - break; - } - } + //let retrieve_transactions = + // JobDeclaratorDownstream::retrieve_transactions_via_rpc(self_mutex.clone()) + // .await; + //match retrieve_transactions { + // Ok(_) => (), + // Err(error) => { + // handle_result!(tx_status, Err(error)); + // break; + // } + //} } }); } @@ -359,10 +365,11 @@ impl JobDeclarator { status_tx: crate::status::Sender, mempool: Arc>, new_block_sender: Sender, + sender_add_txs_to_mempool: Sender, ) { let self_ = Arc::new(Mutex::new(Self {})); info!("JD INITIALIZED"); - Self::accept_incoming_connection(self_, config, status_tx, mempool, new_block_sender).await; + Self::accept_incoming_connection(self_, config, status_tx, mempool, new_block_sender, sender_add_txs_to_mempool).await; } async fn accept_incoming_connection( _self_: Arc>, @@ -370,6 +377,7 @@ impl JobDeclarator { status_tx: crate::status::Sender, mempool: Arc>, new_block_sender: Sender, + sender_add_txs_to_mempool: Sender, ) { let listner = TcpListener::bind(&config.listen_jd_address).await.unwrap(); while let Ok((stream, _)) = listner.accept().await { @@ -408,6 +416,8 @@ impl JobDeclarator { sender.clone(), &config, mempool.clone(), + // each downstream has its own sender (multi producer single consumer) + sender_add_txs_to_mempool.clone(), ))); JobDeclaratorDownstream::start( diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index 69fca884fe..a7db47d651 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -1,5 +1,6 @@ pub mod error; use crate::mempool::error::JdsMempoolError; +use super::job_declarator::AddTrasactionsToMempool; use async_channel::Receiver; use bitcoin::blockdata::transaction::Transaction; use hashbrown::HashMap; @@ -58,14 +59,31 @@ impl JDsMempool { } } - pub fn add_tx_data_to_mempool( - self_: Arc>, - txid: Txid, - transaction: Option, - ) { - let _ = self_.safe_lock(|x| { - x.mempool.insert(txid, transaction); - }); + // this functions fill in the mempool the transactions with the given txid and insert the given + // transactions. The ids are for the transactions that are already known to the node, the + // unknown transactions are provided directly as a vector + pub async fn add_tx_data_to_mempool(self_: Arc>, add_transactions_to_mempool: AddTrasactionsToMempool) -> Result<(), JdsMempoolError>{ + let txids = add_transactions_to_mempool.known_transactions; + let transactions = add_transactions_to_mempool.unknown_transactions; + let client = self_ + .safe_lock(|a| a.get_client()) + .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? + .ok_or(JdsMempoolError::NoClient)?; + // fill in the mempool the transactions id in the mempool with the full transactions + // retrieved from the jd client + for txid in txids { + let transaction = client + .get_raw_transaction(&txid.to_string(), None) + .await + .map_err(|e| JdsMempoolError::Rpc(e))?; + let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); + } + + // fill in the mempool the transactions given in input + for transaction in transactions { + let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); + } + Ok(()) } pub async fn update_mempool(self_: Arc>) -> Result<(), JdsMempoolError> { diff --git a/roles/jd-server/src/main.rs b/roles/jd-server/src/main.rs index abf8cc556c..6d0530c35a 100644 --- a/roles/jd-server/src/main.rs +++ b/roles/jd-server/src/main.rs @@ -177,6 +177,7 @@ async fn main() { } } _ => { + // TODO here there should be a better error managmenet mempool::error::handle_error(&err); handle_result!(sender_submit_solution, Err(err)); } @@ -190,8 +191,26 @@ async fn main() { let cloned = config.clone(); let mempool_cloned = mempool.clone(); + let (sender_jdd_for_mempool_update, receiver_jdd_for_mempool_update) = unbounded(); task::spawn(async move { - JobDeclarator::start(cloned, sender, mempool_cloned, new_block_sender).await + JobDeclarator::start(cloned, sender, mempool_cloned, new_block_sender, sender_jdd_for_mempool_update).await + }); + task::spawn(async move { + loop { + if let Ok(add_transactions_to_mempool) = receiver_jdd_for_mempool_update.recv().await { + let mempool_cloned = mempool.clone(); + task::spawn(async move { + match lib::mempool::JDsMempool::add_tx_data_to_mempool(mempool_cloned, add_transactions_to_mempool).await { + Ok(_) => (), + Err(err) => { + // TODO + // here there should be a better error management + mempool::error::handle_error(&err); + }, + } + }); + } + } }); // Start the error handling loop From 208e4f557679b8545f0e88a9f5af0fd75ccb0755 Mon Sep 17 00:00:00 2001 From: lorban Date: Tue, 19 Mar 2024 10:54:18 +0100 Subject: [PATCH 30/38] Verify that all txs in job are present As suggested in https://github.com/stratum-mining/stratum/pull/772#issuecomment-1999112198 This commit introduces a verification that all the transactions in a job are correctly recognized when a submit block message appears. --- roles/jd-server/src/lib/job_declarator/mod.rs | 110 ++++-------------- 1 file changed, 24 insertions(+), 86 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index bf0976575c..c7ed4387ee 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -25,22 +25,11 @@ use stratum_common::bitcoin::{ }; #[derive(Clone, Debug)] -enum TransactionState { +pub enum TransactionState { PresentInMempool(Txid), Missing, } -pub fn get_ids_of_known_transactions(transactions: &Vec) -> Vec { - let mut known_transactions: Vec = Vec::new(); - for transaction in transactions { - match *transaction { - TransactionState::PresentInMempool(txid) => known_transactions.push(txid.clone()), - TransactionState::Missing => continue, - } - }; - known_transactions -} - #[derive(Clone, Debug)] pub struct AddTrasactionsToMempool { pub known_transactions: Vec, @@ -101,70 +90,6 @@ impl JobDeclaratorDownstream { } } - //// This only errors that are returned are PoisonLock, Custom, MempoolError - //// this function is called in JobDeclaratorDowenstream::start(), if different errors are - //// returned, change also the error management there - //async fn send_transactions_needed_in_mempool( - // self_mutex: Arc>, - //) -> Result<(), JdsError> { - // let sender_add_txs_to_mempool = self_mutex.safe_lock(|a| a.sender_add_txs_to_mempool.clone()).unwrap(); - // let mut transactions_to_be_retrieved: Vec = Vec::new(); - // let transactions_with_state = self_mutex - // .clone() - // .safe_lock(|a| a.declared_mining_job.clone()) - // .map_err(|e| JdsError::PoisonLock(e.to_string()))? - // .1; - // for tx_with_state in transactions_with_state { - // match tx_with_state { - // TransactionState::PresentInMempool(txid) => transactions_to_be_retrieved.push(txid), - // TransactionState::Missing => continue, - // } - // }; - // let _ = sender_add_txs_to_mempool.send(transactions_to_be_retrieved); - // //let mempool_ = self_mutex - // // .safe_lock(|a| a.mempool.clone()) - // // .map_err(|e| JdsError::PoisonLock(e.to_string()))?; - // //let client = mempool_ - // // .clone() - // // .safe_lock(|a| a.get_client()) - // // .map_err(|e| JdsError::PoisonLock(e.to_string()))? - // // .ok_or(JdsError::MempoolError( - // // mempool::error::JdsMempoolError::NoClient, - // // ))?; - // //for txid in transactions_to_be_retrieved { - // // let transaction = client - // // .get_raw_transaction(&txid.to_string(), None) - // // .await - // // .map_err(|e| JdsError::MempoolError(mempool::error::JdsMempoolError::Rpc(e)))?; - // // let txid = transaction.txid(); - // // mempool::JDsMempool::add_tx_data_to_mempool( - // // mempool_.clone(), - // // txid, - // // Some(transaction.clone()), - // // ); - // // new_transactions.push(transaction); - // //} - // //for transaction in new_transactions { - // // self_mutex - // // .clone() - // // .safe_lock(|a| { - // // for transaction_with_state in &mut a.declared_mining_job.1 { - // // match transaction_with_state { - // // TransactionState::Present(_) => continue, - // // TransactionState::ToBeRetrievedFromNodeMempool(_) => { - // // *transaction_with_state = - // // TransactionState::Present(transaction.txid()); - // // break; - // // } - // // TransactionState::Missing => continue, - // // } - // // } - // // }) - // // .map_err(|e| JdsError::PoisonLock(e.to_string()))? - // //} - // Ok(()) - //} - fn get_block_hex( self_mutex: Arc>, message: SubmitSolutionJd, @@ -205,6 +130,20 @@ impl JobDeclaratorDownstream { Ok(hex::encode(serialize(&block))) } + fn are_all_job_transactions_present(self_mutex:Arc>) -> Result { + let transactions_ = self_mutex.safe_lock(|a| a.declared_mining_job.1.clone()).map_err(|e| JdsError::PoisonLock(e.to_string())); + let transactions = match transactions_ { + Ok(transactions_inner) => transactions_inner, + Err(error) => return Err(error), + }; + for transaction in transactions { + if let TransactionState::Missing = transaction { + return Ok(false); + } + }; + return Ok(true); + } + pub async fn send( self_mutex: Arc>, message: roles_logic_sv2::parsers::JobDeclaration<'static>, @@ -255,6 +194,15 @@ impl JobDeclaratorDownstream { } Ok(SendTo::None(m)) => match m { Some(JobDeclaration::SubmitSolution(message)) => { + match JobDeclaratorDownstream::are_all_job_transactions_present(self_mutex.clone()) { + Ok(true_or_false) => if true_or_false { + info!("All transactions in downstream job are recognized correctly by the JD Server"); + } else { + // TODO print here the ip of the downstream + error!("Missing transactions at submit solution!"); + }, + Err(error) => handle_result!(tx_status, Err(error)), + } let hexdata = match JobDeclaratorDownstream::get_block_hex( self_mutex.clone(), message, @@ -318,16 +266,6 @@ impl JobDeclaratorDownstream { break; } } - //let retrieve_transactions = - // JobDeclaratorDownstream::retrieve_transactions_via_rpc(self_mutex.clone()) - // .await; - //match retrieve_transactions { - // Ok(_) => (), - // Err(error) => { - // handle_result!(tx_status, Err(error)); - // break; - // } - //} } }); } From 4081324d8d9fcc5b66ca54389a6ab72f06ffeeed Mon Sep 17 00:00:00 2001 From: lorban Date: Tue, 19 Mar 2024 22:28:03 +0100 Subject: [PATCH 31/38] Error management Error management when some transactions are missing. The jds should not break. Instead, tries to recover it by triggering the jds mempool to retrieve the transactions from its bitcoin node --- .../src/lib/job_declarator/message_handler.rs | 21 +- roles/jd-server/src/lib/job_declarator/mod.rs | 316 ++++++++++++------ roles/jd-server/src/lib/mempool/mod.rs | 15 +- roles/jd-server/src/main.rs | 22 +- 4 files changed, 259 insertions(+), 115 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 8e038f1818..e883692b85 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -65,8 +65,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { // mempool which use the rpc client to retrieve the whole data for each transactions. // The unknown transactions is a vector that contains the transactions that are not in the // jds mempool, and will be non-empty in the ProvideMissingTransactionsSuccess message - let mut known_transactions: Vec = Vec::new(); - let unknown_transactions: Vec = Vec::new(); + let mut known_transactions: Vec = vec![]; self.tx_hash_list_hash = Some(message.tx_hash_list_hash.clone().into_static()); if self.verify_job(&message) { let short_hash_list: Vec = message @@ -92,7 +91,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { Some(tx_data) => { transactions_with_state[i] = TransactionState::PresentInMempool(tx_data.id); known_transactions.push(tx_data.id); - }, + } None => { transactions_with_state[i] = TransactionState::Missing; missing_txs.push(i as u16); @@ -105,7 +104,11 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { missing_txs.clone(), ); // here we send the transactions that we want to be stored in jds mempool with full data - self.sender_add_txs_to_mempool.send(super::AddTrasactionsToMempool { known_transactions, unknown_transactions}); + + self.add_txs_to_mempool + .add_txs_to_mempool_inner + .known_transactions + .append(&mut known_transactions); if missing_txs.is_empty() { let message_success = DeclareMiningJobSuccess { @@ -152,13 +155,12 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { message: ProvideMissingTransactionsSuccess, ) -> Result { let (_, ref mut transactions_with_state, missing_indexes) = &mut self.declared_mining_job; - let mut unknown_transactions: Vec = Vec::new(); - let known_transactions: Vec = Vec::new(); + let mut unknown_transactions: Vec = vec![]; for (i, tx) in message.transaction_list.inner_as_ref().iter().enumerate() { let mut cursor = Cursor::new(tx); let transaction = Transaction::consensus_decode_from_finite_reader(&mut cursor) .map_err(|e| Error::TxDecodingError(e.to_string()))?; - &unknown_transactions.push(transaction.clone()); + Vec::push(&mut unknown_transactions, transaction.clone()); let index = *missing_indexes .get(i) .ok_or(Error::LogicErrorMessage(Box::new( @@ -169,7 +171,10 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { // insert the missing transactions in the mempool transactions_with_state[index] = TransactionState::PresentInMempool(transaction.txid()); } - self.sender_add_txs_to_mempool.send(super::AddTrasactionsToMempool { known_transactions, unknown_transactions: unknown_transactions.clone()}); + self.add_txs_to_mempool + .add_txs_to_mempool_inner + .unknown_transactions + .append(&mut unknown_transactions); // if there still a missing transaction return an error for tx_with_state in transactions_with_state { match tx_with_state { diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index c7ed4387ee..4437df16c7 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -16,7 +16,10 @@ use roles_logic_sv2::{ }; use secp256k1::{Keypair, Message as SecpMessage, Secp256k1}; use std::{collections::HashMap, convert::TryInto, sync::Arc}; -use tokio::net::TcpListener; +use tokio::{ + net::TcpListener, + time::Duration, +}; use tracing::{error, info}; use stratum_common::bitcoin::{ @@ -31,11 +34,18 @@ pub enum TransactionState { } #[derive(Clone, Debug)] -pub struct AddTrasactionsToMempool { +pub struct AddTrasactionsToMempoolInner { pub known_transactions: Vec, pub unknown_transactions: Vec, } +// TODO implement send method that sends the inner via the sender +#[derive(Clone, Debug)] +pub struct AddTrasactionsToMempool { + pub add_txs_to_mempool_inner: AddTrasactionsToMempoolInner, + pub sender_add_txs_to_mempool: Sender, +} + #[derive(Debug)] pub struct JobDeclaratorDownstream { sender: Sender, @@ -56,7 +66,7 @@ pub struct JobDeclaratorDownstream { Vec, ), tx_hash_list_hash: Option>, - sender_add_txs_to_mempool: Sender, + add_txs_to_mempool: AddTrasactionsToMempool, } impl JobDeclaratorDownstream { @@ -65,12 +75,16 @@ impl JobDeclaratorDownstream { sender: Sender, config: &Configuration, mempool: Arc>, - sender_add_txs_to_mempool: Sender + sender_add_txs_to_mempool: Sender, ) -> Self { let mut coinbase_output = vec![]; // TODO: use next variables let token_to_job_map = HashMap::with_hasher(BuildNoHashHasher::default()); let tokens = Id::new(); + let add_txs_to_mempool_inner = AddTrasactionsToMempoolInner { + known_transactions: vec![], + unknown_transactions: vec![], + }; super::get_coinbase_output(config).expect("Invalid coinbase output in config")[0] .consensus_encode(&mut coinbase_output) .expect("Invalid coinbase output in config"); @@ -86,7 +100,10 @@ impl JobDeclaratorDownstream { mempool, declared_mining_job: (None, Vec::new(), Vec::new()), tx_hash_list_hash: None, - sender_add_txs_to_mempool, + add_txs_to_mempool: AddTrasactionsToMempool { + add_txs_to_mempool_inner, + sender_add_txs_to_mempool, + }, } } @@ -94,54 +111,83 @@ impl JobDeclaratorDownstream { self_mutex: Arc>, message: SubmitSolutionJd, ) -> Result> { - let (last_declare_, transactions_with_state, _) = self_mutex + let (last_declare_, _, _) = self_mutex + .clone() + .safe_lock(|x| x.declared_mining_job.clone()) + .map_err(|e| Box::new(JdsError::PoisonLock(e.to_string())))?; + let last_declare = last_declare_.ok_or(Box::new(JdsError::NoLastDeclaredJob))?; + let transactions_list = Self::are_all_job_transactions_present(self_mutex)?; + let block: Block = + roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message) + .into(); + Ok(hex::encode(serialize(&block))) + } + + fn are_all_job_transactions_present( + self_mutex: Arc>, + ) -> Result, Box> { + let (_, transactions_with_state, _) = self_mutex + .clone() .safe_lock(|x| x.declared_mining_job.clone()) - .map_err(|e| JdsError::PoisonLock(e.to_string()))?; - let mempool_ = self_mutex + .map_err(|e| Box::new(JdsError::PoisonLock(e.to_string())))?; + let mempool = self_mutex .safe_lock(|x| x.mempool.clone()) - .map_err(|e| JdsError::PoisonLock(e.to_string()))?; - let last_declare = last_declare_.ok_or(JdsError::NoLastDeclaredJob)?; + .map_err(|e| Box::new(JdsError::PoisonLock(e.to_string())))?; let mut transactions_list: Vec = Vec::new(); for tx_with_state in transactions_with_state.iter().enumerate() { if let TransactionState::PresentInMempool(txid) = tx_with_state.1 { - let tx_ = match mempool_.safe_lock(|x| x.mempool.get(txid).cloned()) { - Ok(tx) => tx, - Err(e) => return Err(Box::new(JdsError::PoisonLock(e.to_string()))), - }; - let tx = tx_.ok_or(JdsError::ImpossibleToReconstructBlock( - "Missing transactions".to_string(), - )); + let tx = mempool + .safe_lock(|x| x.mempool.get(txid).cloned()) + .map_err(|e| JdsError::PoisonLock(e.to_string()))? + .ok_or(JdsError::ImpossibleToReconstructBlock( + "Txid not found in jds mempool".to_string(), + )); if let Ok(Some(tx)) = tx { transactions_list.push(tx); } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( - "Missing transactions".to_string(), + "Txid found in jds mempool but transactions not present".to_string(), ))); } } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( - "Missing transactions".to_string(), + "Unknown transaction".to_string(), ))); }; } - let block: Block = - roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message) - .into(); - Ok(hex::encode(serialize(&block))) + Ok(transactions_list) } - fn are_all_job_transactions_present(self_mutex:Arc>) -> Result { - let transactions_ = self_mutex.safe_lock(|a| a.declared_mining_job.1.clone()).map_err(|e| JdsError::PoisonLock(e.to_string())); - let transactions = match transactions_ { - Ok(transactions_inner) => transactions_inner, - Err(error) => return Err(error), - }; - for transaction in transactions { - if let TransactionState::Missing = transaction { - return Ok(false); - } - }; - return Ok(true); + async fn send_txs_to_mempool(self_mutex: Arc>) { + let add_txs_to_mempool = self_mutex + .safe_lock(|a| a.add_txs_to_mempool.clone()) + .unwrap(); + let sender_add_txs_to_mempool = add_txs_to_mempool.sender_add_txs_to_mempool; + let add_txs_to_mempool_inner = add_txs_to_mempool.add_txs_to_mempool_inner; + let _ = sender_add_txs_to_mempool + .send(add_txs_to_mempool_inner) + .await; + // the trasnactions sent to the mempool can be freed + let _ = self_mutex.safe_lock(|a| { + a.add_txs_to_mempool.add_txs_to_mempool_inner = AddTrasactionsToMempoolInner { + known_transactions: vec![], + unknown_transactions: vec![], + }; + }); + } + + fn get_transactions_in_job(self_mutex: Arc>) -> Vec { + let mut known_transactions: Vec = Vec::new(); + let job_transactions = self_mutex + .safe_lock(|a| a.declared_mining_job.1.clone()) + .unwrap(); + for transaction in job_transactions { + match transaction { + TransactionState::PresentInMempool(txid) => known_transactions.push(txid), + TransactionState::Missing => continue, + }; + } + known_transactions } pub async fn send( @@ -177,8 +223,40 @@ impl JobDeclaratorDownstream { payload, ); match next_message_to_send { - Ok(SendTo::Respond(message)) => { - Self::send(self_mutex.clone(), message).await.unwrap(); + Ok(SendTo::Respond(m)) => { + match m { + JobDeclaration::AllocateMiningJobToken(_) => { + info!("Received AMJT") + } + JobDeclaration::AllocateMiningJobTokenSuccess(_) => { + error!("Unexpected message: AMJTS") + } + JobDeclaration::DeclareMiningJob(_) => { + info!("Received DMJ"); + Self::send_txs_to_mempool(self_mutex.clone()).await; + } + JobDeclaration::DeclareMiningJobError(_) => { + error!("Unexpected message: DMJE") + } + JobDeclaration::DeclareMiningJobSuccess(_) => { + error!("Unexpected message: DMJS") + } + JobDeclaration::IdentifyTransactions(_) => { + error!("Unexpected message: IT") + } + JobDeclaration::IdentifyTransactionsSuccess(_) => { + error!("Unexpected message: ITS") + } + JobDeclaration::ProvideMissingTransactions(_) => { + error!("Unexpected message: PMT") + } + JobDeclaration::ProvideMissingTransactionsSuccess(_) => { + info!("Received PMTS"); + Self::send_txs_to_mempool(self_mutex.clone()).await; + } + JobDeclaration::SubmitSolution(_) => todo!(), + } + Self::send(self_mutex.clone(), m).await.unwrap(); } Ok(SendTo::RelayNewMessage(message)) => { error!("JD Server: unexpected relay new message {:?}", message); @@ -192,64 +270,102 @@ impl JobDeclaratorDownstream { Ok(SendTo::Multiple(multiple)) => { error!("JD Server: unexpected multiple messages: {:?}", multiple); } - Ok(SendTo::None(m)) => match m { - Some(JobDeclaration::SubmitSolution(message)) => { - match JobDeclaratorDownstream::are_all_job_transactions_present(self_mutex.clone()) { - Ok(true_or_false) => if true_or_false { - info!("All transactions in downstream job are recognized correctly by the JD Server"); - } else { - // TODO print here the ip of the downstream - error!("Missing transactions at submit solution!"); - }, - Err(error) => handle_result!(tx_status, Err(error)), + Ok(SendTo::None(m)) => { + match m { + Some(JobDeclaration::SubmitSolution(message)) => { + match Self::are_all_job_transactions_present( + self_mutex.clone(), + ) { + Ok(_) => { + info!("All transactions in downstream job are recognized correctly by the JD Server"); + let hexdata = + match JobDeclaratorDownstream::get_block_hex( + self_mutex.clone(), + message, + ) { + Ok(inner) => inner, + Err(e) => { + error!( + "Received solution but encountered error: {:?}", + e + ); + recv.close(); + //TODO should we brake it? + break; + } + }; + let _ = new_block_sender.send(hexdata).await; + } + Err(error) => { + error!("Missing transactions: {:?}", error); + // TODO print here the ip of the downstream + let known_transactions = + JobDeclaratorDownstream::get_transactions_in_job( + self_mutex.clone(), + ); + let retrieve_transactions = + AddTrasactionsToMempoolInner { + known_transactions, + unknown_transactions: Vec::new(), + }; + let mempool = self_mutex + .clone() + .safe_lock(|a| a.mempool.clone()) + .unwrap(); + tokio::select! { + _ = JDsMempool::add_tx_data_to_mempool(mempool, retrieve_transactions) => { + let hexdata = match JobDeclaratorDownstream::get_block_hex( + self_mutex.clone(), + message.clone(), + ) { + Ok(inner) => inner, + Err(e) => { + error!( + "Error retrieving transactions: {:?}", + e + ); + recv.close(); + //TODO should we brake it? + break; + } + }; + let _ = new_block_sender.send(hexdata).await; + } + _ = tokio::time::sleep(Duration::from_secs(60)) => {} + }; + } + }; } - let hexdata = match JobDeclaratorDownstream::get_block_hex( - self_mutex.clone(), - message, - ) { - Ok(inner) => inner, - Err(e) => { - error!( - "Received solution but encountered error: {:?}", - e - ); - recv.close(); - //TODO should we brake it? - break; - } - }; - - let _ = new_block_sender.send(hexdata).await; - } - Some(JobDeclaration::DeclareMiningJob(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::DeclareMiningJobSuccess(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::DeclareMiningJobError(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::IdentifyTransactions(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::IdentifyTransactionsSuccess(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::AllocateMiningJobToken(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::AllocateMiningJobTokenSuccess(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::ProvideMissingTransactions(_)) => { - error!("JD Server received an unexpected message {:?}", m); - } - Some(JobDeclaration::ProvideMissingTransactionsSuccess(_)) => { - error!("JD Server received an unexpected message {:?}", m); + Some(JobDeclaration::DeclareMiningJob(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::DeclareMiningJobSuccess(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::DeclareMiningJobError(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::IdentifyTransactions(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::IdentifyTransactionsSuccess(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::AllocateMiningJobToken(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::AllocateMiningJobTokenSuccess(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::ProvideMissingTransactions(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + Some(JobDeclaration::ProvideMissingTransactionsSuccess(_)) => { + error!("JD Server received an unexpected message {:?}", m); + } + None => (), } - None => (), - }, + } Err(e) => { error!("{:?}", e); handle_result!( @@ -303,11 +419,19 @@ impl JobDeclarator { status_tx: crate::status::Sender, mempool: Arc>, new_block_sender: Sender, - sender_add_txs_to_mempool: Sender, + sender_add_txs_to_mempool: Sender, ) { let self_ = Arc::new(Mutex::new(Self {})); info!("JD INITIALIZED"); - Self::accept_incoming_connection(self_, config, status_tx, mempool, new_block_sender, sender_add_txs_to_mempool).await; + Self::accept_incoming_connection( + self_, + config, + status_tx, + mempool, + new_block_sender, + sender_add_txs_to_mempool, + ) + .await; } async fn accept_incoming_connection( _self_: Arc>, @@ -315,7 +439,7 @@ impl JobDeclarator { status_tx: crate::status::Sender, mempool: Arc>, new_block_sender: Sender, - sender_add_txs_to_mempool: Sender, + sender_add_txs_to_mempool: Sender, ) { let listner = TcpListener::bind(&config.listen_jd_address).await.unwrap(); while let Ok((stream, _)) = listner.accept().await { diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index a7db47d651..f176fb3e1f 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -1,6 +1,6 @@ pub mod error; +use super::job_declarator::AddTrasactionsToMempoolInner; use crate::mempool::error::JdsMempoolError; -use super::job_declarator::AddTrasactionsToMempool; use async_channel::Receiver; use bitcoin::blockdata::transaction::Transaction; use hashbrown::HashMap; @@ -62,20 +62,23 @@ impl JDsMempool { // this functions fill in the mempool the transactions with the given txid and insert the given // transactions. The ids are for the transactions that are already known to the node, the // unknown transactions are provided directly as a vector - pub async fn add_tx_data_to_mempool(self_: Arc>, add_transactions_to_mempool: AddTrasactionsToMempool) -> Result<(), JdsMempoolError>{ - let txids = add_transactions_to_mempool.known_transactions; - let transactions = add_transactions_to_mempool.unknown_transactions; + pub async fn add_tx_data_to_mempool( + self_: Arc>, + add_txs_to_mempool_inner: AddTrasactionsToMempoolInner, + ) -> Result<(), JdsMempoolError> { + let txids = add_txs_to_mempool_inner.known_transactions; + let transactions = add_txs_to_mempool_inner.unknown_transactions; let client = self_ .safe_lock(|a| a.get_client()) .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? .ok_or(JdsMempoolError::NoClient)?; - // fill in the mempool the transactions id in the mempool with the full transactions + // fill in the mempool the transactions id in the mempool with the full transactions // retrieved from the jd client for txid in txids { let transaction = client .get_raw_transaction(&txid.to_string(), None) .await - .map_err(|e| JdsMempoolError::Rpc(e))?; + .map_err(JdsMempoolError::Rpc)?; let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); } diff --git a/roles/jd-server/src/main.rs b/roles/jd-server/src/main.rs index 6d0530c35a..3391a93be8 100644 --- a/roles/jd-server/src/main.rs +++ b/roles/jd-server/src/main.rs @@ -191,22 +191,34 @@ async fn main() { let cloned = config.clone(); let mempool_cloned = mempool.clone(); - let (sender_jdd_for_mempool_update, receiver_jdd_for_mempool_update) = unbounded(); + let (sender_add_txs_to_mempool, receiver_add_txs_to_mempool) = unbounded(); task::spawn(async move { - JobDeclarator::start(cloned, sender, mempool_cloned, new_block_sender, sender_jdd_for_mempool_update).await + JobDeclarator::start( + cloned, + sender, + mempool_cloned, + new_block_sender, + sender_add_txs_to_mempool, + ) + .await }); task::spawn(async move { loop { - if let Ok(add_transactions_to_mempool) = receiver_jdd_for_mempool_update.recv().await { + if let Ok(add_transactions_to_mempool) = receiver_add_txs_to_mempool.recv().await { let mempool_cloned = mempool.clone(); task::spawn(async move { - match lib::mempool::JDsMempool::add_tx_data_to_mempool(mempool_cloned, add_transactions_to_mempool).await { + match lib::mempool::JDsMempool::add_tx_data_to_mempool( + mempool_cloned, + add_transactions_to_mempool, + ) + .await + { Ok(_) => (), Err(err) => { // TODO // here there should be a better error management mempool::error::handle_error(&err); - }, + } } }); } From e577d7a745b6d5fbc26c02831912dff670054e1e Mon Sep 17 00:00:00 2001 From: lorban Date: Thu, 21 Mar 2024 09:37:36 +0100 Subject: [PATCH 32/38] Fix bug in SendTo output from message_handler In the message handler the message the triggers the JDS mempool to fill the transactions is implemented in a bad way. Now it is fixed Add some documentation --- .../src/lib/job_declarator/message_handler.rs | 2 +- roles/jd-server/src/lib/job_declarator/mod.rs | 35 +++++++++++++------ 2 files changed, 25 insertions(+), 12 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index e883692b85..8d0f04011d 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -130,7 +130,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { JobDeclaration::ProvideMissingTransactions( message_provide_missing_transactions, ); - Ok(SendTo_::Respond(message_enum_provide_missing_transactions)) + Ok(SendTo::Respond(message_enum_provide_missing_transactions)) } } else { let message_error = DeclareMiningJobError { diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 4437df16c7..5e7ec6178e 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -222,37 +222,50 @@ impl JobDeclaratorDownstream { message_type, payload, ); + // How works the txs recognition and txs storing in JDS mempool + // when a DMJ arrives, the JDS compares the received transactions with the + // ids in the the JDS mempool. Then there are two scenarios + // 1. the JDS recognizes all the transactions. Then, just before a DMJS is + // sent, the JDS mempool is triggered to fill in the JDS mempool the id + // of declared job with the full transaction (with send_tx_to_mempool + // method(), that eventually will ask the transactions to a bitcoin node + // via RPC) + // 2. there are some unknown txids. Just before sending PMT, the JDS + // mempool is triggered to fill the known txids with the full + // transactions. When a PMTS arrives, just before sending a DMJS, the + // unknown full transactions provided by the downstream are added to the + // JDS mempool match next_message_to_send { Ok(SendTo::Respond(m)) => { match m { JobDeclaration::AllocateMiningJobToken(_) => { - info!("Received AMJT") + error!("Send unexpected message: AMJT") } JobDeclaration::AllocateMiningJobTokenSuccess(_) => { - error!("Unexpected message: AMJTS") + info!("Send message: AMJTS") } JobDeclaration::DeclareMiningJob(_) => { - info!("Received DMJ"); - Self::send_txs_to_mempool(self_mutex.clone()).await; + error!("Send unexpected message: DMJ"); } JobDeclaration::DeclareMiningJobError(_) => { - error!("Unexpected message: DMJE") + info!("Send nmessage: DMJE") } JobDeclaration::DeclareMiningJobSuccess(_) => { - error!("Unexpected message: DMJS") + info!("Send message: DMJS. Updating the JDS mempool."); + Self::send_txs_to_mempool(self_mutex.clone()).await; } JobDeclaration::IdentifyTransactions(_) => { - error!("Unexpected message: IT") + info!("Send message: IT") } JobDeclaration::IdentifyTransactionsSuccess(_) => { - error!("Unexpected message: ITS") + error!("Send unexpected message: ITS") } JobDeclaration::ProvideMissingTransactions(_) => { - error!("Unexpected message: PMT") + info!("Send message: PMT. Updating the JDS mempool."); + Self::send_txs_to_mempool(self_mutex.clone()).await; } JobDeclaration::ProvideMissingTransactionsSuccess(_) => { - info!("Received PMTS"); - Self::send_txs_to_mempool(self_mutex.clone()).await; + error!("Send unexpected PMTS"); } JobDeclaration::SubmitSolution(_) => todo!(), } From c6647cde762af187cc3c1e61c1f7543fe9657119 Mon Sep 17 00:00:00 2001 From: lorban Date: Thu, 21 Mar 2024 10:03:48 +0100 Subject: [PATCH 33/38] Check if a tx is already present in mempool apply Fi3 suggestion https://github.com/GitGab19/stratum/pull/16#discussion_r1532513707 --- .../src/lib/job_declarator/message_handler.rs | 2 +- roles/jd-server/src/lib/mempool/mod.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/message_handler.rs b/roles/jd-server/src/lib/job_declarator/message_handler.rs index 8d0f04011d..58a40fcba0 100644 --- a/roles/jd-server/src/lib/job_declarator/message_handler.rs +++ b/roles/jd-server/src/lib/job_declarator/message_handler.rs @@ -62,7 +62,7 @@ impl ParseClientJobDeclarationMessages for JobDeclaratorDownstream { fn handle_declare_mining_job(&mut self, message: DeclareMiningJob) -> Result { // the transactions that are present in the mempool are stored here, that is sent to the - // mempool which use the rpc client to retrieve the whole data for each transactions. + // mempool which use the rpc client to retrieve the whole data for each transaction. // The unknown transactions is a vector that contains the transactions that are not in the // jds mempool, and will be non-empty in the ProvideMissingTransactionsSuccess message let mut known_transactions: Vec = vec![]; diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index f176fb3e1f..d6a7bbd87c 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -75,11 +75,13 @@ impl JDsMempool { // fill in the mempool the transactions id in the mempool with the full transactions // retrieved from the jd client for txid in txids { - let transaction = client - .get_raw_transaction(&txid.to_string(), None) - .await - .map_err(JdsMempoolError::Rpc)?; - let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); + if let None = self_.safe_lock(|a| a.mempool.get(&txid).cloned()).map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? { + let transaction = client + .get_raw_transaction(&txid.to_string(), None) + .await + .map_err(JdsMempoolError::Rpc)?; + let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); + } } // fill in the mempool the transactions given in input From e96761782401700d4506ad37efa4b03d468f7c4c Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 21 Mar 2024 19:08:53 +0100 Subject: [PATCH 34/38] fix in add_tx_data_to_mempool --- roles/jd-server/src/lib/job_declarator/mod.rs | 6 +++--- roles/jd-server/src/lib/mempool/mod.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 5e7ec6178e..5da6010b84 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -141,8 +141,8 @@ impl JobDeclaratorDownstream { .map_err(|e| JdsError::PoisonLock(e.to_string()))? .ok_or(JdsError::ImpossibleToReconstructBlock( "Txid not found in jds mempool".to_string(), - )); - if let Ok(Some(tx)) = tx { + ))?; + if let Some(tx) = tx { transactions_list.push(tx); } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( @@ -505,4 +505,4 @@ impl JobDeclarator { } } } -} +} \ No newline at end of file diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index d6a7bbd87c..a08ac07693 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -75,7 +75,7 @@ impl JDsMempool { // fill in the mempool the transactions id in the mempool with the full transactions // retrieved from the jd client for txid in txids { - if let None = self_.safe_lock(|a| a.mempool.get(&txid).cloned()).map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? { + if let Some(None) = self_.safe_lock(|a| a.mempool.get(&txid).cloned()).map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? { let transaction = client .get_raw_transaction(&txid.to_string(), None) .await @@ -170,4 +170,4 @@ impl JDsMempool { } Some(ret) } -} +} \ No newline at end of file From 76aa4bdaeb7159c0b6a6a8aa5f50af05aff29c4b Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 21 Mar 2024 19:09:11 +0100 Subject: [PATCH 35/38] fmt --- roles/jd-server/src/lib/job_declarator/mod.rs | 9 +++------ roles/jd-server/src/lib/mempool/mod.rs | 10 +++++++--- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 5da6010b84..9c807415da 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -16,10 +16,7 @@ use roles_logic_sv2::{ }; use secp256k1::{Keypair, Message as SecpMessage, Secp256k1}; use std::{collections::HashMap, convert::TryInto, sync::Arc}; -use tokio::{ - net::TcpListener, - time::Duration, -}; +use tokio::{net::TcpListener, time::Duration}; use tracing::{error, info}; use stratum_common::bitcoin::{ @@ -222,7 +219,7 @@ impl JobDeclaratorDownstream { message_type, payload, ); - // How works the txs recognition and txs storing in JDS mempool + // How works the txs recognition and txs storing in JDS mempool // when a DMJ arrives, the JDS compares the received transactions with the // ids in the the JDS mempool. Then there are two scenarios // 1. the JDS recognizes all the transactions. Then, just before a DMJS is @@ -505,4 +502,4 @@ impl JobDeclarator { } } } -} \ No newline at end of file +} diff --git a/roles/jd-server/src/lib/mempool/mod.rs b/roles/jd-server/src/lib/mempool/mod.rs index a08ac07693..43ffc7055f 100644 --- a/roles/jd-server/src/lib/mempool/mod.rs +++ b/roles/jd-server/src/lib/mempool/mod.rs @@ -75,12 +75,16 @@ impl JDsMempool { // fill in the mempool the transactions id in the mempool with the full transactions // retrieved from the jd client for txid in txids { - if let Some(None) = self_.safe_lock(|a| a.mempool.get(&txid).cloned()).map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? { + if let Some(None) = self_ + .safe_lock(|a| a.mempool.get(&txid).cloned()) + .map_err(|e| JdsMempoolError::PoisonLock(e.to_string()))? + { let transaction = client .get_raw_transaction(&txid.to_string(), None) .await .map_err(JdsMempoolError::Rpc)?; - let _ = self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); + let _ = + self_.safe_lock(|a| a.mempool.insert(transaction.txid(), Some(transaction))); } } @@ -170,4 +174,4 @@ impl JDsMempool { } Some(ret) } -} \ No newline at end of file +} From 928d4758c5f676f5af8126d0af548cce6b582331 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 21 Mar 2024 19:20:40 +0100 Subject: [PATCH 36/38] debug logging --- roles/jd-server/src/lib/job_declarator/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 9c807415da..eda03b458c 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -17,7 +17,7 @@ use roles_logic_sv2::{ use secp256k1::{Keypair, Message as SecpMessage, Secp256k1}; use std::{collections::HashMap, convert::TryInto, sync::Arc}; use tokio::{net::TcpListener, time::Duration}; -use tracing::{error, info}; +use tracing::{error, info, debug}; use stratum_common::bitcoin::{ consensus::{encode::serialize, Encodable}, @@ -239,26 +239,26 @@ impl JobDeclaratorDownstream { error!("Send unexpected message: AMJT") } JobDeclaration::AllocateMiningJobTokenSuccess(_) => { - info!("Send message: AMJTS") + debug!("Send message: AMJTS") } JobDeclaration::DeclareMiningJob(_) => { error!("Send unexpected message: DMJ"); } JobDeclaration::DeclareMiningJobError(_) => { - info!("Send nmessage: DMJE") + debug!("Send nmessage: DMJE") } JobDeclaration::DeclareMiningJobSuccess(_) => { - info!("Send message: DMJS. Updating the JDS mempool."); + debug!("Send message: DMJS. Updating the JDS mempool."); Self::send_txs_to_mempool(self_mutex.clone()).await; } JobDeclaration::IdentifyTransactions(_) => { - info!("Send message: IT") + debug!("Send message: IT") } JobDeclaration::IdentifyTransactionsSuccess(_) => { error!("Send unexpected message: ITS") } JobDeclaration::ProvideMissingTransactions(_) => { - info!("Send message: PMT. Updating the JDS mempool."); + debug!("Send message: PMT. Updating the JDS mempool."); Self::send_txs_to_mempool(self_mutex.clone()).await; } JobDeclaration::ProvideMissingTransactionsSuccess(_) => { From 3793b141108052c438a2b7909a8d0fea1705657c Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 21 Mar 2024 19:25:56 +0100 Subject: [PATCH 37/38] fmt --- roles/jd-server/src/lib/job_declarator/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index eda03b458c..509a435571 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -17,7 +17,7 @@ use roles_logic_sv2::{ use secp256k1::{Keypair, Message as SecpMessage, Secp256k1}; use std::{collections::HashMap, convert::TryInto, sync::Arc}; use tokio::{net::TcpListener, time::Duration}; -use tracing::{error, info, debug}; +use tracing::{debug, error, info}; use stratum_common::bitcoin::{ consensus::{encode::serialize, Encodable}, From 909f86524f7187eda483c0788725002711ca2227 Mon Sep 17 00:00:00 2001 From: GitGab19 Date: Thu, 21 Mar 2024 21:18:00 +0100 Subject: [PATCH 38/38] renaming fn from are_all_job_transactions_present to collect_txs_in_job --- roles/jd-server/src/lib/job_declarator/mod.rs | 23 +++++++------------ 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/roles/jd-server/src/lib/job_declarator/mod.rs b/roles/jd-server/src/lib/job_declarator/mod.rs index 509a435571..b47b7d6a80 100644 --- a/roles/jd-server/src/lib/job_declarator/mod.rs +++ b/roles/jd-server/src/lib/job_declarator/mod.rs @@ -113,16 +113,14 @@ impl JobDeclaratorDownstream { .safe_lock(|x| x.declared_mining_job.clone()) .map_err(|e| Box::new(JdsError::PoisonLock(e.to_string())))?; let last_declare = last_declare_.ok_or(Box::new(JdsError::NoLastDeclaredJob))?; - let transactions_list = Self::are_all_job_transactions_present(self_mutex)?; + let transactions_list = Self::collect_txs_in_job(self_mutex)?; let block: Block = roles_logic_sv2::utils::BlockCreator::new(last_declare, transactions_list, message) .into(); Ok(hex::encode(serialize(&block))) } - fn are_all_job_transactions_present( - self_mutex: Arc>, - ) -> Result, Box> { + fn collect_txs_in_job(self_mutex: Arc>) -> Result, Box> { let (_, transactions_with_state, _) = self_mutex .clone() .safe_lock(|x| x.declared_mining_job.clone()) @@ -136,16 +134,13 @@ impl JobDeclaratorDownstream { let tx = mempool .safe_lock(|x| x.mempool.get(txid).cloned()) .map_err(|e| JdsError::PoisonLock(e.to_string()))? - .ok_or(JdsError::ImpossibleToReconstructBlock( + .ok_or(Box::new(JdsError::ImpossibleToReconstructBlock( "Txid not found in jds mempool".to_string(), - ))?; - if let Some(tx) = tx { - transactions_list.push(tx); - } else { - return Err(Box::new(JdsError::ImpossibleToReconstructBlock( + )))? + .ok_or(Box::new(JdsError::ImpossibleToReconstructBlock( "Txid found in jds mempool but transactions not present".to_string(), - ))); - } + )))?; + transactions_list.push(tx); } else { return Err(Box::new(JdsError::ImpossibleToReconstructBlock( "Unknown transaction".to_string(), @@ -283,9 +278,7 @@ impl JobDeclaratorDownstream { Ok(SendTo::None(m)) => { match m { Some(JobDeclaration::SubmitSolution(message)) => { - match Self::are_all_job_transactions_present( - self_mutex.clone(), - ) { + match Self::collect_txs_in_job(self_mutex.clone()) { Ok(_) => { info!("All transactions in downstream job are recognized correctly by the JD Server"); let hexdata =