From ecfd738e2c610bb32ca826ac3ece00c50957f428 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=B3=CE=BB?= Date: Tue, 15 Oct 2024 19:19:25 +1100 Subject: [PATCH] Add logging (#143) * Add logging * Replace println with tracing * Fix formatting * Fix code rabbit issues * Fix error handling over channel --- packages/ciphernode/Cargo.lock | 9 ++ packages/ciphernode/aggregator/Cargo.toml | 1 + .../aggregator/src/plaintext_aggregator.rs | 7 +- .../aggregator/src/publickey_aggregator.rs | 8 +- packages/ciphernode/core/src/events.rs | 124 ++++++++++++++++-- packages/ciphernode/enclave/Cargo.toml | 2 + .../ciphernode/enclave/src/bin/aggregator.rs | 4 +- packages/ciphernode/enclave/src/main.rs | 4 +- packages/ciphernode/enclave_node/src/main.rs | 68 ---------- packages/ciphernode/evm/Cargo.toml | 1 + .../evm/src/ciphernode_registry_sol.rs | 14 +- .../ciphernode/evm/src/enclave_sol_reader.rs | 14 +- .../ciphernode/evm/src/enclave_sol_writer.rs | 3 +- packages/ciphernode/evm/src/helpers.rs | 4 + .../ciphernode/evm/src/registry_filter_sol.rs | 3 +- packages/ciphernode/keyshare/src/keyshare.rs | 2 - packages/ciphernode/logger/Cargo.toml | 1 + packages/ciphernode/logger/src/logger.rs | 37 ++---- packages/ciphernode/p2p/src/libp2p_router.rs | 32 ++--- packages/ciphernode/p2p/src/p2p.rs | 16 ++- packages/ciphernode/router/Cargo.toml | 2 + .../router/src/ciphernode_selector.rs | 4 +- packages/ciphernode/router/src/hooks.rs | 11 +- packages/ciphernode/sortition/Cargo.toml | 1 + .../ciphernode/sortition/src/sortition.rs | 43 +++--- packages/ciphernode/test_helpers/Cargo.toml | 1 + .../test_helpers/src/plaintext_writer.rs | 3 +- .../test_helpers/src/public_key_writer.rs | 3 +- packages/ciphernode/test_helpers/src/utils.rs | 13 +- .../tests/test_aggregation_and_decryption.rs | 1 - tests/basic_integration/test.sh | 1 + 31 files changed, 256 insertions(+), 181 deletions(-) delete mode 100644 packages/ciphernode/enclave_node/src/main.rs diff --git a/packages/ciphernode/Cargo.lock b/packages/ciphernode/Cargo.lock index d7492d72..7d1089e4 100644 --- a/packages/ciphernode/Cargo.lock +++ b/packages/ciphernode/Cargo.lock @@ -119,6 +119,7 @@ dependencies = [ "enclave-core", "fhe 0.1.0", "sortition", + "tracing", ] [[package]] @@ -2086,6 +2087,8 @@ dependencies = [ "config", "enclave_node", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -2217,6 +2220,7 @@ dependencies = [ "futures-util", "sortition", "tokio", + "tracing", ] [[package]] @@ -3811,6 +3815,7 @@ dependencies = [ "actix", "base64 0.22.1", "enclave-core", + "tracing", ] [[package]] @@ -5119,11 +5124,13 @@ version = "0.1.0" dependencies = [ "actix", "aggregator", + "anyhow", "data", "enclave-core", "fhe 0.1.0", "keyshare", "sortition", + "tracing", ] [[package]] @@ -5614,6 +5621,7 @@ dependencies = [ "enclave-core", "num", "rand", + "tracing", ] [[package]] @@ -5803,6 +5811,7 @@ dependencies = [ "fhe-traits", "rand", "rand_chacha", + "tracing", ] [[package]] diff --git a/packages/ciphernode/aggregator/Cargo.toml b/packages/ciphernode/aggregator/Cargo.toml index 695e6bf6..55a76bb9 100644 --- a/packages/ciphernode/aggregator/Cargo.toml +++ b/packages/ciphernode/aggregator/Cargo.toml @@ -10,3 +10,4 @@ bincode = { workspace = true } enclave-core = { path = "../core" } fhe = { path = "../fhe" } sortition = { path = "../sortition" } +tracing = { workspace = true } diff --git a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs index c63a4dda..eef3a010 100644 --- a/packages/ciphernode/aggregator/src/plaintext_aggregator.rs +++ b/packages/ciphernode/aggregator/src/plaintext_aggregator.rs @@ -7,6 +7,7 @@ use enclave_core::{ use fhe::{Fhe, GetAggregatePlaintext}; use sortition::{GetHasNode, Sortition}; use std::sync::Arc; +use tracing::error; #[derive(Debug, Clone)] pub enum PlaintextAggregatorState { @@ -124,7 +125,7 @@ impl Handler for PlaintextAggregator { threshold_m, seed, .. } = self.state else { - println!("Aggregator has been closed for collecting."); + error!(state=?self.state, "Aggregator has been closed for collecting."); return Box::pin(fut::ready(Ok(()))); }; @@ -144,12 +145,12 @@ impl Handler for PlaintextAggregator { .map(move |res, act, ctx| { let has_node = res?; if !has_node { - println!("Node not found in committee"); // TODO: log properly + error!("Node not found in committee"); return Ok(()); } if e3_id != act.e3_id { - println!("Wrong e3_id sent to aggregator. This should not happen."); + error!("Wrong e3_id sent to aggregator. This should not happen."); return Ok(()); } diff --git a/packages/ciphernode/aggregator/src/publickey_aggregator.rs b/packages/ciphernode/aggregator/src/publickey_aggregator.rs index 7274dafe..37fbdd8f 100644 --- a/packages/ciphernode/aggregator/src/publickey_aggregator.rs +++ b/packages/ciphernode/aggregator/src/publickey_aggregator.rs @@ -6,6 +6,7 @@ use enclave_core::{ use fhe::{Fhe, GetAggregatePublicKey}; use sortition::{GetHasNode, GetNodes, Sortition}; use std::sync::Arc; +use tracing::error; #[derive(Debug, Clone)] pub enum PublicKeyAggregatorState { @@ -132,8 +133,7 @@ impl Handler for PublicKeyAggregator { threshold_m, seed, .. } = self.state.clone() else { - println!("Aggregator has been closed for collecting keyshares."); // TODO: log properly - + error!(state=?self.state, "Aggregator has been closed for collecting keyshares."); return Box::pin(fut::ready(Ok(()))); }; @@ -155,12 +155,12 @@ impl Handler for PublicKeyAggregator { // we will not be doing a send let has_node = res?; if !has_node { - println!("Node not found in committee"); // TODO: log properly + error!("Node not found in committee"); return Ok(()); } if e3_id != act.e3_id { - println!("Wrong e3_id sent to aggregator. This should not happen."); + error!("Wrong e3_id sent to aggregator. This should not happen."); return Ok(()); } diff --git a/packages/ciphernode/core/src/events.rs b/packages/ciphernode/core/src/events.rs index b50b3df6..527ad643 100644 --- a/packages/ciphernode/core/src/events.rs +++ b/packages/ciphernode/core/src/events.rs @@ -1,4 +1,4 @@ -use actix::{Actor, Addr, Message}; +use actix::Message; use alloy::{ hex, primitives::{Uint, U256}, @@ -69,7 +69,7 @@ impl EventId { impl fmt::Display for EventId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let base58_string = bs58::encode(&self.0).into_string(); - write!(f, "eid_{}", base58_string) + write!(f, "evt:{}", &base58_string[0..8]) } } @@ -183,6 +183,22 @@ impl EnclaveEvent { _ => None, } } + pub fn get_data(&self) -> String { + match self.clone() { + EnclaveEvent::KeyshareCreated { data, .. } => format!("{}", data), + EnclaveEvent::E3Requested { data, .. } => format!("{}", data), + EnclaveEvent::PublicKeyAggregated { data, .. } => format!("{}", data), + EnclaveEvent::CiphertextOutputPublished { data, .. } => format!("{}", data), + EnclaveEvent::DecryptionshareCreated { data, .. } => format!("{}", data), + EnclaveEvent::PlaintextAggregated { data, .. } => format!("{}", data), + EnclaveEvent::CiphernodeSelected { data, .. } => format!("{}", data), + EnclaveEvent::CiphernodeAdded { data, .. } => format!("{}", data), + EnclaveEvent::CiphernodeRemoved { data, .. } => format!("{}", data), + EnclaveEvent::E3RequestComplete { data, .. } => format!("{}", data), + EnclaveEvent::EnclaveError { data, .. } => format!("{:?}", data), + // _ => "".to_string(), + } + } } pub trait FromError { @@ -299,7 +315,7 @@ impl FromError for EnclaveEvent { impl fmt::Display for EnclaveEvent { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(&format!("{}({})", self.event_type(), self.get_id())) + f.write_str(&format!("{}({})", self.event_type(), self.get_data())) } } @@ -311,6 +327,12 @@ pub struct KeyshareCreated { pub node: String, } +impl Display for KeyshareCreated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "e3_id: {}, node: {}", self.e3_id, self.node,) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "anyhow::Result<()>")] pub struct DecryptionshareCreated { @@ -319,6 +341,12 @@ pub struct DecryptionshareCreated { pub node: String, } +impl Display for DecryptionshareCreated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "e3_id: {}, node: {}", self.e3_id, self.node,) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct PublicKeyAggregated { @@ -328,6 +356,16 @@ pub struct PublicKeyAggregated { pub src_chain_id: u64, } +impl Display for PublicKeyAggregated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "e3_id: {}, src_chain_id: {}, nodes: , pubkey: ", + self.e3_id, self.src_chain_id, + ) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct E3Requested { @@ -335,11 +373,17 @@ pub struct E3Requested { pub threshold_m: usize, pub seed: Seed, pub params: Vec, - pub src_chain_id: u64, // threshold: usize, // TODO: - // computation_type: ??, // TODO: - // execution_model_type: ??, // TODO: - // input_deadline: ??, // TODO: - // availability_duration: ??, // TODO: + pub src_chain_id: u64, +} + +impl Display for E3Requested { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "e3_id: {}, threshold_m: {}, src_chain_id: {}, seed: {}, params: ", + self.e3_id, self.threshold_m, self.src_chain_id, self.seed + ) + } } #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -349,6 +393,16 @@ pub struct CiphernodeSelected { pub threshold_m: usize, } +impl Display for CiphernodeSelected { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "e3_id: {}, threshold_m: {}", + self.e3_id, self.threshold_m, + ) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct CiphertextOutputPublished { @@ -356,6 +410,12 @@ pub struct CiphertextOutputPublished { pub ciphertext_output: Vec, } +impl Display for CiphertextOutputPublished { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "e3_id: {}", self.e3_id,) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct PlaintextAggregated { @@ -364,6 +424,16 @@ pub struct PlaintextAggregated { pub src_chain_id: u64, } +impl Display for PlaintextAggregated { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "e3_id: {}, src_chain_id: {}", + self.e3_id, self.src_chain_id + ) + } +} + /// E3RequestComplete event is a local only event #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] @@ -371,6 +441,12 @@ pub struct E3RequestComplete { pub e3_id: E3id, } +impl Display for E3RequestComplete { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "e3_id: {}", self.e3_id) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct CiphernodeAdded { @@ -379,6 +455,16 @@ pub struct CiphernodeAdded { pub num_nodes: usize, } +impl Display for CiphernodeAdded { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "address: {}, index: {}, num_nodes: {}", + self.address, self.index, self.num_nodes + ) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct CiphernodeRemoved { @@ -387,6 +473,16 @@ pub struct CiphernodeRemoved { pub num_nodes: usize, } +impl Display for CiphernodeRemoved { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + f, + "address: {}, index: {}, num_nodes: {}", + self.address, self.index, self.num_nodes + ) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct EnclaveError { @@ -394,9 +490,20 @@ pub struct EnclaveError { pub message: String, } +impl Display for EnclaveError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "{:?}", self) + } +} + #[derive(Message, Clone, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] #[rtype(result = "()")] pub struct Die; +impl Display for Die { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + write!(f, "Die",) + } +} #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct Seed(pub [u8; 32]); @@ -432,6 +539,7 @@ pub enum EnclaveErrorType { IO, PlaintextAggregation, Decryption, + Sortition, } impl EnclaveError { diff --git a/packages/ciphernode/enclave/Cargo.toml b/packages/ciphernode/enclave/Cargo.toml index 992aadf7..064b2e7e 100644 --- a/packages/ciphernode/enclave/Cargo.toml +++ b/packages/ciphernode/enclave/Cargo.toml @@ -14,3 +14,5 @@ clap = { workspace = true } actix-rt = { workspace = true } tokio = { workspace = true } config = "0.14.0" +tracing-subscriber = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/enclave/src/bin/aggregator.rs b/packages/ciphernode/enclave/src/bin/aggregator.rs index cad49a8e..4d628b36 100644 --- a/packages/ciphernode/enclave/src/bin/aggregator.rs +++ b/packages/ciphernode/enclave/src/bin/aggregator.rs @@ -1,6 +1,7 @@ use clap::Parser; use enclave::load_config; use enclave_node::MainAggregator; +use tracing::info; #[derive(Parser, Debug)] #[command(author, version, about, long_about = None)] @@ -18,8 +19,9 @@ struct Args { #[actix_rt::main] async fn main() -> Result<(), Box> { + tracing_subscriber::fmt::init(); let args = Args::parse(); - println!("LAUNCHING AGGREGATOR"); + info!("LAUNCHING AGGREGATOR"); let config = load_config(&args.config)?; let (_, handle) = MainAggregator::attach( config, diff --git a/packages/ciphernode/enclave/src/main.rs b/packages/ciphernode/enclave/src/main.rs index 1c67b053..39bb2105 100644 --- a/packages/ciphernode/enclave/src/main.rs +++ b/packages/ciphernode/enclave/src/main.rs @@ -2,6 +2,7 @@ use alloy::primitives::Address; use clap::Parser; use enclave::load_config; use enclave_node::MainCiphernode; +use tracing::info; const OWO: &str = r#" ___ ___ ___ ___ ___ @@ -29,11 +30,12 @@ pub struct Args { #[actix_rt::main] async fn main() -> Result<(), Box> { + tracing_subscriber::fmt::init(); println!("\n\n\n\n\n{}", OWO); println!("\n\n\n\n"); let args = Args::parse(); let address = Address::parse_checksummed(&args.address, None).expect("Invalid address"); - println!("LAUNCHING CIPHERNODE: ({})", address); + info!("LAUNCHING CIPHERNODE: ({})", address); let config = load_config(&args.config)?; let (_, handle) = MainCiphernode::attach(config, address).await?; let _ = tokio::join!(handle); diff --git a/packages/ciphernode/enclave_node/src/main.rs b/packages/ciphernode/enclave_node/src/main.rs deleted file mode 100644 index 6ca69372..00000000 --- a/packages/ciphernode/enclave_node/src/main.rs +++ /dev/null @@ -1,68 +0,0 @@ -use std::error::Error; - -use alloy::primitives::address; -use bfv::EnclaveBFV; -use p2p::EnclaveRouter; -use sortition::DistanceSortition; -use tokio::{ - self, - io::{self, AsyncBufReadExt, BufReader}, -}; - -const OWO: &str = r#" - ___ ___ ___ ___ ___ - /\__\ /\ \ /\__\ /\ \ ___ /\__\ - /:/ _/_ \:\ \ /:/ / /::\ \ /\ \ /:/ _/_ - /:/ /\__\ \:\ \ /:/ / /:/\:\ \ \:\ \ /:/ /\__\ - /:/ /:/ _/_ _____\:\ \ /:/ / ___ ___ ___ /:/ /::\ \ \:\ \ /:/ /:/ _/_ - /:/_/:/ /\__\ /::::::::\__\ /:/__/ /\__\ /\ \ /\__\ /:/_/:/\:\__\ ___ \:\__\ /:/_/:/ /\__\ - \:\/:/ /:/ / \:\~~\~~\/__/ \:\ \ /:/ / \:\ \ /:/ / \:\/:/ \/__/ /\ \ |:| | \:\/:/ /:/ / - \::/_/:/ / \:\ \ \:\ /:/ / \:\ /:/ / \::/__/ \:\ \|:| | \::/_/:/ / - \:\/:/ / \:\ \ \:\/:/ / \:\/:/ / \:\ \ \:\__|:|__| \:\/:/ / - \::/ / \:\__\ \::/ / \::/ / \:\__\ \::::/__/ \::/ / - \/__/ \/__/ \/__/ \/__/ \/__/ ~~~~ \/__/ - -"#; - -#[tokio::main] -async fn main() -> Result<(), Box> { - // boot up p2p network - - // boot up ether client - - // start main loop - - //let ether = eth::EtherClient::new("test".to_string()); - println!("\n\n\n\n\n{}", OWO); - println!("\n\n\n\n"); - println!("Hello, cipher world!"); - - let mut committee = DistanceSortition::new( - 12, - vec![address!("d8da6bf26964af9d7eed9e03e53415d37aa96045")], - 10, - ); - committee.get_committee(); - - let mut new_bfv = EnclaveBFV::new(4096, 4096, vec![0xffffee001, 0xffffc4001, 0x1ffffe0001]); - let pk_bytes = new_bfv.serialize_pk(); - let param_bytes = new_bfv.serialize_params(); - let crp_bytes = new_bfv.serialize_crp(); - let deserialized_pk = new_bfv.deserialize_pk(pk_bytes, param_bytes, crp_bytes); - - let (mut p2p, tx, mut rx) = EnclaveRouter::new()?; - p2p.connect_swarm("mdns".to_string())?; - p2p.join_topic("enclave-keygen-01")?; - let mut stdin = BufReader::new(io::stdin()).lines(); - tokio::spawn(async move { p2p.start().await }); - tokio::spawn(async move { - while let Some(msg) = rx.recv().await { - println!("msg: {}", String::from_utf8(msg).unwrap()); - } - }); - loop { - if let Ok(Some(line)) = stdin.next_line().await { - tx.send(line.as_bytes().to_vec().clone()).await.unwrap(); - } - } -} diff --git a/packages/ciphernode/evm/Cargo.toml b/packages/ciphernode/evm/Cargo.toml index 05c53930..8e0164b0 100644 --- a/packages/ciphernode/evm/Cargo.toml +++ b/packages/ciphernode/evm/Cargo.toml @@ -12,3 +12,4 @@ enclave-core = { path = "../core" } futures-util = { workspace = true } sortition = { path = "../sortition" } tokio = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs index fa90c23b..033c41c6 100644 --- a/packages/ciphernode/evm/src/ciphernode_registry_sol.rs +++ b/packages/ciphernode/evm/src/ciphernode_registry_sol.rs @@ -8,6 +8,7 @@ use alloy::{ }; use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus}; +use tracing::{error, info, trace}; use crate::helpers::{self, create_readonly_provider, ReadonlyProvider}; @@ -69,7 +70,7 @@ fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option { let Ok(event) = ICiphernodeRegistry::CiphernodeAdded::decode_log_data(data, true) else { - println!("Error parsing event CiphernodeAdded"); // TODO: provide more info + error!("Error parsing event CiphernodeAdded after topic was matched!"); return None; }; Some(EnclaveEvent::from(event)) @@ -77,14 +78,17 @@ fn extractor(data: &LogData, topic: Option<&B256>, _: u64) -> Option { let Ok(event) = ICiphernodeRegistry::CiphernodeRemoved::decode_log_data(data, true) else { - println!("Error parsing event CiphernodeRemoved"); // TODO: provide more info + error!("Error parsing event CiphernodeRemoved after topic was matched!"); return None; }; Some(EnclaveEvent::from(event)) } - _ => { - println!("Unknown event"); + _topic => { + trace!( + topic=?_topic, + "Unknown event was received by Enclave.sol parser buut was ignored" + ); return None; } } @@ -122,7 +126,7 @@ impl CiphernodeRegistrySolReader { .await? .start(); - println!("CiphernodeRegistrySol is listening to {}", contract_address); + info!(address=%contract_address, "CiphernodeRegistrySol is listening to address"); Ok(addr) } } diff --git a/packages/ciphernode/evm/src/enclave_sol_reader.rs b/packages/ciphernode/evm/src/enclave_sol_reader.rs index 5fc561f5..7b29f6e3 100644 --- a/packages/ciphernode/evm/src/enclave_sol_reader.rs +++ b/packages/ciphernode/evm/src/enclave_sol_reader.rs @@ -7,6 +7,7 @@ use alloy::{ }; use anyhow::Result; use enclave_core::{EnclaveEvent, EventBus}; +use tracing::{error, info, trace}; sol!( #[sol(rpc)] @@ -55,21 +56,24 @@ fn extractor(data: &LogData, topic: Option<&B256>, chain_id: u64) -> Option { let Ok(event) = IEnclave::E3Requested::decode_log_data(data, true) else { - println!("Error parsing event E3Requested"); // TODO: provide more info + error!("Error parsing event E3Requested after topic matched!"); return None; }; Some(EnclaveEvent::from(E3RequestedWithChainId(event, chain_id))) } Some(&IEnclave::CiphertextOutputPublished::SIGNATURE_HASH) => { let Ok(event) = IEnclave::CiphertextOutputPublished::decode_log_data(data, true) else { - println!("Error parsing event CiphertextOutputPublished"); // TODO: provide more info + error!("Error parsing event CiphertextOutputPublished after topic matched!"); // TODO: provide more info return None; }; Some(EnclaveEvent::from(event)) } - _ => { - println!("Unknown event"); + _topic => { + trace!( + topic=?_topic, + "Unknown event was received by Enclave.sol parser buut was ignored" + ); return None; } } @@ -105,7 +109,7 @@ impl EnclaveSolReader { .await? .start(); - println!("Evm is listening to {}", contract_address); + info!(address=%contract_address, "Evm is listening to address"); Ok(addr) } } diff --git a/packages/ciphernode/evm/src/enclave_sol_writer.rs b/packages/ciphernode/evm/src/enclave_sol_writer.rs index 0fdf5bb7..f7444240 100644 --- a/packages/ciphernode/evm/src/enclave_sol_writer.rs +++ b/packages/ciphernode/evm/src/enclave_sol_writer.rs @@ -13,6 +13,7 @@ use alloy::{ use anyhow::Result; use enclave_core::{BusError, E3id, EnclaveErrorType, PlaintextAggregated, Subscribe}; use enclave_core::{EnclaveEvent, EventBus}; +use tracing::info; sol!( #[sol(rpc)] @@ -93,7 +94,7 @@ impl Handler for EnclaveSolWriter { .await; match result { Ok(receipt) => { - println!("tx:{}", receipt.transaction_hash) + info!(tx=%receipt.transaction_hash, "tx") } Err(err) => bus.err(EnclaveErrorType::Evm, err), } diff --git a/packages/ciphernode/evm/src/helpers.rs b/packages/ciphernode/evm/src/helpers.rs index 81a2e7d6..680e0e39 100644 --- a/packages/ciphernode/evm/src/helpers.rs +++ b/packages/ciphernode/evm/src/helpers.rs @@ -15,6 +15,7 @@ use alloy::{ use anyhow::{Context, Result}; use enclave_core::{BusError, EnclaveErrorType, EnclaveEvent}; use futures_util::stream::StreamExt; +use tracing::{info, trace}; pub async fn stream_from_evm( provider: WithChainId

, @@ -31,10 +32,13 @@ pub async fn stream_from_evm( Ok(subscription) => { let mut stream = subscription.into_stream(); while let Some(log) = stream.next().await { + trace!("Received log from EVM"); let Some(event) = extractor(log.data(), log.topic0(), provider.get_chain_id()) else { + trace!("Failed to extract log from EVM"); continue; }; + info!("Extracted log from evm sending now."); bus.do_send(event); } } diff --git a/packages/ciphernode/evm/src/registry_filter_sol.rs b/packages/ciphernode/evm/src/registry_filter_sol.rs index bd96561c..318e065e 100644 --- a/packages/ciphernode/evm/src/registry_filter_sol.rs +++ b/packages/ciphernode/evm/src/registry_filter_sol.rs @@ -12,6 +12,7 @@ use enclave_core::{ Subscribe, }; use std::sync::Arc; +use tracing::info; sol!( #[sol(rpc)] @@ -92,7 +93,7 @@ impl Handler for RegistryFilterSolWriter { publish_committee(provider, contract_address, e3_id, nodes, pubkey).await; match result { Ok(receipt) => { - println!("tx:{}", receipt.transaction_hash); + info!(tx=%receipt.transaction_hash,"tx"); } Err(err) => bus.err(EnclaveErrorType::Evm, err), } diff --git a/packages/ciphernode/keyshare/src/keyshare.rs b/packages/ciphernode/keyshare/src/keyshare.rs index 2d036a97..4d8cef5e 100644 --- a/packages/ciphernode/keyshare/src/keyshare.rs +++ b/packages/ciphernode/keyshare/src/keyshare.rs @@ -124,8 +124,6 @@ async fn on_decryption_requested( return Err(anyhow::anyhow!("Secret key not stored for {}", e3_id)); }; - println!("\n\nDECRYPTING!\n\n"); - let decryption_share = fhe .decrypt_ciphertext(DecryptCiphertext { ciphertext: ciphertext_output, diff --git a/packages/ciphernode/logger/Cargo.toml b/packages/ciphernode/logger/Cargo.toml index 3e70edd8..9f87116f 100644 --- a/packages/ciphernode/logger/Cargo.toml +++ b/packages/ciphernode/logger/Cargo.toml @@ -7,3 +7,4 @@ edition = "2021" enclave-core = { path = "../core" } actix = { workspace = true } base64 = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/logger/src/logger.rs b/packages/ciphernode/logger/src/logger.rs index fd241329..8fe3a3cb 100644 --- a/packages/ciphernode/logger/src/logger.rs +++ b/packages/ciphernode/logger/src/logger.rs @@ -1,7 +1,6 @@ use actix::{Actor, Addr, Context, Handler}; -use base64::prelude::*; - use enclave_core::{EnclaveEvent, EventBus, Subscribe}; +use tracing::{error, info}; pub struct SimpleLogger { name: String, @@ -17,7 +16,7 @@ impl SimpleLogger { listener: addr.clone().recipient(), event_type: "*".to_string(), }); - println!("[{}]: READY", name); + info!(node=%name, "READY!"); addr } } @@ -29,30 +28,12 @@ impl Actor for SimpleLogger { impl Handler for SimpleLogger { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { - match msg.clone() { - EnclaveEvent::PublicKeyAggregated { data, .. } => { - let pubkey_str = BASE64_STANDARD.encode(&data.pubkey); - println!( - "\n[{}]: PUBKEY: {}...{}\n", - self.name, - &pubkey_str[..20], - &pubkey_str[pubkey_str.len() - 20..] - ); - println!("[{}]: {}", self.name, msg); - } - EnclaveEvent::CiphernodeAdded { data, .. } => { - println!("[{}]: CiphernodeAdded({})", self.name, data.address); - } - EnclaveEvent::E3Requested { data, .. } => { - println!( - "[{}]: E3Requested(e3_id: {}, threshold_m: {} , seed: {})", - self.name, data.e3_id, data.threshold_m, data.seed - ); - } - EnclaveEvent::EnclaveError { data, .. } => { - println!("[{}]: EnclaveError('{}')", self.name, data.message); - } - _ => println!("[{}]: {}", self.name, msg), - } + match msg { + EnclaveEvent::EnclaveError { .. } => error!(event=%msg, "ERROR!"), + _ => match msg.get_e3_id() { + Some(e3_id) => info!(me=self.name, evt=%msg, e3_id=%e3_id, "Event Broadcasted"), + None => info!(me=self.name, evt=%msg, "Event Broadcasted"), + }, + }; } } diff --git a/packages/ciphernode/p2p/src/libp2p_router.rs b/packages/ciphernode/p2p/src/libp2p_router.rs index 55511bff..9ed1a8c8 100644 --- a/packages/ciphernode/p2p/src/libp2p_router.rs +++ b/packages/ciphernode/p2p/src/libp2p_router.rs @@ -8,6 +8,7 @@ use std::hash::{Hash, Hasher}; use std::time::Duration; use tokio::sync::mpsc::{channel, Receiver, Sender}; use tokio::{io, select}; +use tracing::{error, info, trace}; use tracing_subscriber::EnvFilter; #[derive(NetworkBehaviour)] @@ -93,7 +94,7 @@ impl EnclaveRouter { .build(); self.swarm = Some(swarm); } - _ => println!("Defaulting to MDNS discovery"), + _ => info!("Defaulting to MDNS discovery"), } Ok(self) } @@ -110,6 +111,7 @@ impl EnclaveRouter { Ok(self) } + /// Listen on the default multiaddr pub async fn start(&mut self) -> Result<(), Box> { self.swarm .as_mut() @@ -125,19 +127,19 @@ impl EnclaveRouter { if let Err(e) = self.swarm.as_mut().unwrap() .behaviour_mut().gossipsub .publish(self.topic.as_mut().unwrap().clone(), line) { - println!("Publish error: {e:?}"); + error!(error=?e, "Error publishing line to swarm"); } } event = self.swarm.as_mut().unwrap().select_next_some() => match event { SwarmEvent::Behaviour(MyBehaviourEvent::Mdns(mdns::Event::Discovered(list))) => { for (peer_id, _multiaddr) in list { - // println!("mDNS discovered a new peer: {peer_id}"); + trace!("mDNS discovered a new peer: {peer_id}"); self.swarm.as_mut().unwrap().behaviour_mut().gossipsub.add_explicit_peer(&peer_id); } }, SwarmEvent::Behaviour(MyBehaviourEvent::Mdns(mdns::Event::Expired(list))) => { for (peer_id, _multiaddr) in list { - // println!("mDNS discover peer has expired: {peer_id}"); + trace!("mDNS discover peer has expired: {peer_id}"); self.swarm.as_mut().unwrap().behaviour_mut().gossipsub.remove_explicit_peer(&peer_id); } }, @@ -146,14 +148,14 @@ impl EnclaveRouter { message_id: id, message, })) => { - // println!( - // "Got message with id: {id} from peer: {peer_id}", - // ); - // println!("{:?}", message); + trace!( + "Got message with id: {id} from peer: {peer_id}", + ); + trace!("{:?}", message); self.evt_tx.send(message.data).await?; }, SwarmEvent::NewListenAddr { address, .. } => { - // println!("Local node is listening on {address}"); + trace!("Local node is listening on {address}"); } _ => {} } @@ -161,15 +163,3 @@ impl EnclaveRouter { } } } - -// #[tokio::main] -// async fn main() -> Result<(), Box> { - -// let mut p2p = EnclaveRouter::new()?; -// p2p.connect_swarm("mdns".to_string())?; -// p2p.join_topic("enclave-keygen-01")?; -// p2p.start().await?; - -// println!("Hello, cipher world!"); -// Ok(()) -// } diff --git a/packages/ciphernode/p2p/src/p2p.rs b/packages/ciphernode/p2p/src/p2p.rs index fd70c42f..635ad8e5 100644 --- a/packages/ciphernode/p2p/src/p2p.rs +++ b/packages/ciphernode/p2p/src/p2p.rs @@ -7,7 +7,9 @@ use actix::prelude::*; use tokio::sync::mpsc::{Receiver, Sender}; use enclave_core::{EnclaveEvent, EventBus, EventId, Subscribe}; +use tracing::{error, trace}; +/// P2p Actor converts between EVentBus events and Libp2p events pub struct P2p { bus: Addr, tx: Sender>, @@ -23,6 +25,7 @@ impl Actor for P2p { struct LibP2pEvent(pub Vec); impl P2p { + /// Create a new P2p actor pub fn new(bus: Addr, tx: Sender>) -> Self { Self { bus, @@ -59,6 +62,7 @@ impl P2p { p2p } + /// Spawn a Libp2p instance. Calls spawn and listen pub fn spawn_libp2p( bus: Addr, ) -> Result<(Addr, tokio::task::JoinHandle<()>), Box> { @@ -81,7 +85,7 @@ impl Handler for P2p { self.bus.do_send(event.clone()); self.sent_events.insert(event.into()); } - Err(err) => println!("Error: {}", err), + Err(err) => error!(error=?err, "Could not create EnclaveEvent from Libp2p Bytes!"), } Ok(()) } @@ -98,19 +102,25 @@ impl Handler for P2p { // if we have seen this event before dont rebroadcast if sent_events.contains(&id) { + trace!(evt_id=%id,"Have seen event before not rebroadcasting!"); return; } // Ignore events that should be considered local if evt.is_local_only() { + trace!(evt_id=%id,"Local events should not be rebroadcast so ignoring"); return; } match evt.to_bytes() { Ok(bytes) => { - let _ = tx.send(bytes).await; + if let Err(e) = tx.send(bytes).await { + error!(error=?e, "Error sending bytes to libp2p"); + }; + } + Err(error) => { + error!(error=?error, "Could not convert event to bytes for serialization!") } - Err(error) => println!("Error: {}", error), } }) } diff --git a/packages/ciphernode/router/Cargo.toml b/packages/ciphernode/router/Cargo.toml index aef11dc7..7504a96b 100644 --- a/packages/ciphernode/router/Cargo.toml +++ b/packages/ciphernode/router/Cargo.toml @@ -11,3 +11,5 @@ fhe = { path = "../fhe" } data = { path = "../data" } keyshare = { path = "../keyshare" } aggregator = { path = "../aggregator" } +anyhow = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/router/src/ciphernode_selector.rs b/packages/ciphernode/router/src/ciphernode_selector.rs index d3cacdb2..236ff0fe 100644 --- a/packages/ciphernode/router/src/ciphernode_selector.rs +++ b/packages/ciphernode/router/src/ciphernode_selector.rs @@ -3,6 +3,7 @@ use actix::prelude::*; use enclave_core::{CiphernodeSelected, EnclaveEvent, EventBus, Subscribe}; use sortition::{GetHasNode, Sortition}; +use tracing::info; pub struct CiphernodeSelector { bus: Addr, @@ -51,12 +52,13 @@ impl Handler for CiphernodeSelector { if let Ok(is_selected) = sortition .send(GetHasNode { seed, - address, + address: address.clone(), size, }) .await { if !is_selected { + info!(node = address, "Ciphernode was not selected"); return; } diff --git a/packages/ciphernode/router/src/hooks.rs b/packages/ciphernode/router/src/hooks.rs index 9e50c45a..20275497 100644 --- a/packages/ciphernode/router/src/hooks.rs +++ b/packages/ciphernode/router/src/hooks.rs @@ -1,13 +1,13 @@ use crate::EventHook; use actix::{Actor, Addr}; use aggregator::{PlaintextAggregator, PublicKeyAggregator}; +use anyhow::anyhow; use data::Data; -use enclave_core::{E3Requested, EnclaveEvent, EventBus}; +use enclave_core::{BusError, E3Requested, EnclaveErrorType, EnclaveEvent, EventBus}; use fhe::{Fhe, SharedRng}; use keyshare::Keyshare; use sortition::Sortition; use std::sync::Arc; - pub struct LazyFhe; impl LazyFhe { @@ -37,6 +37,7 @@ impl LazyKeyshare { }; let Some(ref fhe) = ctx.fhe else { + bus.err(EnclaveErrorType::KeyGeneration, anyhow!("Could not create Keyshare because the fhe instance it depends on was not set on the context.")); return; }; @@ -55,9 +56,11 @@ impl LazyPlaintextAggregator { return; }; let Some(ref fhe) = ctx.fhe else { + bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the fhe instance it depends on was not set on the context.")); return; }; let Some(ref meta) = ctx.meta else { + bus.err(EnclaveErrorType::PlaintextAggregation, anyhow!("Could not create PlaintextAggregator because the meta instance it depends on was not set on the context.")); return; }; @@ -88,11 +91,11 @@ impl LazyPublicKeyAggregator { }; let Some(ref fhe) = ctx.fhe else { - println!("fhe was not on ctx"); + bus.err(EnclaveErrorType::PublickeyAggregation, anyhow!("Could not create PublicKeyAggregator because the fhe instance it depends on was not set on the context.")); return; }; let Some(ref meta) = ctx.meta else { - println!("meta was not on ctx"); + bus.err(EnclaveErrorType::PublickeyAggregation, anyhow!("Could not create PublicKeyAggregator because the meta instance it depends on was not set on the context.")); return; }; diff --git a/packages/ciphernode/sortition/Cargo.toml b/packages/ciphernode/sortition/Cargo.toml index eb856507..46d9f893 100644 --- a/packages/ciphernode/sortition/Cargo.toml +++ b/packages/ciphernode/sortition/Cargo.toml @@ -15,3 +15,4 @@ alloy = { workspace = true, features = ["full"] } actix = { workspace = true } enclave-core = { path = "../core" } anyhow = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/sortition/src/sortition.rs b/packages/ciphernode/sortition/src/sortition.rs index 57894e8c..d3cdd08e 100644 --- a/packages/ciphernode/sortition/src/sortition.rs +++ b/packages/ciphernode/sortition/src/sortition.rs @@ -1,9 +1,12 @@ -use std::collections::HashSet; - use crate::DistanceSortition; use actix::prelude::*; use alloy::primitives::Address; -use enclave_core::{CiphernodeAdded, CiphernodeRemoved, EnclaveEvent, EventBus, Seed, Subscribe}; +use anyhow::{anyhow, Result}; +use enclave_core::{ + BusError, CiphernodeAdded, CiphernodeRemoved, EnclaveErrorType, EnclaveEvent, EventBus, Seed, + Subscribe, +}; +use std::collections::HashSet; #[derive(Message, Clone, Debug, PartialEq, Eq)] #[rtype(result = "bool")] @@ -14,7 +17,7 @@ pub struct GetHasNode { } pub trait SortitionList { - fn contains(&self, seed: Seed, size: usize, address: T) -> bool; + fn contains(&self, seed: Seed, size: usize, address: T) -> Result; fn add(&mut self, address: T); fn remove(&mut self, address: T); } @@ -38,10 +41,9 @@ impl Default for SortitionModule { } impl SortitionList for SortitionModule { - fn contains(&self, seed: Seed, size: usize, address: String) -> bool { + fn contains(&self, seed: Seed, size: usize, address: String) -> Result { if self.nodes.len() == 0 { - eprintln!("ERROR: No nodes registered!"); - return false; + return Err(anyhow!("No nodes registered!")); } let registered_nodes: Vec

= self @@ -55,13 +57,12 @@ impl SortitionList for SortitionModule { let Ok(committee) = DistanceSortition::new(seed.into(), registered_nodes, size).get_committee() else { - eprintln!("Error: Could not get committee!"); - return false; + return Err(anyhow!("Could not get committee!")); }; - committee + Ok(committee .iter() - .any(|(_, addr)| addr.to_string() == address) + .any(|(_, addr)| addr.to_string() == address)) } fn add(&mut self, address: String) { @@ -79,17 +80,19 @@ pub struct GetNodes; pub struct Sortition { list: SortitionModule, + bus: Addr, } impl Sortition { - pub fn new() -> Self { + pub fn new(bus: Addr) -> Self { Self { list: SortitionModule::new(), + bus, } } pub fn attach(bus: Addr) -> Addr { - let addr = Sortition::new().start(); + let addr = Sortition::new(bus.clone()).start(); bus.do_send(Subscribe::new("CiphernodeAdded", addr.clone().into())); addr } @@ -99,12 +102,6 @@ impl Sortition { } } -impl Default for Sortition { - fn default() -> Self { - Self::new() - } -} - impl Actor for Sortition { type Context = actix::Context; } @@ -137,7 +134,13 @@ impl Handler for Sortition { impl Handler for Sortition { type Result = bool; fn handle(&mut self, msg: GetHasNode, _ctx: &mut Self::Context) -> Self::Result { - self.list.contains(msg.seed, msg.size, msg.address) + match self.list.contains(msg.seed, msg.size, msg.address) { + Ok(val) => val, + Err(err) => { + self.bus.err(EnclaveErrorType::Sortition, err); + false + } + } } } diff --git a/packages/ciphernode/test_helpers/Cargo.toml b/packages/ciphernode/test_helpers/Cargo.toml index 52fa85a7..e91ac8dd 100644 --- a/packages/ciphernode/test_helpers/Cargo.toml +++ b/packages/ciphernode/test_helpers/Cargo.toml @@ -13,3 +13,4 @@ fhe_rs = { workspace = true } fhe-traits = { workspace = true } rand_chacha = { workspace = true } rand = { workspace = true } +tracing = { workspace = true } diff --git a/packages/ciphernode/test_helpers/src/plaintext_writer.rs b/packages/ciphernode/test_helpers/src/plaintext_writer.rs index 967b15d2..851d2f1f 100644 --- a/packages/ciphernode/test_helpers/src/plaintext_writer.rs +++ b/packages/ciphernode/test_helpers/src/plaintext_writer.rs @@ -1,6 +1,7 @@ use super::write_file_with_dirs; use actix::{Actor, Addr, Context, Handler}; use enclave_core::{EnclaveEvent, EventBus, Subscribe}; +use tracing::info; pub struct PlaintextWriter { path: String, @@ -30,7 +31,7 @@ impl Handler for PlaintextWriter { if let EnclaveEvent::PlaintextAggregated { data, .. } = msg.clone() { let output: Vec = bincode::deserialize(&data.decrypted_output).unwrap(); - println!("Write plaintext to {}", &self.path); + info!(path = &self.path, "Writing Plaintext To Path"); let contents: Vec = output.iter().map(|&num| num.to_string()).collect(); // NOTE: panicking is kind of what we want here for now as we don't really need to handle the diff --git a/packages/ciphernode/test_helpers/src/public_key_writer.rs b/packages/ciphernode/test_helpers/src/public_key_writer.rs index e797dd73..7310ee2e 100644 --- a/packages/ciphernode/test_helpers/src/public_key_writer.rs +++ b/packages/ciphernode/test_helpers/src/public_key_writer.rs @@ -1,6 +1,7 @@ use super::write_file_with_dirs; use actix::{Actor, Addr, Context, Handler}; use enclave_core::{EnclaveEvent, EventBus, Subscribe}; +use tracing::info; pub struct PublicKeyWriter { path: String, @@ -28,7 +29,7 @@ impl Handler for PublicKeyWriter { type Result = (); fn handle(&mut self, msg: EnclaveEvent, _: &mut Self::Context) -> Self::Result { if let EnclaveEvent::PublicKeyAggregated { data, .. } = msg.clone() { - println!("Write pubkey to {}", &self.path); + info!(path = &self.path, "Writing Pubkey To Path"); write_file_with_dirs(&self.path, &data.pubkey).unwrap(); } } diff --git a/packages/ciphernode/test_helpers/src/utils.rs b/packages/ciphernode/test_helpers/src/utils.rs index f100610b..5d88f8b9 100644 --- a/packages/ciphernode/test_helpers/src/utils.rs +++ b/packages/ciphernode/test_helpers/src/utils.rs @@ -1,4 +1,6 @@ use std::{fs, io::Write, path::Path}; + +use tracing::{error, trace}; pub fn write_file_with_dirs(path: &str, content: &[u8]) -> std::io::Result<()> { let abs_path = if Path::new(path).is_absolute() { Path::new(path).to_path_buf() @@ -7,6 +9,11 @@ pub fn write_file_with_dirs(path: &str, content: &[u8]) -> std::io::Result<()> { cwd.join(path) }; + match abs_path.to_str() { + Some(s) => trace!(path = s, "Writing to path"), + None => error!(path=?abs_path, "Cannot parse path"), + } + // Ensure the directory structure exists if let Some(parent) = abs_path.parent() { fs::create_dir_all(parent)?; @@ -15,7 +22,9 @@ pub fn write_file_with_dirs(path: &str, content: &[u8]) -> std::io::Result<()> { // Open the file (creates it if it doesn't exist) and write the content let mut file = fs::File::create(&abs_path)?; file.write_all(content)?; - - println!("File written successfully: {:?}", abs_path); + trace!( + path = abs_path.to_str().unwrap(), + "File written successfully!" + ); Ok(()) } diff --git a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs index a6caa44c..fa4e7f9e 100644 --- a/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs +++ b/packages/ciphernode/tests/tests/test_aggregation_and_decryption.rs @@ -142,7 +142,6 @@ async fn test_public_key_aggregation_and_decryption() -> Result<()> { .into_iter() .aggregate()?; - println!("&&&& {}", history[8].event_type()); assert_eq!(history.len(), 9); assert_eq!( history, diff --git a/tests/basic_integration/test.sh b/tests/basic_integration/test.sh index 6da566c5..70be5115 100755 --- a/tests/basic_integration/test.sh +++ b/tests/basic_integration/test.sh @@ -12,6 +12,7 @@ if [[ "$ROOT_DIR" != "$(pwd)" ]]; then exit 1 fi +export RUST_LOG=info # Environment variables RPC_URL="ws://localhost:8545"