From 7ae5b2d0a477d7101c7b3d6fb772d5e53e5d8622 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Feb 2023 16:44:25 +0100 Subject: [PATCH 01/83] Make establisher `get_listener` non-async --- massa-bootstrap/src/establisher.rs | 2 +- massa-bootstrap/src/server.rs | 1 - massa-bootstrap/src/tests/mock_establisher.rs | 16 +++++++++------- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 69b458438e9..837af24341c 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -76,7 +76,7 @@ pub mod types { /// /// # Argument /// * `addr`: `SocketAddr` we want to bind to. - pub async fn get_listener(&mut self, addr: SocketAddr) -> io::Result { + pub fn get_listener(&mut self, addr: SocketAddr) -> io::Result { // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag let socket = socket2::Socket::new(socket2::Domain::IPV6, socket2::Type::STREAM, None)?; diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 32c54a0c974..26024101e66 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -133,7 +133,6 @@ pub async fn start_bootstrap_server( let listener = establisher .get_listener(listen_addr) - .await .map_err(BootstrapError::IoError)?; // This is the primary interface between the async-listener, and the "sync" worker diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 7fc04769167..ca0c24e1f26 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -13,7 +13,7 @@ pub type Duplex = DuplexStream; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = - mpsc::channel::<(SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); + crossbeam::channel::bounded::<(SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); let (connection_connector_tx, connection_connector_rx) = mpsc::channel::<(Duplex, SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); @@ -32,12 +32,12 @@ pub fn new() -> (MockEstablisher, MockEstablisherInterface) { #[derive(Debug)] pub struct MockListener { - connection_listener_rx: mpsc::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) + connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) } impl MockListener { pub async fn accept(&mut self) -> std::io::Result<(Duplex, SocketAddr)> { - let (addr, sender) = self.connection_listener_rx.recv().await.ok_or_else(|| { + let (addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener accept channel from Establisher closed".to_string(), @@ -100,12 +100,13 @@ impl MockConnector { #[derive(Debug)] pub struct MockEstablisher { - connection_listener_rx: Option)>>, + connection_listener_rx: + Option)>>, connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, } impl MockEstablisher { - pub async fn get_listener(&mut self, _addr: SocketAddr) -> io::Result { + pub fn get_listener(&mut self, _addr: SocketAddr) -> io::Result { Ok(MockListener { connection_listener_rx: self .connection_listener_rx @@ -128,7 +129,8 @@ impl MockEstablisher { } pub struct MockEstablisherInterface { - connection_listener_tx: Option)>>, + connection_listener_tx: + Option)>>, connection_connector_rx: mpsc::Receiver<(Duplex, SocketAddr, oneshot::Sender)>, } @@ -141,7 +143,7 @@ impl MockEstablisherInterface { ) })?; let (response_tx, response_rx) = oneshot::channel::(); - sender.send((*addr, response_tx)).await.map_err(|_err| { + sender.send((*addr, response_tx)).map_err(|_err| { io::Error::new( io::ErrorKind::Other, "mock connect_to_controller_listener channel to listener closed".to_string(), From bdd8e544ab9e6a6a0ea8e5b96b3a5fac3176a973 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Feb 2023 16:47:29 +0100 Subject: [PATCH 02/83] Make establisher `fn get_connector` non-async --- massa-bootstrap/src/client.rs | 6 ++---- massa-bootstrap/src/establisher.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 5 +---- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 6e7baa81ca9..9d96ce2b8aa 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -377,10 +377,8 @@ async fn connect_to_server( pub_key: &PublicKey, ) -> Result { // connect - let mut connector = establisher - .get_connector(bootstrap_config.connect_timeout) - .await?; // cancellable - let socket = connector.connect(*addr).await?; // cancellable + let mut connector = establisher.get_connector(bootstrap_config.connect_timeout)?; + let socket = connector.connect(*addr).await?; Ok(BootstrapClientBinder::new( socket, *pub_key, diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 837af24341c..6c7b9af3686 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -94,7 +94,7 @@ pub mod types { /// /// # Argument /// * `timeout_duration`: timeout duration in milliseconds - pub async fn get_connector( + pub fn get_connector( &mut self, timeout_duration: MassaTime, ) -> io::Result { diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index ca0c24e1f26..fb68d5db692 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -115,10 +115,7 @@ impl MockEstablisher { }) } - pub async fn get_connector( - &mut self, - timeout_duration: MassaTime, - ) -> std::io::Result { + pub fn get_connector(&mut self, timeout_duration: MassaTime) -> std::io::Result { // create connector stream Ok(MockConnector { From b77875c3e65510669469189bd64b8e862273e719 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Feb 2023 19:33:05 +0100 Subject: [PATCH 03/83] Set the mockers to use a TcpStream as a duplex --- massa-bootstrap/src/establisher.rs | 4 +-- massa-bootstrap/src/tests/binders.rs | 24 +++++++++---- massa-bootstrap/src/tests/mock_establisher.rs | 36 ++++++++++++------- massa-bootstrap/src/tests/tools.rs | 7 ++-- 4 files changed, 47 insertions(+), 24 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 6c7b9af3686..c98aed760c4 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -22,9 +22,9 @@ pub mod types { }; /// duplex connection pub type Duplex = TcpStream; - /// listener + /// listener, used by server pub type Listener = DefaultListener; - /// connector + /// connector, used by client pub type Connector = DefaultConnector; /// connection establisher pub type Establisher = DefaultEstablisher; diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 77111e9c238..b8035432698 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -21,7 +21,6 @@ use massa_signature::{KeyPair, PublicKey}; use massa_time::MassaTime; use serial_test::serial; use std::str::FromStr; -use tokio::io::duplex; lazy_static::lazy_static! { pub static ref BOOTSTRAP_CONFIG_KEYPAIR: (BootstrapConfig, KeyPair) = { @@ -66,9 +65,13 @@ impl BootstrapClientBinder { #[serial] async fn test_binders() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let (client, server) = duplex(1000000); + let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + let client = tokio::net::TcpStream::connect(addr).await.unwrap(); + let server = server.accept().await.unwrap(); + // let (client, server) = duplex(1000000); let mut server = BootstrapServerBinder::new( - server, + server.0, server_keypair.clone(), BootstrapSrvBindCfg { max_bytes_read_write: f64::INFINITY, @@ -165,10 +168,14 @@ async fn test_binders() { async fn test_binders_double_send_server_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let (client, server) = duplex(1000000); + let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let client = tokio::net::TcpStream::connect(server.local_addr().unwrap()) + .await + .unwrap(); + let server = server.accept().await.unwrap(); let mut server = BootstrapServerBinder::new( - server, + server.0, server_keypair.clone(), BootstrapSrvBindCfg { max_bytes_read_write: f64::INFINITY, @@ -250,9 +257,12 @@ async fn test_binders_double_send_server_works() { async fn test_binders_try_double_send_client_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let (client, server) = duplex(1000000); + let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let addr = server.local_addr().unwrap(); + let client = tokio::net::TcpStream::connect(addr).await.unwrap(); + let server = server.accept().await.unwrap(); let mut server = BootstrapServerBinder::new( - server, + server.0, server_keypair.clone(), BootstrapSrvBindCfg { max_bytes_read_write: f64::INFINITY, diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index fb68d5db692..6c63978b5e0 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,15 +1,14 @@ // Copyright (c) 2022 MASSA LABS -use massa_models::config::{CHANNEL_SIZE, MAX_DUPLEX_BUFFER_SIZE}; +use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; use socket2 as _; use std::io; use std::net::SocketAddr; -use tokio::io::DuplexStream; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -pub type Duplex = DuplexStream; +pub type Duplex = tokio::net::TcpStream; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -32,18 +31,24 @@ pub fn new() -> (MockEstablisher, MockEstablisherInterface) { #[derive(Debug)] pub struct MockListener { - connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) + connection_listener_rx: + crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) } impl MockListener { - pub async fn accept(&mut self) -> std::io::Result<(Duplex, SocketAddr)> { - let (addr, sender) = self.connection_listener_rx.recv().map_err(|_| { + pub async fn accept(&mut self) -> std::io::Result<(tokio::net::TcpStream, SocketAddr)> { + let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener accept channel from Establisher closed".to_string(), ) })?; - let (duplex_controller, duplex_mock) = tokio::io::duplex(MAX_DUPLEX_BUFFER_SIZE); + let duplex_controller = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let duplex_mock = tokio::net::TcpStream::connect(duplex_controller.local_addr().unwrap()) + .await + .unwrap(); + let duplex_controller = duplex_controller.accept().await.unwrap(); + sender.send(duplex_mock).map_err(|_| { io::Error::new( io::ErrorKind::Other, @@ -51,27 +56,32 @@ impl MockListener { ) })?; - Ok((duplex_controller, addr)) + Ok(duplex_controller) } } #[derive(Debug)] pub struct MockConnector { - connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, + connection_connector_tx: + mpsc::Sender<(tokio::net::TcpStream, SocketAddr, oneshot::Sender)>, timeout_duration: MassaTime, } impl MockConnector { - pub async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { - // task the controller connection if exist. - let (duplex_controller, duplex_mock) = tokio::io::duplex(MAX_DUPLEX_BUFFER_SIZE); + pub async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { + let duplex_mock = tokio::net::TcpListener::bind(addr).await.unwrap(); + let duplex_controller = tokio::net::TcpStream::connect(addr).await.unwrap(); + let duplex_mock = duplex_mock.accept().await.unwrap(); + + // // task the controller connection if exist. + // let (duplex_controller, duplex_mock) = tokio::io::duplex(MAX_DUPLEX_BUFFER_SIZE); // to see if the connection is accepted let (accept_tx, accept_rx) = oneshot::channel::(); // send new connection to mock timeout(self.timeout_duration.to_duration(), async move { self.connection_connector_tx - .send((duplex_mock, addr, accept_tx)) + .send((duplex_mock.0, addr, accept_tx)) .await .map_err(|_err| { io::Error::new( diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index fe99bd03521..695b0283dbf 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -64,7 +64,7 @@ use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::{sync::mpsc::Receiver, time::sleep}; -pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(169, 202, 0, 10)); +pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 0)); /// generates a small random number of bytes fn get_some_random_bytes() -> Vec { @@ -301,7 +301,10 @@ pub fn get_bootstrap_config(bootstrap_public_key: NodeId) -> BootstrapConfig { write_timeout: 1000.into(), read_error_timeout: 200.into(), write_error_timeout: 200.into(), - bootstrap_list: vec![(SocketAddr::new(BASE_BOOTSTRAP_IP, 16), bootstrap_public_key)], + bootstrap_list: vec![( + SocketAddr::new(BASE_BOOTSTRAP_IP, 8069), + bootstrap_public_key, + )], bootstrap_whitelist_path: PathBuf::from( "../massa-node/base_config/bootstrap_whitelist.json", ), From 4e6c89fa63422f31d70e81da6a38f19853ca2b24 Mon Sep 17 00:00:00 2001 From: modship Date: Tue, 28 Feb 2023 10:38:54 +0100 Subject: [PATCH 04/83] fix(bootstrap) : fix base_bootstrap_ip --- massa-bootstrap/src/tests/tools.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index 695b0283dbf..8e81f0af00c 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -64,7 +64,7 @@ use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::{sync::mpsc::Receiver, time::sleep}; -pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 0)); +pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); /// generates a small random number of bytes fn get_some_random_bytes() -> Vec { From bf0301ca7623bce56c439b962221559be274f00b Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 14:48:55 +0100 Subject: [PATCH 05/83] Duplex -> tokio::net::TcpStream. same for Listener --- massa-bootstrap/src/client_binder.rs | 2 +- massa-bootstrap/src/establisher.rs | 26 +++++++++---------- massa-bootstrap/src/server_binder.rs | 2 +- massa-bootstrap/src/tests/binders.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 18 ++++++------- massa-bootstrap/src/tests/tools.rs | 2 +- 6 files changed, 24 insertions(+), 28 deletions(-) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 117a39a00a8..57df199d13b 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -1,7 +1,7 @@ // Copyright (c) 2022 MASSA LABS use crate::error::BootstrapError; -use crate::establisher::types::Duplex; +use crate::establisher::Duplex; use crate::messages::{ BootstrapClientMessage, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index c98aed760c4..6a5466f207f 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -1,29 +1,27 @@ // Copyright (c) 2022 MASSA LABS +use std::{io, net::SocketAddr}; #[cfg(test)] pub mod types { - pub type Duplex = crate::tests::mock_establisher::Duplex; - pub type Listener = crate::tests::mock_establisher::MockListener; pub type Connector = crate::tests::mock_establisher::MockConnector; pub type Establisher = crate::tests::mock_establisher::MockEstablisher; +/// duplex connection +pub type Duplex = tokio::net::TcpStream; +/// duplex connection +pub type DuplexListener = tokio::net::TcpListener; + } #[cfg(not(test))] /// Connection types pub mod types { use massa_time::MassaTime; - use std::{io, net::SocketAddr}; - use tokio::{ - net::{TcpListener, TcpStream}, - time::timeout, - }; - /// duplex connection - pub type Duplex = TcpStream; - /// listener, used by server - pub type Listener = DefaultListener; + + use super::{io, Duplex, DuplexListener, SocketAddr}; + /// connector, used by client pub type Connector = DefaultConnector; /// connection establisher @@ -31,7 +29,7 @@ pub mod types { /// The listener we are using #[derive(Debug)] - pub struct DefaultListener(TcpListener); + pub struct DefaultListener(DuplexListener); impl DefaultListener { /// Accepts a new incoming connection from this listener. @@ -54,7 +52,7 @@ pub mod types { /// # Argument /// * `addr`: `SocketAddr` we are trying to connect to. pub async fn connect(&mut self, addr: SocketAddr) -> io::Result { - match timeout(self.0.to_duration(), TcpStream::connect(addr)).await { + match tokio::time::timeout(self.0.to_duration(), Duplex::connect(addr)).await { Ok(Ok(sock)) => Ok(sock), Ok(Err(e)) => Err(e), Err(e) => Err(io::Error::new(io::ErrorKind::TimedOut, e)), @@ -87,7 +85,7 @@ pub mod types { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - Ok(DefaultListener(TcpListener::from_std(socket.into())?)) + Ok(DefaultListener(DuplexListener::from_std(socket.into())?)) } /// Get the connector with associated timeout diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 8ceb5bcf694..9e96118fcb9 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -1,7 +1,7 @@ // Copyright (c) 2022 MASSA LABS use crate::error::BootstrapError; -use crate::establisher::types::Duplex; +use crate::establisher::Duplex; use crate::messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapServerMessage, BootstrapServerMessageSerializer, diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index b8035432698..d9609a18e77 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -1,6 +1,6 @@ +use crate::establisher::Duplex; use crate::messages::{BootstrapClientMessage, BootstrapServerMessage}; use crate::settings::{BootstrapClientConfig, BootstrapSrvBindCfg}; -use crate::types::Duplex; use crate::BootstrapConfig; use crate::{ client_binder::BootstrapClientBinder, server_binder::BootstrapServerBinder, diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 6c63978b5e0..2fef9ca7618 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -8,7 +8,7 @@ use std::net::SocketAddr; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -pub type Duplex = tokio::net::TcpStream; +use crate::establisher::{Duplex, DuplexListener}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -31,8 +31,7 @@ pub fn new() -> (MockEstablisher, MockEstablisherInterface) { #[derive(Debug)] pub struct MockListener { - connection_listener_rx: - crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) + connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) } impl MockListener { @@ -43,8 +42,8 @@ impl MockListener { "MockListener accept channel from Establisher closed".to_string(), ) })?; - let duplex_controller = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); - let duplex_mock = tokio::net::TcpStream::connect(duplex_controller.local_addr().unwrap()) + let duplex_controller = DuplexListener::bind("localhost:0").await.unwrap(); + let duplex_mock = Duplex::connect(duplex_controller.local_addr().unwrap()) .await .unwrap(); let duplex_controller = duplex_controller.accept().await.unwrap(); @@ -62,15 +61,14 @@ impl MockListener { #[derive(Debug)] pub struct MockConnector { - connection_connector_tx: - mpsc::Sender<(tokio::net::TcpStream, SocketAddr, oneshot::Sender)>, + connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, timeout_duration: MassaTime, } impl MockConnector { - pub async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { - let duplex_mock = tokio::net::TcpListener::bind(addr).await.unwrap(); - let duplex_controller = tokio::net::TcpStream::connect(addr).await.unwrap(); + pub async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { + let duplex_mock = DuplexListener::bind(addr).await.unwrap(); + let duplex_controller = Duplex::connect(addr).await.unwrap(); let duplex_mock = duplex_mock.accept().await.unwrap(); // // task the controller connection if exist. diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index 8e81f0af00c..6250f39fec3 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -1,6 +1,6 @@ // Copyright (c) 2022 MASSA LABS -use super::mock_establisher::Duplex; +use crate::establisher::Duplex; use crate::settings::{BootstrapConfig, IpType}; use bitvec::vec::BitVec; use massa_async_pool::test_exports::{create_async_pool, get_random_message}; From baac844f73726ce7a2506eef4480fe2d6259b11b Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 14:53:21 +0100 Subject: [PATCH 06/83] unify listener interface through trait --- Cargo.lock | 1 + massa-bootstrap/Cargo.toml | 1 + massa-bootstrap/src/establisher.rs | 20 +++++++++++++------ massa-bootstrap/src/server.rs | 4 ++-- massa-bootstrap/src/tests/mock_establisher.rs | 8 +++++--- 5 files changed, 23 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 41534ab8448..23e410eb961 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2427,6 +2427,7 @@ name = "massa_bootstrap" version = "0.1.0" dependencies = [ "async-speed-limit", + "async-trait", "bitvec", "crossbeam", "displaydoc", diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 04fcece5b80..804a6082aef 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -40,6 +40,7 @@ massa_signature = { path = "../massa-signature" } massa_pos_exports = { path = "../massa-pos-exports" } massa_time = { path = "../massa-time" } crossbeam = "0.8.2" +async-trait = "0.1.64" [dev-dependencies] bitvec = { version = "1.0", features = ["serde"] } diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 6a5466f207f..01d5a30f123 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -1,18 +1,25 @@ // Copyright (c) 2022 MASSA LABS +use async_trait::async_trait; use std::{io, net::SocketAddr}; +#[cfg(test)] +use crate::tests::mock_establisher::{MockConnector, MockEstablisher}; #[cfg(test)] pub mod types { - pub type Listener = crate::tests::mock_establisher::MockListener; - pub type Connector = crate::tests::mock_establisher::MockConnector; + pub type Connector = super::MockConnector; - pub type Establisher = crate::tests::mock_establisher::MockEstablisher; + pub type Establisher = super::MockEstablisher; +} /// duplex connection pub type Duplex = tokio::net::TcpStream; + /// duplex connection pub type DuplexListener = tokio::net::TcpListener; +#[async_trait] +pub trait BSListener { + async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)>; } #[cfg(not(test))] @@ -20,7 +27,7 @@ pub type DuplexListener = tokio::net::TcpListener; pub mod types { use massa_time::MassaTime; - use super::{io, Duplex, DuplexListener, SocketAddr}; + use super::{async_trait, io, Duplex, DuplexListener, SocketAddr}; /// connector, used by client pub type Connector = DefaultConnector; @@ -31,9 +38,10 @@ pub mod types { #[derive(Debug)] pub struct DefaultListener(DuplexListener); - impl DefaultListener { + #[async_trait] + impl super::BSListener for DefaultListener { /// Accepts a new incoming connection from this listener. - pub async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)> { + async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)> { // accept let (sock, mut remote_addr) = self.0.accept().await?; // normalize address diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 26024101e66..86e0ad6d174 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -55,9 +55,9 @@ use tracing::{debug, error, info, warn}; use crate::{ error::BootstrapError, + establisher::{BSListener, Duplex}, messages::{BootstrapClientMessage, BootstrapServerMessage}, server_binder::BootstrapServerBinder, - types::{Duplex, Listener}, BootstrapConfig, Establisher, }; @@ -248,7 +248,7 @@ impl BootstrapServer<'_> { /// Err(..) Error accepting a connection /// TODO: Integrate the listener into the bootstrap-main-loop async fn run_listener( - mut listener: Listener, + mut listener: impl BSListener, listener_tx: crossbeam::channel::Sender, ) -> Result, Box> { loop { diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 2fef9ca7618..b02d6536835 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,5 +1,6 @@ // Copyright (c) 2022 MASSA LABS +use async_trait::async_trait; use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; use socket2 as _; @@ -8,7 +9,7 @@ use std::net::SocketAddr; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -use crate::establisher::{Duplex, DuplexListener}; +use crate::establisher::{BSListener, Duplex, DuplexListener}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -34,8 +35,9 @@ pub struct MockListener { connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) } -impl MockListener { - pub async fn accept(&mut self) -> std::io::Result<(tokio::net::TcpStream, SocketAddr)> { +#[async_trait] +impl BSListener for MockListener { + async fn accept(&mut self) -> std::io::Result<(Duplex, SocketAddr)> { let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, From 1e00ce4ff37346134ce8e44e871d380f4f24fbf7 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 15:23:30 +0100 Subject: [PATCH 07/83] remove `#[cfg(test)]` from bootstrap establisher --- massa-bootstrap/src/client.rs | 7 +- massa-bootstrap/src/establisher.rs | 174 +++++++++--------- massa-bootstrap/src/lib.rs | 3 +- massa-bootstrap/src/server.rs | 6 +- massa-bootstrap/src/tests/mock_establisher.rs | 16 +- massa-node/src/main.rs | 4 +- 6 files changed, 105 insertions(+), 105 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 9d96ce2b8aa..8489c710459 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -17,9 +17,10 @@ use tracing::{debug, info, warn}; use crate::{ client_binder::BootstrapClientBinder, error::BootstrapError, + establisher::{BSConnector, BSEstablisher}, messages::{BootstrapClientMessage, BootstrapServerMessage}, settings::IpType, - BootstrapConfig, Establisher, GlobalBootstrapState, + BootstrapConfig, GlobalBootstrapState, }; /// This function will send the starting point to receive a stream of the ledger and will receive and process each part until receive a `BootstrapServerMessage::FinalStateFinished` message from the server. @@ -371,7 +372,7 @@ async fn send_client_message( } async fn connect_to_server( - establisher: &mut Establisher, + establisher: &mut impl BSEstablisher, bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, @@ -417,7 +418,7 @@ fn filter_bootstrap_list( pub async fn get_state( bootstrap_config: &BootstrapConfig, final_state: Arc>, - mut establisher: Establisher, + mut establisher: impl BSEstablisher, version: Version, genesis_timestamp: MassaTime, end_timestamp: Option, diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 01d5a30f123..f2da2dde7f0 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -1,16 +1,8 @@ // Copyright (c) 2022 MASSA LABS use async_trait::async_trait; +use massa_time::MassaTime; use std::{io, net::SocketAddr}; -#[cfg(test)] -use crate::tests::mock_establisher::{MockConnector, MockEstablisher}; -#[cfg(test)] -pub mod types { - - pub type Connector = super::MockConnector; - - pub type Establisher = super::MockEstablisher; -} /// duplex connection pub type Duplex = tokio::net::TcpStream; @@ -22,95 +14,99 @@ pub trait BSListener { async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)>; } -#[cfg(not(test))] -/// Connection types -pub mod types { - use massa_time::MassaTime; - - use super::{async_trait, io, Duplex, DuplexListener, SocketAddr}; - - /// connector, used by client - pub type Connector = DefaultConnector; - /// connection establisher - pub type Establisher = DefaultEstablisher; - - /// The listener we are using - #[derive(Debug)] - pub struct DefaultListener(DuplexListener); - - #[async_trait] - impl super::BSListener for DefaultListener { - /// Accepts a new incoming connection from this listener. - async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)> { - // accept - let (sock, mut remote_addr) = self.0.accept().await?; - // normalize address - remote_addr.set_ip(remote_addr.ip().to_canonical()); - Ok((sock, remote_addr)) - } - } +#[async_trait] +pub trait BSConnector { + async fn connect(&mut self, addr: SocketAddr) -> io::Result; +} - /// Initiates a connection with given timeout in milliseconds - #[derive(Debug)] - pub struct DefaultConnector(MassaTime); - - impl DefaultConnector { - /// Tries to connect to address - /// - /// # Argument - /// * `addr`: `SocketAddr` we are trying to connect to. - pub async fn connect(&mut self, addr: SocketAddr) -> io::Result { - match tokio::time::timeout(self.0.to_duration(), Duplex::connect(addr)).await { - Ok(Ok(sock)) => Ok(sock), - Ok(Err(e)) => Err(e), - Err(e) => Err(io::Error::new(io::ErrorKind::TimedOut, e)), - } - } - } +pub trait BSEstablisher { + // TODO: this is needed for thread spawning. Once the listener is on-thread, the static + // lifetime can be thrown away. + // TODO: use super-advanced lifetime/GAT/other shenanigans to + // make the listener compatable with being moved into a thread + type Listener: BSListener + Send + 'static; + type Connector: BSConnector; + fn get_listener(&mut self, addr: SocketAddr) -> io::Result; + fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result; +} - /// Establishes a connection - #[derive(Debug)] - pub struct DefaultEstablisher; +/// The listener we are using +#[derive(Debug)] +pub struct DefaultListener(DuplexListener); - impl DefaultEstablisher { - /// Creates an Establisher. - pub fn new() -> Self { - DefaultEstablisher {} - } +#[async_trait] +impl BSListener for DefaultListener { + /// Accepts a new incoming connection from this listener. + async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)> { + // accept + let (sock, mut remote_addr) = self.0.accept().await?; + // normalize address + remote_addr.set_ip(remote_addr.ip().to_canonical()); + Ok((sock, remote_addr)) + } +} +/// Initiates a connection with given timeout in milliseconds +#[derive(Debug)] +pub struct DefaultConnector(MassaTime); - /// Gets the associated listener - /// - /// # Argument - /// * `addr`: `SocketAddr` we want to bind to. - pub fn get_listener(&mut self, addr: SocketAddr) -> io::Result { - // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag - let socket = socket2::Socket::new(socket2::Domain::IPV6, socket2::Type::STREAM, None)?; +#[async_trait] +impl BSConnector for DefaultConnector { + /// Tries to connect to address + /// + /// # Argument + /// * `addr`: `SocketAddr` we are trying to connect to. + async fn connect(&mut self, addr: SocketAddr) -> io::Result { + match tokio::time::timeout(self.0.to_duration(), Duplex::connect(addr)).await { + Ok(Ok(sock)) => Ok(sock), + Ok(Err(e)) => Err(e), + Err(e) => Err(io::Error::new(io::ErrorKind::TimedOut, e)), + } + } +} - socket.set_only_v6(false)?; - socket.set_nonblocking(true)?; - socket.bind(&addr.into())?; +/// Establishes a connection +#[derive(Debug)] +pub struct DefaultEstablisher; - // Number of connections to queue, set to the hardcoded value used by tokio - socket.listen(1024)?; +impl DefaultEstablisher { + /// Creates an Establisher. + pub fn new() -> Self { + DefaultEstablisher {} + } +} - Ok(DefaultListener(DuplexListener::from_std(socket.into())?)) - } +impl BSEstablisher for DefaultEstablisher { + type Connector = DefaultConnector; + type Listener = DefaultListener; + /// Gets the associated listener + /// + /// # Argument + /// * `addr`: `SocketAddr` we want to bind to. + fn get_listener(&mut self, addr: SocketAddr) -> io::Result { + // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag + let socket = socket2::Socket::new(socket2::Domain::IPV6, socket2::Type::STREAM, None)?; + + socket.set_only_v6(false)?; + socket.set_nonblocking(true)?; + socket.bind(&addr.into())?; + + // Number of connections to queue, set to the hardcoded value used by tokio + socket.listen(1024)?; + + Ok(DefaultListener(DuplexListener::from_std(socket.into())?)) + } - /// Get the connector with associated timeout - /// - /// # Argument - /// * `timeout_duration`: timeout duration in milliseconds - pub fn get_connector( - &mut self, - timeout_duration: MassaTime, - ) -> io::Result { - Ok(DefaultConnector(timeout_duration)) - } + /// Get the connector with associated timeout + /// + /// # Argument + /// * `timeout_duration`: timeout duration in milliseconds + fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result { + Ok(DefaultConnector(timeout_duration)) } +} - impl Default for DefaultEstablisher { - fn default() -> Self { - Self::new() - } +impl Default for DefaultEstablisher { + fn default() -> Self { + Self::new() } } diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index c55b488e944..b3e16eedaee 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -13,7 +13,6 @@ #![feature(ip)] #![feature(let_chains)] -pub use establisher::types::Establisher; use massa_consensus_exports::bootstrapable_graph::BootstrapableGraph; use massa_final_state::FinalState; use massa_network_exports::BootstrapPeers; @@ -30,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::types; +pub use establisher::DefaultEstablisher; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 86e0ad6d174..0ff52023d02 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -55,10 +55,10 @@ use tracing::{debug, error, info, warn}; use crate::{ error::BootstrapError, - establisher::{BSListener, Duplex}, + establisher::{BSEstablisher, BSListener, Duplex}, messages::{BootstrapClientMessage, BootstrapServerMessage}, server_binder::BootstrapServerBinder, - BootstrapConfig, Establisher, + BootstrapConfig, }; /// Abstraction layer over data produced by the listener, and transported @@ -103,7 +103,7 @@ pub async fn start_bootstrap_server( network_command_sender: NetworkCommandSender, final_state: Arc>, config: BootstrapConfig, - mut establisher: Establisher, + mut establisher: impl BSEstablisher, keypair: KeyPair, version: Version, ) -> Result, Box> { diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index b02d6536835..e465d7c190c 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -9,7 +9,7 @@ use std::net::SocketAddr; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -use crate::establisher::{BSListener, Duplex, DuplexListener}; +use crate::establisher::{BSConnector, BSEstablisher, BSListener, Duplex, DuplexListener}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -67,8 +67,9 @@ pub struct MockConnector { timeout_duration: MassaTime, } -impl MockConnector { - pub async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { +#[async_trait] +impl BSConnector for MockConnector { + async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { let duplex_mock = DuplexListener::bind(addr).await.unwrap(); let duplex_controller = Duplex::connect(addr).await.unwrap(); let duplex_mock = duplex_mock.accept().await.unwrap(); @@ -115,8 +116,11 @@ pub struct MockEstablisher { connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, } -impl MockEstablisher { - pub fn get_listener(&mut self, _addr: SocketAddr) -> io::Result { +impl BSEstablisher for MockEstablisher { + type Listener = MockListener; + type Connector = MockConnector; + + fn get_listener(&mut self, _addr: SocketAddr) -> io::Result { Ok(MockListener { connection_listener_rx: self .connection_listener_rx @@ -125,7 +129,7 @@ impl MockEstablisher { }) } - pub fn get_connector(&mut self, timeout_duration: MassaTime) -> std::io::Result { + fn get_connector(&mut self, timeout_duration: MassaTime) -> std::io::Result { // create connector stream Ok(MockConnector { diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 0f23147a669..85e455210e7 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -236,7 +236,7 @@ async fn launch( res = get_state( &bootstrap_config, final_state.clone(), - massa_bootstrap::types::Establisher::default(), + massa_bootstrap::DefaultEstablisher::default(), *VERSION, *GENESIS_TIMESTAMP, *END_TIMESTAMP, @@ -508,7 +508,7 @@ async fn launch( network_command_sender.clone(), final_state.clone(), bootstrap_config, - massa_bootstrap::Establisher::new(), + massa_bootstrap::DefaultEstablisher::new(), private_key, *VERSION, ) From 77d0b8e6739fd63bf352fa23e7c285e0555690a6 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 15:41:49 +0100 Subject: [PATCH 08/83] Add FIXME to where static lifetime is being used as a hot-fix --- massa-bootstrap/src/server.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 0ff52023d02..2f651ace88c 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -164,6 +164,8 @@ pub async fn start_bootstrap_server( let listen_rt_handle = bs_server_runtime.handle().clone(); let listen_handle = thread::Builder::new() .name("bs_listener".to_string()) + // FIXME: The interface being used shouldn't have `: Send + 'static` as a constraint on the listener assosciated type. + // GAT lifetime is likely to remedy this, however. .spawn(move || { let res = listen_rt_handle.block_on(BootstrapServer::run_listener(listener, listener_tx)); From 710a8c8573bdd3c64b6693ea786adc8a89a173c3 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 16:06:52 +0100 Subject: [PATCH 09/83] Light house-keeping --- massa-bootstrap/Cargo.toml | 4 ++-- massa-bootstrap/src/lib.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 1 - massa-bootstrap/src/tests/mod.rs | 4 ++-- massa-bootstrap/src/tests/tools.rs | 10 ---------- 5 files changed, 5 insertions(+), 16 deletions(-) diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 804a6082aef..272e9794d60 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -24,6 +24,8 @@ tokio = { version = "1.23", features = ["full"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" +crossbeam = "0.8.2" +async-trait = "0.1.64" # custom modules massa_async_pool = { path = "../massa-async-pool" } @@ -39,8 +41,6 @@ massa_serialization = { path = "../massa-serialization" } massa_signature = { path = "../massa-signature" } massa_pos_exports = { path = "../massa-pos-exports" } massa_time = { path = "../massa-time" } -crossbeam = "0.8.2" -async-trait = "0.1.64" [dev-dependencies] bitvec = { version = "1.0", features = ["serde"] } diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index b3e16eedaee..d666fb7877c 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -39,7 +39,7 @@ pub use settings::IpType; pub use settings::{BootstrapConfig, BootstrapServerMessageDeserializerArgs}; #[cfg(test)] -pub mod tests; +pub(crate) mod tests; /// a collection of the bootstrap state snapshots of all relevant modules pub struct GlobalBootstrapState { diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index e465d7c190c..fc8a92496ca 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,5 +1,4 @@ // Copyright (c) 2022 MASSA LABS - use async_trait::async_trait; use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; diff --git a/massa-bootstrap/src/tests/mod.rs b/massa-bootstrap/src/tests/mod.rs index 85238187251..42f01eae9dc 100644 --- a/massa-bootstrap/src/tests/mod.rs +++ b/massa-bootstrap/src/tests/mod.rs @@ -1,6 +1,6 @@ // Copyright (c) 2022 MASSA LABS mod binders; -pub mod mock_establisher; +pub(crate) mod mock_establisher; mod scenarios; -pub mod tools; +pub(crate) mod tools; diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index 6250f39fec3..9cd1a07baff 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -274,21 +274,11 @@ pub fn get_dummy_block_id(s: &str) -> BlockId { BlockId(Hash::compute_from(s.as_bytes())) } -pub fn get_random_public_key() -> PublicKey { - let priv_key = KeyPair::generate(); - priv_key.get_public_key() -} - pub fn get_random_address() -> Address { let priv_key = KeyPair::generate(); Address::from_public_key(&priv_key.get_public_key()) } -pub fn get_dummy_signature(s: &str) -> Signature { - let priv_key = KeyPair::generate(); - priv_key.sign(&Hash::compute_from(s.as_bytes())).unwrap() -} - pub fn get_bootstrap_config(bootstrap_public_key: NodeId) -> BootstrapConfig { BootstrapConfig { listen_addr: Some("0.0.0.0:31244".parse().unwrap()), From 4d14a0948347efbedee4b96b642fbdf75c9bb6ed Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 16:25:03 +0100 Subject: [PATCH 10/83] Clear clippy lint --- massa-bootstrap/src/tests/tools.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index 9cd1a07baff..bae4fc52ab9 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -50,7 +50,7 @@ use massa_models::{ use massa_network_exports::{BootstrapPeers, NetworkCommand}; use massa_pos_exports::{CycleInfo, DeferredCredits, PoSChanges, PoSFinalState, ProductionStats}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; -use massa_signature::{KeyPair, PublicKey, Signature}; +use massa_signature::KeyPair; use massa_time::MassaTime; use rand::Rng; use std::collections::{HashMap, VecDeque}; From 7c131db55a26855ba31233873bf34b2140b9cd97 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 3 Mar 2023 16:46:11 +0100 Subject: [PATCH 11/83] Some code-comments --- massa-bootstrap/src/establisher.rs | 3 +++ massa-bootstrap/src/tests/mock_establisher.rs | 4 +--- massa-bootstrap/src/tests/tools.rs | 1 + 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index f2da2dde7f0..18dd5d5226c 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -10,15 +10,18 @@ pub type Duplex = tokio::net::TcpStream; pub type DuplexListener = tokio::net::TcpListener; #[async_trait] +/// Specifies a common interface that can be used by standard, or mockers pub trait BSListener { async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)>; } #[async_trait] +/// Specifies a common interface that can be used by standard, or mockers pub trait BSConnector { async fn connect(&mut self, addr: SocketAddr) -> io::Result; } +/// Specifies a common interface that can be used by standard, or mockers pub trait BSEstablisher { // TODO: this is needed for thread spawning. Once the listener is on-thread, the static // lifetime can be thrown away. diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index fc8a92496ca..9c6d0d5727a 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -73,9 +73,7 @@ impl BSConnector for MockConnector { let duplex_controller = Duplex::connect(addr).await.unwrap(); let duplex_mock = duplex_mock.accept().await.unwrap(); - // // task the controller connection if exist. - // let (duplex_controller, duplex_mock) = tokio::io::duplex(MAX_DUPLEX_BUFFER_SIZE); - // to see if the connection is accepted + // Used to see if the connection is accepted let (accept_tx, accept_rx) = oneshot::channel::(); // send new connection to mock diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index bae4fc52ab9..eee0f2da8d4 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -64,6 +64,7 @@ use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; use tokio::{sync::mpsc::Receiver, time::sleep}; +// Use loop-back address. use port 0 to auto-assign a port pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); /// generates a small random number of bytes From 2150b034588e38e70913f7ca5eba421dc745d6e5 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 13 Mar 2023 13:57:44 +0100 Subject: [PATCH 12/83] Using a blocking tcp interface, but things lock up --- massa-bootstrap/src/tests/mock_establisher.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 9c6d0d5727a..e4b7629ed0c 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -43,11 +43,11 @@ impl BSListener for MockListener { "MockListener accept channel from Establisher closed".to_string(), ) })?; - let duplex_controller = DuplexListener::bind("localhost:0").await.unwrap(); + let duplex_controller = std::net::TcpListener::bind("localhost:0").unwrap(); let duplex_mock = Duplex::connect(duplex_controller.local_addr().unwrap()) .await .unwrap(); - let duplex_controller = duplex_controller.accept().await.unwrap(); + let (duplex_controller, addr) = duplex_controller.accept().unwrap(); sender.send(duplex_mock).map_err(|_| { io::Error::new( @@ -56,7 +56,7 @@ impl BSListener for MockListener { ) })?; - Ok(duplex_controller) + Ok((Duplex::from_std(duplex_controller).unwrap(), addr)) } } From 2e086c1bbfe491c457d2b96d8c44442308d70369 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 13:08:55 +0100 Subject: [PATCH 13/83] Fix blocking by making the tokio conversion using non-blocking objects --- massa-bootstrap/src/server.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 26 +++++++++++-------- massa-bootstrap/src/tests/scenarios.rs | 1 - massa-node/src/main.rs | 1 - 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 2f651ace88c..b0f50894650 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -98,7 +98,7 @@ impl BootstrapManager { } /// See module level documentation for details -pub async fn start_bootstrap_server( +pub fn start_bootstrap_server( consensus_controller: Box, network_command_sender: NetworkCommandSender, final_state: Arc>, diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index e4b7629ed0c..3ea6fe98add 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -4,7 +4,7 @@ use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; use socket2 as _; use std::io; -use std::net::SocketAddr; +use std::net::{SocketAddr, TcpListener, TcpStream}; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; @@ -43,18 +43,22 @@ impl BSListener for MockListener { "MockListener accept channel from Establisher closed".to_string(), ) })?; - let duplex_controller = std::net::TcpListener::bind("localhost:0").unwrap(); - let duplex_mock = Duplex::connect(duplex_controller.local_addr().unwrap()) - .await - .unwrap(); + let duplex_controller = TcpListener::bind("localhost:0").unwrap(); + let duplex_mock = TcpStream::connect(duplex_controller.local_addr().unwrap()).unwrap(); let (duplex_controller, addr) = duplex_controller.accept().unwrap(); - sender.send(duplex_mock).map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener accept return oneshot channel to Establisher closed".to_string(), - ) - })?; + // Tokio `from_std` have non-blocking Tcp objects as a requirement + duplex_mock.set_nonblocking(true).unwrap(); + duplex_controller.set_nonblocking(true).unwrap(); + + sender + .send(Duplex::from_std(duplex_mock).unwrap()) + .map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + "MockListener accept return oneshot channel to Establisher closed".to_string(), + ) + })?; Ok((Duplex::from_std(duplex_controller).unwrap(), addr)) } diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index f384a945b39..74df3053b55 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -157,7 +157,6 @@ async fn test_bootstrap_server() { keypair.clone(), Version::from_str("TEST.1.10").unwrap(), ) - .await .unwrap() .unwrap(); diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 85e455210e7..1002f625dc1 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -512,7 +512,6 @@ async fn launch( private_key, *VERSION, ) - .await .unwrap(); let api_config: APIConfig = APIConfig { From a637e002f852c0e950503fe77a6a3c28d9d60133 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 13:24:45 +0100 Subject: [PATCH 14/83] Remove the async Tcp stuff from mock connector --- massa-bootstrap/src/tests/mock_establisher.rs | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 3ea6fe98add..ece38cc6342 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -73,9 +73,13 @@ pub struct MockConnector { #[async_trait] impl BSConnector for MockConnector { async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { - let duplex_mock = DuplexListener::bind(addr).await.unwrap(); - let duplex_controller = Duplex::connect(addr).await.unwrap(); - let duplex_mock = duplex_mock.accept().await.unwrap(); + let duplex_mock = TcpListener::bind(addr).unwrap(); + let duplex_controller = TcpStream::connect(addr).unwrap(); + let duplex_mock = duplex_mock.accept().unwrap(); + + // Requirement of tokio from_std things + duplex_controller.set_nonblocking(true).unwrap(); + duplex_mock.0.set_nonblocking(true).unwrap(); // Used to see if the connection is accepted let (accept_tx, accept_rx) = oneshot::channel::(); @@ -83,7 +87,7 @@ impl BSConnector for MockConnector { // send new connection to mock timeout(self.timeout_duration.to_duration(), async move { self.connection_connector_tx - .send((duplex_mock.0, addr, accept_tx)) + .send((Duplex::from_std(duplex_mock.0).unwrap(), addr, accept_tx)) .await .map_err(|_err| { io::Error::new( @@ -92,7 +96,7 @@ impl BSConnector for MockConnector { ) })?; if accept_rx.await.expect("mock accept_tx disappeared") { - Ok(duplex_controller) + Ok(Duplex::from_std(duplex_controller).unwrap()) } else { Err(io::Error::new( io::ErrorKind::ConnectionRefused, From 725902f13f392777e7722b6caf909498fbeca983 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 15:49:00 +0100 Subject: [PATCH 15/83] Use less async channels. Use Arc as signaller --- massa-bootstrap/src/tests/mock_establisher.rs | 37 +++++++++++-------- massa-bootstrap/src/tests/scenarios.rs | 12 ++++-- 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index ece38cc6342..afe57504b58 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -5,17 +5,19 @@ use massa_time::MassaTime; use socket2 as _; use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; -use crate::establisher::{BSConnector, BSEstablisher, BSListener, Duplex, DuplexListener}; +use crate::establisher::{BSConnector, BSEstablisher, BSListener, Duplex}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = crossbeam::channel::bounded::<(SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); let (connection_connector_tx, connection_connector_rx) = - mpsc::channel::<(Duplex, SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); + mpsc::channel::<(Duplex, SocketAddr, Arc)>(CHANNEL_SIZE); ( MockEstablisher { @@ -66,7 +68,7 @@ impl BSListener for MockListener { #[derive(Debug)] pub struct MockConnector { - connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, + connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, Arc)>, timeout_duration: MassaTime, } @@ -82,27 +84,30 @@ impl BSConnector for MockConnector { duplex_mock.0.set_nonblocking(true).unwrap(); // Used to see if the connection is accepted - let (accept_tx, accept_rx) = oneshot::channel::(); + let waker = Arc::new(AtomicBool::from(false)); + let provided_waker = Arc::clone(&waker); // send new connection to mock timeout(self.timeout_duration.to_duration(), async move { self.connection_connector_tx - .send((Duplex::from_std(duplex_mock.0).unwrap(), addr, accept_tx)) + .send(( + Duplex::from_std(duplex_mock.0).unwrap(), + addr, + provided_waker, + )) .await .map_err(|_err| { io::Error::new( io::ErrorKind::Other, "MockConnector connect channel to Establisher closed".to_string(), ) - })?; - if accept_rx.await.expect("mock accept_tx disappeared") { - Ok(Duplex::from_std(duplex_controller).unwrap()) - } else { - Err(io::Error::new( - io::ErrorKind::ConnectionRefused, - "mock refused the connection".to_string(), - )) + }) + .unwrap(); + // this will lock the system up with a std::thread... + while !waker.load(Ordering::Relaxed) { + tokio::task::yield_now().await; } + Ok(Duplex::from_std(duplex_controller).unwrap()) }) .await .map_err(|_| { @@ -118,7 +123,7 @@ impl BSConnector for MockConnector { pub struct MockEstablisher { connection_listener_rx: Option)>>, - connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, oneshot::Sender)>, + connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, Arc)>, } impl BSEstablisher for MockEstablisher { @@ -147,7 +152,7 @@ impl BSEstablisher for MockEstablisher { pub struct MockEstablisherInterface { connection_listener_tx: Option)>>, - connection_connector_rx: mpsc::Receiver<(Duplex, SocketAddr, oneshot::Sender)>, + connection_connector_rx: mpsc::Receiver<(Duplex, SocketAddr, Arc)>, } impl MockEstablisherInterface { @@ -177,7 +182,7 @@ impl MockEstablisherInterface { pub async fn wait_connection_attempt_from_controller( &mut self, - ) -> io::Result<(Duplex, SocketAddr, oneshot::Sender)> { + ) -> io::Result<(Duplex, SocketAddr, Arc)> { self.connection_connector_rx.recv().await.ok_or_else(|| { io::Error::new( io::ErrorKind::Other, diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 74df3053b55..277d483c1e9 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -46,7 +46,12 @@ use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; use serial_test::serial; -use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; +use std::{ + path::PathBuf, + str::FromStr, + sync::{atomic::Ordering, Arc}, + time::Duration, +}; use tempfile::TempDir; use tokio::sync::mpsc; @@ -176,7 +181,7 @@ async fn test_bootstrap_server() { }); // accept connection attempt from remote - let (remote_rw, conn_addr, resp) = tokio::time::timeout( + let (remote_rw, conn_addr, waker) = tokio::time::timeout( std::time::Duration::from_millis(1000), remote_interface.wait_connection_attempt_from_controller(), ) @@ -188,8 +193,7 @@ async fn test_bootstrap_server() { conn_addr, expect_conn_addr, "client connected to wrong bootstrap ip" ); - resp.send(true) - .expect("could not send connection accept to remote"); + waker.store(true, Ordering::Relaxed); // connect to bootstrap let remote_addr = std::net::SocketAddr::from_str("82.245.72.98:10000").unwrap(); // not checked From 0236535cd59b9d1d44b6e2bf6f390b4b4dd84ee6 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 16:36:12 +0100 Subject: [PATCH 16/83] Fewer async channels --- massa-bootstrap/src/establisher.rs | 4 +- massa-bootstrap/src/tests/mock_establisher.rs | 97 ++++++++++--------- massa-bootstrap/src/tests/scenarios.rs | 43 +++++--- massa-network-exports/src/establisher.rs | 4 +- 4 files changed, 83 insertions(+), 65 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 18dd5d5226c..b9f7f9f684d 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -96,7 +96,9 @@ impl BSEstablisher for DefaultEstablisher { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - Ok(DefaultListener(DuplexListener::from_std(socket.into())?)) + let socket: std::net::TcpListener = socket.into(); + socket.set_nonblocking(true).unwrap(); + Ok(DefaultListener(DuplexListener::from_std(socket)?)) } /// Get the connector with associated timeout diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index afe57504b58..ce31affdfbb 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,5 +1,6 @@ // Copyright (c) 2022 MASSA LABS use async_trait::async_trait; +use crossbeam::channel::{bounded, Receiver, Sender}; use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; use socket2 as _; @@ -7,17 +8,16 @@ use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use tokio::sync::{mpsc, oneshot}; use tokio::time::timeout; use crate::establisher::{BSConnector, BSEstablisher, BSListener, Duplex}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = - crossbeam::channel::bounded::<(SocketAddr, oneshot::Sender)>(CHANNEL_SIZE); + bounded::<(SocketAddr, Sender)>(CHANNEL_SIZE); let (connection_connector_tx, connection_connector_rx) = - mpsc::channel::<(Duplex, SocketAddr, Arc)>(CHANNEL_SIZE); + bounded::<(TcpStream, SocketAddr, Arc)>(CHANNEL_SIZE); ( MockEstablisher { @@ -33,18 +33,20 @@ pub fn new() -> (MockEstablisher, MockEstablisherInterface) { #[derive(Debug)] pub struct MockListener { - connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, oneshot::Sender)>, // (controller, mock) + connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, Sender)>, // (controller, mock) } #[async_trait] impl BSListener for MockListener { async fn accept(&mut self) -> std::io::Result<(Duplex, SocketAddr)> { + dbg!("accept recving"); let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener accept channel from Establisher closed".to_string(), ) })?; + dbg!("accept received"); let duplex_controller = TcpListener::bind("localhost:0").unwrap(); let duplex_mock = TcpStream::connect(duplex_controller.local_addr().unwrap()).unwrap(); let (duplex_controller, addr) = duplex_controller.accept().unwrap(); @@ -53,14 +55,15 @@ impl BSListener for MockListener { duplex_mock.set_nonblocking(true).unwrap(); duplex_controller.set_nonblocking(true).unwrap(); - sender - .send(Duplex::from_std(duplex_mock).unwrap()) - .map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener accept return oneshot channel to Establisher closed".to_string(), - ) - })?; + dbg!("accept sending mock", &duplex_mock); + sender.send(duplex_mock).map_err(|_| { + io::Error::new( + io::ErrorKind::Other, + "MockListener accept return \"oneshot\" channel to Establisher closed".to_string(), + ) + })?; + dbg!("accept mock sent"); + dbg!("accept returning", &duplex_controller); Ok((Duplex::from_std(duplex_controller).unwrap(), addr)) } @@ -68,13 +71,14 @@ impl BSListener for MockListener { #[derive(Debug)] pub struct MockConnector { - connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, Arc)>, + connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, timeout_duration: MassaTime, } #[async_trait] impl BSConnector for MockConnector { async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { + dbg!("connect"); let duplex_mock = TcpListener::bind(addr).unwrap(); let duplex_controller = TcpStream::connect(addr).unwrap(); let duplex_mock = duplex_mock.accept().unwrap(); @@ -87,15 +91,12 @@ impl BSConnector for MockConnector { let waker = Arc::new(AtomicBool::from(false)); let provided_waker = Arc::clone(&waker); - // send new connection to mock - timeout(self.timeout_duration.to_duration(), async move { - self.connection_connector_tx - .send(( - Duplex::from_std(duplex_mock.0).unwrap(), - addr, - provided_waker, - )) - .await + let sender = self.connection_connector_tx.clone(); + let send = std::thread::spawn(move || { + // send new connection to mock + dbg!("connect thread sending"); + sender + .send((duplex_mock.0, addr, provided_waker)) .map_err(|_err| { io::Error::new( io::ErrorKind::Other, @@ -103,27 +104,24 @@ impl BSConnector for MockConnector { ) }) .unwrap(); - // this will lock the system up with a std::thread... - while !waker.load(Ordering::Relaxed) { - tokio::task::yield_now().await; - } - Ok(Duplex::from_std(duplex_controller).unwrap()) - }) - .await - .map_err(|_| { - io::Error::new( - io::ErrorKind::TimedOut, - "MockConnector connection attempt timed out".to_string(), - ) - })? + dbg!("connect thread sent"); + }); + dbg!("sent spun up"); + // this will lock the system up with a std::thread... + while !waker.load(Ordering::Relaxed) { + tokio::task::yield_now().await; + } + dbg!("flag is true"); + send.join().unwrap(); + dbg!("joined"); + Ok(Duplex::from_std(duplex_controller).unwrap()) } } #[derive(Debug)] pub struct MockEstablisher { - connection_listener_rx: - Option)>>, - connection_connector_tx: mpsc::Sender<(Duplex, SocketAddr, Arc)>, + connection_listener_rx: Option)>>, + connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, } impl BSEstablisher for MockEstablisher { @@ -150,40 +148,45 @@ impl BSEstablisher for MockEstablisher { } pub struct MockEstablisherInterface { - connection_listener_tx: - Option)>>, - connection_connector_rx: mpsc::Receiver<(Duplex, SocketAddr, Arc)>, + connection_listener_tx: Option)>>, + connection_connector_rx: Receiver<(TcpStream, SocketAddr, Arc)>, } impl MockEstablisherInterface { - pub async fn connect_to_controller(&self, addr: &SocketAddr) -> io::Result { + pub async fn connect_to_controller(&self, addr: &SocketAddr) -> io::Result { + dbg!("conn ctrl"); let sender = self.connection_listener_tx.as_ref().ok_or_else(|| { io::Error::new( io::ErrorKind::Other, "mock connect_to_controller_listener channel not initialized".to_string(), ) })?; - let (response_tx, response_rx) = oneshot::channel::(); + let (response_tx, response_rx) = bounded::(1); + dbg!("conn ctrl: sending resp_tx", addr); sender.send((*addr, response_tx)).map_err(|_err| { io::Error::new( io::ErrorKind::Other, "mock connect_to_controller_listener channel to listener closed".to_string(), ) })?; - let duplex_mock = response_rx.await.map_err(|_| { + dbg!("conn ctrl: sent resp_tx"); + dbg!("conn ctrl: getting duplex mokc"); + let duplex_mock = response_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener connect_to_controller_listener channel from listener closed" .to_string(), ) })?; + dbg!("conn ctrl: got", &duplex_mock); Ok(duplex_mock) } - pub async fn wait_connection_attempt_from_controller( + pub fn wait_connection_attempt_from_controller( &mut self, - ) -> io::Result<(Duplex, SocketAddr, Arc)> { - self.connection_connector_rx.recv().await.ok_or_else(|| { + ) -> io::Result<(TcpStream, SocketAddr, Arc)> { + dbg!("wait conn attempt"); + self.connection_connector_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener get_connect_stream channel from connector closed".to_string(), diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 277d483c1e9..fec5ceaa0d3 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -7,10 +7,13 @@ use super::{ get_random_ledger_changes, wait_network_command, }, }; -use crate::tests::tools::{ - get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, -}; use crate::BootstrapConfig; +use crate::{ + establisher::Duplex, + tests::tools::{ + get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, + }, +}; use crate::{ get_state, start_bootstrap_server, tests::tools::{assert_eq_bootstrap_graph, get_bootstrap_config}, @@ -165,6 +168,7 @@ async fn test_bootstrap_server() { .unwrap() .unwrap(); + dbg!("test: launching get state"); // launch the get_state process let (remote_establisher, mut remote_interface) = mock_establisher::new(); let get_state_h = tokio::spawn(async move { @@ -179,21 +183,23 @@ async fn test_bootstrap_server() { .await .unwrap() }); + dbg!("test: get state launched, waiting on connection"); // accept connection attempt from remote - let (remote_rw, conn_addr, waker) = tokio::time::timeout( - std::time::Duration::from_millis(1000), - remote_interface.wait_connection_attempt_from_controller(), - ) - .await - .expect("timeout waiting for connection attempt from remote") - .expect("error receiving connection attempt from remote"); - let expect_conn_addr = bootstrap_config.bootstrap_list[0].0; - assert_eq!( - conn_addr, expect_conn_addr, - "client connected to wrong bootstrap ip" - ); - waker.store(true, Ordering::Relaxed); + let remote_rw = std::thread::spawn(move || { + dbg!("conn wait thread"); + let (remote_rw, conn_addr, waker) = remote_interface + .wait_connection_attempt_from_controller() + .expect("timeout waiting for connection attempt from remote"); + dbg!("test: got conn", &conn_addr); + let expect_conn_addr = bootstrap_config.bootstrap_list[0].0; + assert_eq!( + conn_addr, expect_conn_addr, + "client connected to wrong bootstrap ip" + ); + waker.store(true, Ordering::Relaxed); + remote_rw + }); // connect to bootstrap let remote_addr = std::net::SocketAddr::from_str("82.245.72.98:10000").unwrap(); // not checked @@ -206,7 +212,12 @@ async fn test_bootstrap_server() { .expect("could not connect to bootstrap"); // launch bridge + bootstrap_rw.set_nonblocking(true).unwrap(); + let bootstrap_rw = Duplex::from_std(bootstrap_rw).unwrap(); let bridge = tokio::spawn(async move { + let remote_rw = remote_rw.join().unwrap(); + remote_rw.set_nonblocking(true).unwrap(); + let remote_rw = Duplex::from_std(remote_rw).unwrap(); bridge_mock_streams(remote_rw, bootstrap_rw).await; }); diff --git a/massa-network-exports/src/establisher.rs b/massa-network-exports/src/establisher.rs index 7f8dcfe90de..91a637d97de 100644 --- a/massa-network-exports/src/establisher.rs +++ b/massa-network-exports/src/establisher.rs @@ -93,7 +93,9 @@ mod types { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - Ok(DefaultListener(TcpListener::from_std(socket.into())?)) + let socket: std::net::TcpListener = socket.into(); + socket.set_nonblocking(true).unwrap(); + Ok(DefaultListener(TcpListener::from_std(socket)?)) } /// Get the connector with associated timeout From b32246bbe8efb19a2c75ac5dc8fa45cfd2750ae9 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 17:03:51 +0100 Subject: [PATCH 17/83] So close to not using tokio... --- massa-bootstrap/src/client.rs | 5 ++-- massa-bootstrap/src/establisher.rs | 23 ++++++++++--------- massa-bootstrap/src/server.rs | 6 +++-- massa-bootstrap/src/tests/mock_establisher.rs | 8 +++---- 4 files changed, 23 insertions(+), 19 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 8489c710459..69dac1fb7b0 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -17,7 +17,7 @@ use tracing::{debug, info, warn}; use crate::{ client_binder::BootstrapClientBinder, error::BootstrapError, - establisher::{BSConnector, BSEstablisher}, + establisher::{BSConnector, BSEstablisher, Duplex}, messages::{BootstrapClientMessage, BootstrapServerMessage}, settings::IpType, BootstrapConfig, GlobalBootstrapState, @@ -380,8 +380,9 @@ async fn connect_to_server( // connect let mut connector = establisher.get_connector(bootstrap_config.connect_timeout)?; let socket = connector.connect(*addr).await?; + socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( - socket, + Duplex::from_std(socket).unwrap(), *pub_key, bootstrap_config.into(), )) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index b9f7f9f684d..35ada05fb27 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -1,7 +1,10 @@ // Copyright (c) 2022 MASSA LABS use async_trait::async_trait; use massa_time::MassaTime; -use std::{io, net::SocketAddr}; +use std::{ + io, + net::{SocketAddr, TcpListener, TcpStream}, +}; /// duplex connection pub type Duplex = tokio::net::TcpStream; @@ -12,13 +15,13 @@ pub type DuplexListener = tokio::net::TcpListener; #[async_trait] /// Specifies a common interface that can be used by standard, or mockers pub trait BSListener { - async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)>; + async fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; } #[async_trait] /// Specifies a common interface that can be used by standard, or mockers pub trait BSConnector { - async fn connect(&mut self, addr: SocketAddr) -> io::Result; + async fn connect(&mut self, addr: SocketAddr) -> io::Result; } /// Specifies a common interface that can be used by standard, or mockers @@ -35,14 +38,14 @@ pub trait BSEstablisher { /// The listener we are using #[derive(Debug)] -pub struct DefaultListener(DuplexListener); +pub struct DefaultListener(TcpListener); #[async_trait] impl BSListener for DefaultListener { /// Accepts a new incoming connection from this listener. - async fn accept(&mut self) -> io::Result<(Duplex, SocketAddr)> { + async fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)> { // accept - let (sock, mut remote_addr) = self.0.accept().await?; + let (sock, mut remote_addr) = self.0.accept()?; // normalize address remote_addr.set_ip(remote_addr.ip().to_canonical()); Ok((sock, remote_addr)) @@ -58,8 +61,8 @@ impl BSConnector for DefaultConnector { /// /// # Argument /// * `addr`: `SocketAddr` we are trying to connect to. - async fn connect(&mut self, addr: SocketAddr) -> io::Result { - match tokio::time::timeout(self.0.to_duration(), Duplex::connect(addr)).await { + async fn connect(&mut self, addr: SocketAddr) -> io::Result { + match tokio::time::timeout(self.0.to_duration(), async { TcpStream::connect(addr) }).await { Ok(Ok(sock)) => Ok(sock), Ok(Err(e)) => Err(e), Err(e) => Err(io::Error::new(io::ErrorKind::TimedOut, e)), @@ -96,9 +99,7 @@ impl BSEstablisher for DefaultEstablisher { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - let socket: std::net::TcpListener = socket.into(); - socket.set_nonblocking(true).unwrap(); - Ok(DefaultListener(DuplexListener::from_std(socket)?)) + Ok(DefaultListener(socket.into())) } /// Get the connector with associated timeout diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index b0f50894650..1cbf8fcce7a 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -254,8 +254,10 @@ impl BootstrapServer<'_> { listener_tx: crossbeam::channel::Sender, ) -> Result, Box> { loop { - let msg = listener.accept().await.map_err(BootstrapError::IoError)?; - match listener_tx.send(msg) { + let (msg, addr) = listener.accept().await.map_err(BootstrapError::IoError)?; + msg.set_nonblocking(true).unwrap(); + let msg = Duplex::from_std(msg).unwrap(); + match listener_tx.send((msg, addr)) { Ok(_) => continue, Err(SendError((dplx, remote_addr))) => { warn!( diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index ce31affdfbb..6c2d9ac76a6 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -38,7 +38,7 @@ pub struct MockListener { #[async_trait] impl BSListener for MockListener { - async fn accept(&mut self) -> std::io::Result<(Duplex, SocketAddr)> { + async fn accept(&mut self) -> std::io::Result<(TcpStream, SocketAddr)> { dbg!("accept recving"); let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( @@ -65,7 +65,7 @@ impl BSListener for MockListener { dbg!("accept mock sent"); dbg!("accept returning", &duplex_controller); - Ok((Duplex::from_std(duplex_controller).unwrap(), addr)) + Ok((duplex_controller, addr)) } } @@ -77,7 +77,7 @@ pub struct MockConnector { #[async_trait] impl BSConnector for MockConnector { - async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { + async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { dbg!("connect"); let duplex_mock = TcpListener::bind(addr).unwrap(); let duplex_controller = TcpStream::connect(addr).unwrap(); @@ -114,7 +114,7 @@ impl BSConnector for MockConnector { dbg!("flag is true"); send.join().unwrap(); dbg!("joined"); - Ok(Duplex::from_std(duplex_controller).unwrap()) + Ok(duplex_controller) } } From a7f3c51472e406a5ffd294c10b3ffd0052dee3cd Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 14 Mar 2023 17:05:36 +0100 Subject: [PATCH 18/83] accept method now synchronous!!! --- massa-bootstrap/src/establisher.rs | 6 ++---- massa-bootstrap/src/server.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 3 +-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 35ada05fb27..fe1cd781250 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -12,10 +12,9 @@ pub type Duplex = tokio::net::TcpStream; /// duplex connection pub type DuplexListener = tokio::net::TcpListener; -#[async_trait] /// Specifies a common interface that can be used by standard, or mockers pub trait BSListener { - async fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; + fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; } #[async_trait] @@ -40,10 +39,9 @@ pub trait BSEstablisher { #[derive(Debug)] pub struct DefaultListener(TcpListener); -#[async_trait] impl BSListener for DefaultListener { /// Accepts a new incoming connection from this listener. - async fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)> { + fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)> { // accept let (sock, mut remote_addr) = self.0.accept()?; // normalize address diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 1cbf8fcce7a..3f236279b46 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -254,7 +254,7 @@ impl BootstrapServer<'_> { listener_tx: crossbeam::channel::Sender, ) -> Result, Box> { loop { - let (msg, addr) = listener.accept().await.map_err(BootstrapError::IoError)?; + let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; msg.set_nonblocking(true).unwrap(); let msg = Duplex::from_std(msg).unwrap(); match listener_tx.send((msg, addr)) { diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 6c2d9ac76a6..c2a519edf70 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -36,9 +36,8 @@ pub struct MockListener { connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, Sender)>, // (controller, mock) } -#[async_trait] impl BSListener for MockListener { - async fn accept(&mut self) -> std::io::Result<(TcpStream, SocketAddr)> { + fn accept(&mut self) -> std::io::Result<(TcpStream, SocketAddr)> { dbg!("accept recving"); let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( From e8d53f10aae7cd934de2c6e9068e24d4b3a9dcda Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 15 Mar 2023 14:29:09 +0100 Subject: [PATCH 19/83] make connect synchronous --- massa-bootstrap/src/client.rs | 8 +++----- massa-bootstrap/src/establisher.rs | 12 +++--------- massa-bootstrap/src/tests/mock_establisher.rs | 7 +++---- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 69dac1fb7b0..9cad399d426 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -371,7 +371,7 @@ async fn send_client_message( } } -async fn connect_to_server( +fn connect_to_server( establisher: &mut impl BSEstablisher, bootstrap_config: &BootstrapConfig, addr: &SocketAddr, @@ -379,7 +379,7 @@ async fn connect_to_server( ) -> Result { // connect let mut connector = establisher.get_connector(bootstrap_config.connect_timeout)?; - let socket = connector.connect(*addr).await?; + let socket = connector.connect(*addr)?; socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( Duplex::from_std(socket).unwrap(), @@ -491,9 +491,7 @@ pub async fn get_state( bootstrap_config, addr, &node_id.get_public_key(), - ) - .await - { + ) { Ok(mut client) => { match bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version) .await // cancellable diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index f7cf2b6a06b..e6961547497 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -17,10 +17,9 @@ pub trait BSListener { fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; } -#[async_trait] /// Specifies a common interface that can be used by standard, or mockers pub trait BSConnector { - async fn connect(&mut self, addr: SocketAddr) -> io::Result; + fn connect(&mut self, addr: SocketAddr) -> io::Result; } /// Specifies a common interface that can be used by standard, or mockers @@ -53,18 +52,13 @@ impl BSListener for DefaultListener { #[derive(Debug)] pub struct DefaultConnector(MassaTime); -#[async_trait] impl BSConnector for DefaultConnector { /// Tries to connect to address /// /// # Argument /// * `addr`: `SocketAddr` we are trying to connect to. - async fn connect(&mut self, addr: SocketAddr) -> io::Result { - match tokio::time::timeout(self.0.to_duration(), async { TcpStream::connect(addr) }).await { - Ok(Ok(sock)) => Ok(sock), - Ok(Err(e)) => Err(e), - Err(e) => Err(io::Error::new(io::ErrorKind::TimedOut, e)), - } + fn connect(&mut self, addr: SocketAddr) -> io::Result { + TcpStream::connect_timeout(&addr, self.0.to_duration()) } } diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index dd84e3649ee..9acc763c702 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -73,9 +73,8 @@ pub struct MockConnector { timeout_duration: MassaTime, } -#[async_trait] impl BSConnector for MockConnector { - async fn connect(&mut self, addr: SocketAddr) -> std::io::Result { + fn connect(&mut self, addr: SocketAddr) -> std::io::Result { dbg!("connect"); let duplex_mock = TcpListener::bind(addr).unwrap(); let duplex_controller = TcpStream::connect(addr).unwrap(); @@ -105,9 +104,9 @@ impl BSConnector for MockConnector { dbg!("connect thread sent"); }); dbg!("sent spun up"); - // this will lock the system up with a std::thread... + while !waker.load(Ordering::Relaxed) { - tokio::task::yield_now().await; + std::thread::yield_now(); } dbg!("flag is true"); send.join().unwrap(); From 2cc42561e44215c953fc47413bf6f4640602b514 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 15 Mar 2023 14:33:54 +0100 Subject: [PATCH 20/83] Some cleanup --- Cargo.lock | 1 - massa-bootstrap/Cargo.toml | 1 - massa-bootstrap/src/establisher.rs | 4 ---- massa-bootstrap/src/tests/mock_establisher.rs | 8 +------- 4 files changed, 1 insertion(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04b997118ce..572c8f573af 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2431,7 +2431,6 @@ name = "massa_bootstrap" version = "0.1.0" dependencies = [ "async-speed-limit", - "async-trait", "bitvec", "crossbeam", "displaydoc", diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 272e9794d60..90a89c272fd 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -25,7 +25,6 @@ tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" crossbeam = "0.8.2" -async-trait = "0.1.64" # custom modules massa_async_pool = { path = "../massa-async-pool" } diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index e6961547497..4a0d7aad63a 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -1,5 +1,4 @@ // Copyright (c) 2022 MASSA LABS -use async_trait::async_trait; use massa_time::MassaTime; use std::{ io, @@ -9,9 +8,6 @@ use std::{ /// duplex connection pub type Duplex = tokio::net::TcpStream; -/// Listener used to establish a Duplex -pub type DuplexListener = tokio::net::TcpListener; - /// Specifies a common interface that can be used by standard, or mockers pub trait BSListener { fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 9acc763c702..1bea10598f2 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,5 +1,4 @@ // Copyright (c) 2022 MASSA LABS -use async_trait::async_trait; use crossbeam::channel::{bounded, Receiver, Sender}; use massa_models::config::CHANNEL_SIZE; use massa_time::MassaTime; @@ -8,9 +7,8 @@ use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use tokio::time::timeout; -use crate::establisher::{BSConnector, BSEstablisher, BSListener, Duplex}; +use crate::establisher::{BSConnector, BSEstablisher, BSListener}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -80,10 +78,6 @@ impl BSConnector for MockConnector { let duplex_controller = TcpStream::connect(addr).unwrap(); let duplex_mock = duplex_mock.accept().unwrap(); - // Requirement of tokio from_std things - duplex_controller.set_nonblocking(true).unwrap(); - duplex_mock.0.set_nonblocking(true).unwrap(); - // Used to see if the connection is accepted let waker = Arc::new(AtomicBool::from(false)); let provided_waker = Arc::clone(&waker); From fcdf28e8499eb000507e363d649f51d6c869c6aa Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 15 Mar 2023 15:01:21 +0100 Subject: [PATCH 21/83] Clear clippy lint --- massa-bootstrap/src/client.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 9cad399d426..3949196bf91 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -376,10 +376,12 @@ fn connect_to_server( bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, -) -> Result { +) -> Result> { // connect - let mut connector = establisher.get_connector(bootstrap_config.connect_timeout)?; - let socket = connector.connect(*addr)?; + let mut connector = establisher + .get_connector(bootstrap_config.connect_timeout) + .map_err(|e| Box::new(e.into()))?; + let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( Duplex::from_std(socket).unwrap(), From 573252984921de6bf70ffbbe9df993518f2a45e6 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 15 Mar 2023 15:09:33 +0100 Subject: [PATCH 22/83] Fix upstream merge error --- massa-bootstrap/src/establisher.rs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 4a0d7aad63a..b181f6dd39f 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -78,15 +78,23 @@ impl BSEstablisher for DefaultEstablisher { /// * `addr`: `SocketAddr` we want to bind to. fn get_listener(&mut self, addr: SocketAddr) -> io::Result { // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag - let socket = socket2::Socket::new(socket2::Domain::IPV6, socket2::Type::STREAM, None)?; + // This is needed to get the same behavior on all OS + // However, if IPv6 is disabled system-wide, you may need to bind to an IPv4 address instead. + let domain = match addr.is_ipv4() { + true => socket2::Domain::IPV4, + _ => socket2::Domain::IPV6, + }; - socket.set_only_v6(false)?; + let socket = socket2::Socket::new(domain, socket2::Type::STREAM, None)?; + + if addr.is_ipv6() { + socket.set_only_v6(false)?; + } socket.set_nonblocking(true)?; socket.bind(&addr.into())?; // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - Ok(DefaultListener(socket.into())) } From bf7f89fc0748fd58a54dcf1646a85ecdf99837c3 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 15 Mar 2023 16:12:38 +0100 Subject: [PATCH 23/83] more cleanup --- massa-bootstrap/src/client.rs | 1 + massa-bootstrap/src/tests/mock_establisher.rs | 4 +--- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 3949196bf91..5a49710cdc1 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -384,6 +384,7 @@ fn connect_to_server( let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( + // this from_std will panic if this method doesn't exist within an async runtime... Duplex::from_std(socket).unwrap(), *pub_key, bootstrap_config.into(), diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index 1bea10598f2..f14a25ab830 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -68,7 +68,6 @@ impl BSListener for MockListener { #[derive(Debug)] pub struct MockConnector { connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, - timeout_duration: MassaTime, } impl BSConnector for MockConnector { @@ -128,12 +127,11 @@ impl BSEstablisher for MockEstablisher { }) } - fn get_connector(&mut self, timeout_duration: MassaTime) -> std::io::Result { + fn get_connector(&mut self, _timeout_duration: MassaTime) -> std::io::Result { // create connector stream Ok(MockConnector { connection_connector_tx: self.connection_connector_tx.clone(), - timeout_duration, }) } } From 78459e002b99e9d368f68fe4c142f46918eea416 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 22 Mar 2023 15:11:39 +0100 Subject: [PATCH 24/83] Make listener blocking --- massa-bootstrap/src/establisher.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index b181f6dd39f..f2b6082666d 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -90,7 +90,6 @@ impl BSEstablisher for DefaultEstablisher { if addr.is_ipv6() { socket.set_only_v6(false)?; } - socket.set_nonblocking(true)?; socket.bind(&addr.into())?; // Number of connections to queue, set to the hardcoded value used by tokio From dffa060d5c5bbc780faa4f0f62dcb70cdb847274 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 24 Mar 2023 20:20:55 +0100 Subject: [PATCH 25/83] Make the Duplex a trait, thus mockable --- massa-bootstrap/src/client.rs | 16 +++---- massa-bootstrap/src/client_binder.rs | 10 ++-- massa-bootstrap/src/establisher.rs | 8 +++- massa-bootstrap/src/server.rs | 65 ++++++++++++++------------ massa-bootstrap/src/server_binder.rs | 11 ++--- massa-bootstrap/src/tests/binders.rs | 4 +- massa-bootstrap/src/tests/scenarios.rs | 17 +++---- massa-bootstrap/src/tests/tools.rs | 4 +- 8 files changed, 68 insertions(+), 67 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 5a49710cdc1..ac3677717e1 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -26,9 +26,9 @@ use crate::{ /// This function will send the starting point to receive a stream of the ledger and will receive and process each part until receive a `BootstrapServerMessage::FinalStateFinished` message from the server. /// `next_bootstrap_message` passed as parameter must be `BootstrapClientMessage::AskFinalStatePart` enum variant. /// `next_bootstrap_message` will be updated after receiving each part so that in case of connection lost we can restart from the last message we processed. -async fn stream_final_state_and_consensus( +async fn stream_final_state_and_consensus( cfg: &BootstrapConfig, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, global_bootstrap_state: &mut GlobalBootstrapState, ) -> Result<(), BootstrapError> { @@ -194,9 +194,9 @@ async fn stream_final_state_and_consensus( /// Gets the state from a bootstrap server (internal private function) /// needs to be CANCELLABLE -async fn bootstrap_from_server( +async fn bootstrap_from_server( cfg: &BootstrapConfig, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, global_bootstrap_state: &mut GlobalBootstrapState, our_version: Version, @@ -352,9 +352,9 @@ async fn bootstrap_from_server( Ok(()) } -async fn send_client_message( +async fn send_client_message( message_to_send: &BootstrapClientMessage, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, write_timeout: Duration, read_timeout: Duration, error: &str, @@ -376,7 +376,7 @@ fn connect_to_server( bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, -) -> Result> { +) -> Result, Box> { // connect let mut connector = establisher .get_connector(bootstrap_config.connect_timeout) @@ -385,7 +385,7 @@ fn connect_to_server( socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( // this from_std will panic if this method doesn't exist within an async runtime... - Duplex::from_std(socket).unwrap(), + tokio::net::TcpStream::from_std(socket).unwrap(), *pub_key, bootstrap_config.into(), )) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 57df199d13b..a34a4de94cc 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -19,24 +19,24 @@ use tokio::io::AsyncReadExt; use tokio::io::AsyncWriteExt; /// Bootstrap client binder -pub struct BootstrapClientBinder { +pub struct BootstrapClientBinder { // max_bootstrap_message_size: u32, size_field_len: usize, remote_pubkey: PublicKey, - duplex: Resource, + duplex: Resource, prev_message: Option, version_serializer: VersionSerializer, cfg: BootstrapClientConfig, } -impl BootstrapClientBinder { +impl BootstrapClientBinder { /// Creates a new `WriteBinder`. /// /// # Argument /// * duplex: duplex stream. /// * limit: limit max bytes per second (up and down) #[allow(clippy::too_many_arguments)] - pub fn new(duplex: Duplex, remote_pubkey: PublicKey, cfg: BootstrapClientConfig) -> Self { + pub fn new(duplex: D, remote_pubkey: PublicKey, cfg: BootstrapClientConfig) -> Self { let size_field_len = u32::be_bytes_min_length(cfg.max_bootstrap_message_size); BootstrapClientBinder { size_field_len, @@ -47,9 +47,7 @@ impl BootstrapClientBinder { cfg, } } -} -impl BootstrapClientBinder { /// Performs a handshake. Should be called after connection /// NOT cancel-safe pub async fn handshake(&mut self, version: Version) -> Result<(), BootstrapError> { diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index f2b6082666d..fa190f780f9 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -6,7 +6,13 @@ use std::{ }; /// duplex connection -pub type Duplex = tokio::net::TcpStream; +pub trait Duplex: +// static because need to send between threads :( + 'static + Send + Sync + tokio::io::AsyncReadExt + tokio::io::AsyncWriteExt + std::marker::Unpin +{ +} + +impl Duplex for tokio::net::TcpStream {} /// Specifies a common interface that can be used by standard, or mockers pub trait BSListener { diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 3f236279b46..82a44642386 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -50,7 +50,10 @@ use std::{ thread, time::{Duration, Instant}, }; -use tokio::runtime::{self, Handle, Runtime}; +use tokio::{ + net::TcpStream, + runtime::{self, Handle, Runtime}, +}; use tracing::{debug, error, info, warn}; use crate::{ @@ -63,19 +66,19 @@ use crate::{ /// Abstraction layer over data produced by the listener, and transported /// over to the worker via a channel -type BsConn = (Duplex, SocketAddr); +type BsConn = (D, SocketAddr); /// handle on the bootstrap server -pub struct BootstrapManager { +pub struct BootstrapManager { update_handle: thread::JoinHandle>>, // need to preserve the listener handle up to here to prevent it being destroyed - _listen_handle: thread::JoinHandle, Box>>, + _listen_handle: thread::JoinHandle>, Box>>, main_handle: thread::JoinHandle>>, listen_stopper_tx: crossbeam::channel::Sender<()>, update_stopper_tx: crossbeam::channel::Sender<()>, } -impl BootstrapManager { +impl BootstrapManager { /// stop the bootstrap server pub async fn stop(self) -> Result<(), Box> { massa_trace!("bootstrap.lib.stop", {}); @@ -98,7 +101,7 @@ impl BootstrapManager { } /// See module level documentation for details -pub fn start_bootstrap_server( +pub fn start_bootstrap_server( consensus_controller: Box, network_command_sender: NetworkCommandSender, final_state: Arc>, @@ -106,7 +109,7 @@ pub fn start_bootstrap_server( mut establisher: impl BSEstablisher, keypair: KeyPair, version: Version, -) -> Result, Box> { +) -> Result>, Box> { massa_trace!("bootstrap.lib.start_bootstrap_server", {}); let Some(listen_addr) = config.listen_addr else { return Ok(None); @@ -136,7 +139,8 @@ pub fn start_bootstrap_server( .map_err(BootstrapError::IoError)?; // This is the primary interface between the async-listener, and the "sync" worker - let (listener_tx, listener_rx) = crossbeam::channel::bounded::(max_bootstraps * 2); + let (listener_tx, listener_rx) = + crossbeam::channel::bounded::>(max_bootstraps * 2); let white_black_list = SharedWhiteBlackList::new( config.bootstrap_whitelist_path.clone(), @@ -147,7 +151,7 @@ pub fn start_bootstrap_server( let update_handle = thread::Builder::new() .name("wb_list_updater".to_string()) .spawn(move || { - let res = BootstrapServer::run_updater( + let res = BootstrapServer::::run_updater( updater_lists, config.cache_duration.into(), update_stopper_rx, @@ -167,8 +171,8 @@ pub fn start_bootstrap_server( // FIXME: The interface being used shouldn't have `: Send + 'static` as a constraint on the listener assosciated type. // GAT lifetime is likely to remedy this, however. .spawn(move || { - let res = - listen_rt_handle.block_on(BootstrapServer::run_listener(listener, listener_tx)); + let res = listen_rt_handle + .block_on(BootstrapServer::::run_listener(listener, listener_tx)); res }) // the non-builder spawn doesn't return a Result, and documentation states that @@ -207,11 +211,11 @@ pub fn start_bootstrap_server( })) } -struct BootstrapServer<'a> { +struct BootstrapServer<'a, D: Duplex> { consensus_controller: Box, network_command_sender: NetworkCommandSender, final_state: Arc>, - listener_rx: crossbeam::channel::Receiver, + listener_rx: crossbeam::channel::Receiver>, listen_stopper_rx: crossbeam::channel::Receiver<()>, white_black_list: SharedWhiteBlackList<'a>, keypair: KeyPair, @@ -221,7 +225,7 @@ struct BootstrapServer<'a> { bs_server_runtime: Runtime, } -impl BootstrapServer<'_> { +impl BootstrapServer<'_, D> { fn run_updater( mut list: SharedWhiteBlackList<'_>, interval: Duration, @@ -251,12 +255,12 @@ impl BootstrapServer<'_> { /// TODO: Integrate the listener into the bootstrap-main-loop async fn run_listener( mut listener: impl BSListener, - listener_tx: crossbeam::channel::Sender, - ) -> Result, Box> { + listener_tx: crossbeam::channel::Sender>, + ) -> Result>, Box> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; msg.set_nonblocking(true).unwrap(); - let msg = Duplex::from_std(msg).unwrap(); + let msg = TcpStream::from_std(msg).unwrap(); match listener_tx.send((msg, addr)) { Ok(_) => continue, Err(SendError((dplx, remote_addr))) => { @@ -293,7 +297,7 @@ impl BootstrapServer<'_> { // if a stop-signal is received let Some((dplx, remote_addr)) = self.receive_connection(&mut selector).map_err(BootstrapError::GeneralError)? else { break; }; // claim a slot in the max_bootstrap_sessions - let server = BootstrapServerBinder::new( + let server_binding = BootstrapServerBinder::new( dplx, self.keypair.clone(), (&self.bootstrap_config).into(), @@ -301,7 +305,7 @@ impl BootstrapServer<'_> { // check whether incoming peer IP is allowed. if let Err(error_msg) = self.white_black_list.is_ip_allowed(&remote_addr) { - server.close_and_send_error( + server_binding.close_and_send_error( self.bs_server_runtime.handle().clone(), error_msg.to_string(), remote_addr, @@ -334,7 +338,7 @@ impl BootstrapServer<'_> { } // check IP's bootstrap attempt history - if let Err(msg) = BootstrapServer::greedy_client_check( + if let Err(msg) = BootstrapServer::::greedy_client_check( &mut self.ip_hist_map, remote_addr, now, @@ -351,7 +355,7 @@ impl BootstrapServer<'_> { "remote_addr": remote_addr }) }; - server.close_and_send_error( + server_binding.close_and_send_error( self.bs_server_runtime.handle().clone(), msg, remote_addr, @@ -371,11 +375,12 @@ impl BootstrapServer<'_> { let bootstrap_count_token = bootstrap_sessions_counter.clone(); let session_handle = bs_loop_rt.handle().clone(); + let _ = thread::Builder::new() .name(format!("bootstrap thread, peer: {}", remote_addr)) .spawn(move || { run_bootstrap_session( - server, + server_binding, bootstrap_count_token, config, remote_addr, @@ -391,7 +396,7 @@ impl BootstrapServer<'_> { "active_count": Arc::strong_count(&bootstrap_sessions_counter) - 1 }); } else { - server.close_and_send_error( + server_binding.close_and_send_error( self.bs_server_runtime.handle().clone(), "Bootstrap failed because the bootstrap server currently has no slots available.".to_string(), remote_addr, @@ -413,7 +418,7 @@ impl BootstrapServer<'_> { /// - 3.a. double check the stop-signal is absent /// - 3.b. If present, fall-back to the stop behaviour /// - 3.c. If absent, all's clear to rock-n-roll. - fn receive_connection(&self, selector: &mut Select) -> Result, String> { + fn receive_connection(&self, selector: &mut Select) -> Result>, String> { // 1. Block until _something_ is ready let rdy = selector.ready(); @@ -487,8 +492,8 @@ impl BootstrapServer<'_> { /// The arc_counter variable is used as a proxy to keep track the number of active bootstrap /// sessions. #[allow(clippy::too_many_arguments)] -fn run_bootstrap_session( - mut server: BootstrapServerBinder, +fn run_bootstrap_session( + mut server: BootstrapServerBinder, arc_counter: Arc<()>, config: BootstrapConfig, remote_addr: SocketAddr, @@ -550,8 +555,8 @@ fn run_bootstrap_session( } #[allow(clippy::too_many_arguments)] -pub async fn stream_bootstrap_information( - server: &mut BootstrapServerBinder, +pub async fn stream_bootstrap_information( + server: &mut BootstrapServerBinder, final_state: Arc>, consensus_controller: Box, mut last_slot: Option, @@ -754,9 +759,9 @@ pub async fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -async fn manage_bootstrap( +async fn manage_bootstrap( bootstrap_config: &BootstrapConfig, - server: &mut BootstrapServerBinder, + server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 9e96118fcb9..c779b97faf0 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -26,7 +26,7 @@ use tokio::time::error::Elapsed; use tracing::error; /// Bootstrap server binder -pub struct BootstrapServerBinder { +pub struct BootstrapServerBinder { max_bootstrap_message_size: u32, max_consensus_block_ids: u64, thread_count: u8, @@ -34,14 +34,14 @@ pub struct BootstrapServerBinder { randomness_size_bytes: usize, size_field_len: usize, local_keypair: KeyPair, - duplex: Resource, + duplex: Resource, prev_message: Option, version_serializer: VersionSerializer, version_deserializer: VersionDeserializer, write_error_timeout: MassaTime, } -impl BootstrapServerBinder { +impl BootstrapServerBinder { /// Creates a new `WriteBinder`. /// /// # Argument @@ -49,7 +49,7 @@ impl BootstrapServerBinder { /// * `local_keypair`: local node user keypair /// * `limit`: limit max bytes per second (up and down) #[allow(clippy::too_many_arguments)] - pub fn new(duplex: Duplex, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { + pub fn new(duplex: D, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { let BootstrapSrvBindCfg { max_bytes_read_write: limit, max_bootstrap_message_size, @@ -75,9 +75,6 @@ impl BootstrapServerBinder { write_error_timeout, } } -} - -impl BootstrapServerBinder { /// Performs a handshake. Should be called after connection /// NOT cancel-safe /// MUST always be followed by a send of the `BootstrapMessage::BootstrapTime` diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index d9609a18e77..9bb07061165 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -29,8 +29,8 @@ lazy_static::lazy_static! { }; } -impl BootstrapClientBinder { - pub fn test_default(client_duplex: Duplex, remote_pubkey: PublicKey) -> Self { +impl BootstrapClientBinder { + pub fn test_default(client_duplex: D, remote_pubkey: PublicKey) -> Self { let cfg = BootstrapClientConfig { max_bytes_read_write: f64::INFINITY, max_bootstrap_message_size: MAX_BOOTSTRAP_MESSAGE_SIZE, diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index fec5ceaa0d3..70722b0e6cc 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -7,13 +7,10 @@ use super::{ get_random_ledger_changes, wait_network_command, }, }; -use crate::BootstrapConfig; -use crate::{ - establisher::Duplex, - tests::tools::{ - get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, - }, +use crate::tests::tools::{ + get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, }; +use crate::BootstrapConfig; use crate::{ get_state, start_bootstrap_server, tests::tools::{assert_eq_bootstrap_graph, get_bootstrap_config}, @@ -56,7 +53,7 @@ use std::{ time::Duration, }; use tempfile::TempDir; -use tokio::sync::mpsc; +use tokio::{net::TcpStream, sync::mpsc}; lazy_static::lazy_static! { pub static ref BOOTSTRAP_CONFIG_KEYPAIR: (BootstrapConfig, KeyPair) = { @@ -156,7 +153,7 @@ async fn test_bootstrap_server() { // start bootstrap server let (bootstrap_establisher, bootstrap_interface) = mock_establisher::new(); - let bootstrap_manager = start_bootstrap_server( + let bootstrap_manager = start_bootstrap_server::( consensus_controller, NetworkCommandSender(network_cmd_tx), final_state_server.clone(), @@ -213,11 +210,11 @@ async fn test_bootstrap_server() { // launch bridge bootstrap_rw.set_nonblocking(true).unwrap(); - let bootstrap_rw = Duplex::from_std(bootstrap_rw).unwrap(); + let bootstrap_rw = TcpStream::from_std(bootstrap_rw).unwrap(); let bridge = tokio::spawn(async move { let remote_rw = remote_rw.join().unwrap(); remote_rw.set_nonblocking(true).unwrap(); - let remote_rw = Duplex::from_std(remote_rw).unwrap(); + let remote_rw = TcpStream::from_std(remote_rw).unwrap(); bridge_mock_streams(remote_rw, bootstrap_rw).await; }); diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index eee0f2da8d4..a06c680b956 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -60,8 +60,6 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, }; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWriteExt; use tokio::{sync::mpsc::Receiver, time::sleep}; // Use loop-back address. use port 0 to auto-assign a port @@ -465,7 +463,7 @@ pub fn get_peers() -> BootstrapPeers { ]) } -pub async fn bridge_mock_streams(mut side1: Duplex, mut side2: Duplex) { +pub async fn bridge_mock_streams(mut side1: D, mut side2: D) { let mut buf1 = vec![0u8; 1024]; let mut buf2 = vec![0u8; 1024]; loop { From ea514481d13ed97ea6323b92e6f477ebb23bf366 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 13:58:53 +0200 Subject: [PATCH 26/83] Start the server with the listener instead of estab trait --- massa-bootstrap/src/establisher.rs | 8 ++++---- massa-bootstrap/src/lib.rs | 2 +- massa-bootstrap/src/server.rs | 11 ++--------- massa-bootstrap/src/tests/mock_establisher.rs | 2 +- massa-bootstrap/src/tests/scenarios.rs | 15 ++++++++++----- massa-node/src/main.rs | 16 +++++++++++----- 6 files changed, 29 insertions(+), 25 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index fa190f780f9..ac12cc7a910 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -30,9 +30,9 @@ pub trait BSEstablisher { // lifetime can be thrown away. // TODO: use super-advanced lifetime/GAT/other shenanigans to // make the listener compatable with being moved into a thread - type Listener: BSListener + Send + 'static; + type Listener: BSListener; type Connector: BSConnector; - fn get_listener(&mut self, addr: SocketAddr) -> io::Result; + fn get_listener(&mut self, addr: &SocketAddr) -> io::Result; fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result; } @@ -82,7 +82,7 @@ impl BSEstablisher for DefaultEstablisher { /// /// # Argument /// * `addr`: `SocketAddr` we want to bind to. - fn get_listener(&mut self, addr: SocketAddr) -> io::Result { + fn get_listener(&mut self, addr: &SocketAddr) -> io::Result { // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag // This is needed to get the same behavior on all OS // However, if IPv6 is disabled system-wide, you may need to bind to an IPv4 address instead. @@ -96,7 +96,7 @@ impl BSEstablisher for DefaultEstablisher { if addr.is_ipv6() { socket.set_only_v6(false)?; } - socket.bind(&addr.into())?; + socket.bind(&(*addr).into())?; // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index d666fb7877c..0121fb7abb0 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -29,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::DefaultEstablisher; +pub use establisher::{BSEstablisher, DefaultEstablisher}; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 82a44642386..eb5df8e71e7 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -58,7 +58,7 @@ use tracing::{debug, error, info, warn}; use crate::{ error::BootstrapError, - establisher::{BSEstablisher, BSListener, Duplex}, + establisher::{BSListener, Duplex}, messages::{BootstrapClientMessage, BootstrapServerMessage}, server_binder::BootstrapServerBinder, BootstrapConfig, @@ -106,14 +106,11 @@ pub fn start_bootstrap_server( network_command_sender: NetworkCommandSender, final_state: Arc>, config: BootstrapConfig, - mut establisher: impl BSEstablisher, + listener: impl BSListener + Send + 'static, keypair: KeyPair, version: Version, ) -> Result>, Box> { massa_trace!("bootstrap.lib.start_bootstrap_server", {}); - let Some(listen_addr) = config.listen_addr else { - return Ok(None); - }; // TODO(low prio): See if a zero capacity channel model can work let (listen_stopper_tx, listen_stopper_rx) = crossbeam::channel::bounded::<()>(1); @@ -134,10 +131,6 @@ pub fn start_bootstrap_server( return Err(Box::new(BootstrapError::GeneralError("Failed to creato bootstrap async runtime".to_string()))); }; - let listener = establisher - .get_listener(listen_addr) - .map_err(BootstrapError::IoError)?; - // This is the primary interface between the async-listener, and the "sync" worker let (listener_tx, listener_rx) = crossbeam::channel::bounded::>(max_bootstraps * 2); diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index f14a25ab830..fc43eead2e4 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -118,7 +118,7 @@ impl BSEstablisher for MockEstablisher { type Listener = MockListener; type Connector = MockConnector; - fn get_listener(&mut self, _addr: SocketAddr) -> io::Result { + fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { Ok(MockListener { connection_listener_rx: self .connection_listener_rx diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 70722b0e6cc..1af477b35a5 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -7,14 +7,17 @@ use super::{ get_random_ledger_changes, wait_network_command, }, }; -use crate::tests::tools::{ - get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, -}; use crate::BootstrapConfig; use crate::{ get_state, start_bootstrap_server, tests::tools::{assert_eq_bootstrap_graph, get_bootstrap_config}, }; +use crate::{ + tests::tools::{ + get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, + }, + BSEstablisher, +}; use massa_async_pool::AsyncPoolConfig; use massa_consensus_exports::{ bootstrapable_graph::BootstrapableGraph, @@ -152,13 +155,15 @@ async fn test_bootstrap_server() { let final_state_server_clone = final_state_server.clone(); // start bootstrap server - let (bootstrap_establisher, bootstrap_interface) = mock_establisher::new(); + let (mut bootstrap_establisher, bootstrap_interface) = mock_establisher::new(); let bootstrap_manager = start_bootstrap_server::( consensus_controller, NetworkCommandSender(network_cmd_tx), final_state_server.clone(), bootstrap_config.clone(), - bootstrap_establisher, + bootstrap_establisher + .get_listener(&bootstrap_config.listen_addr.unwrap()) + .unwrap(), keypair.clone(), Version::from_str("TEST.1.10").unwrap(), ) diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 1002f625dc1..c93456c56c4 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -11,7 +11,9 @@ use dialoguer::Password; use massa_api::{ApiServer, ApiV2, Private, Public, RpcServer, StopHandle, API}; use massa_api_exports::config::APIConfig; use massa_async_pool::AsyncPoolConfig; -use massa_bootstrap::{get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager}; +use massa_bootstrap::{ + get_state, start_bootstrap_server, BSEstablisher, BootstrapConfig, BootstrapManager, +}; use massa_consensus_exports::events::ConsensusEvent; use massa_consensus_exports::{ConsensusChannels, ConsensusConfig, ConsensusManager}; use massa_consensus_worker::start_consensus_worker; @@ -68,6 +70,7 @@ use std::thread::sleep; use std::time::Duration; use std::{path::Path, process, sync::Arc}; use structopt::StructOpt; +use tokio::net::TcpStream; use tokio::signal; use tokio::sync::{broadcast, mpsc}; use tracing::{error, info, warn}; @@ -80,7 +83,7 @@ async fn launch( node_wallet: Arc>, ) -> ( Receiver, - Option, + Option>, Box, Box, Box, @@ -503,12 +506,15 @@ async fn launch( let factory_manager = start_factory(factory_config, node_wallet.clone(), factory_channels); // launch bootstrap server - let bootstrap_manager = start_bootstrap_server( + let addr = &bootstrap_config.listen_addr.unwrap(); + let bootstrap_manager = start_bootstrap_server::( consensus_controller.clone(), network_command_sender.clone(), final_state.clone(), bootstrap_config, - massa_bootstrap::DefaultEstablisher::new(), + massa_bootstrap::DefaultEstablisher::new() + .get_listener(addr) + .unwrap(), private_key, *VERSION, ) @@ -641,7 +647,7 @@ async fn launch( } struct Managers { - bootstrap_manager: Option, + bootstrap_manager: Option>, consensus_manager: Box, execution_manager: Box, selector_manager: Box, From 436517be97e8c4608989ce09b70fd9ec4e0fbfd3 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 14:10:10 +0200 Subject: [PATCH 27/83] Update comments --- massa-bootstrap/src/establisher.rs | 8 ++++---- massa-node/src/main.rs | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index ac12cc7a910..755442a6276 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -26,13 +26,13 @@ pub trait BSConnector { /// Specifies a common interface that can be used by standard, or mockers pub trait BSEstablisher { - // TODO: this is needed for thread spawning. Once the listener is on-thread, the static - // lifetime can be thrown away. - // TODO: use super-advanced lifetime/GAT/other shenanigans to - // make the listener compatable with being moved into a thread + /// For non-mock situations is a plain wrapper around a TcpListener type Listener: BSListener; + /// For non-mock situations is a plain wrapper around an interface that provides a TcpStream type Connector: BSConnector; + /// Generates an instance of the assonciated type fn get_listener(&mut self, addr: &SocketAddr) -> io::Result; + /// Generates an instance of the assonciated type fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result; } diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index c93456c56c4..e5097e2cb8b 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -507,6 +507,7 @@ async fn launch( // launch bootstrap server let addr = &bootstrap_config.listen_addr.unwrap(); + // TODO: use std::net::TcpStream let bootstrap_manager = start_bootstrap_server::( consensus_controller.clone(), network_command_sender.clone(), From 11a3b24ab4be74839ff68524f16e124e82b62584 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 14:21:52 +0200 Subject: [PATCH 28/83] Remove mock-establisher interface entirely --- massa-bootstrap/src/client.rs | 12 ++++------- massa-bootstrap/src/establisher.rs | 21 ++----------------- massa-bootstrap/src/lib.rs | 2 +- massa-bootstrap/src/tests/mock_establisher.rs | 15 ++++++------- massa-bootstrap/src/tests/scenarios.rs | 13 +++++------- massa-node/src/main.rs | 6 ++---- 6 files changed, 20 insertions(+), 49 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index ac3677717e1..b63c0e108de 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -17,7 +17,7 @@ use tracing::{debug, info, warn}; use crate::{ client_binder::BootstrapClientBinder, error::BootstrapError, - establisher::{BSConnector, BSEstablisher, Duplex}, + establisher::{BSConnector, Duplex}, messages::{BootstrapClientMessage, BootstrapServerMessage}, settings::IpType, BootstrapConfig, GlobalBootstrapState, @@ -372,15 +372,11 @@ async fn send_client_message( } fn connect_to_server( - establisher: &mut impl BSEstablisher, + connector: &mut impl BSConnector, bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, ) -> Result, Box> { - // connect - let mut connector = establisher - .get_connector(bootstrap_config.connect_timeout) - .map_err(|e| Box::new(e.into()))?; let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( @@ -422,7 +418,7 @@ fn filter_bootstrap_list( pub async fn get_state( bootstrap_config: &BootstrapConfig, final_state: Arc>, - mut establisher: impl BSEstablisher, + mut connector: impl BSConnector, version: Version, genesis_timestamp: MassaTime, end_timestamp: Option, @@ -490,7 +486,7 @@ pub async fn get_state( } info!("Start bootstrapping from {}", addr); match connect_to_server( - &mut establisher, + &mut connector, bootstrap_config, addr, &node_id.get_public_key(), diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 755442a6276..86c085779f5 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -24,18 +24,6 @@ pub trait BSConnector { fn connect(&mut self, addr: SocketAddr) -> io::Result; } -/// Specifies a common interface that can be used by standard, or mockers -pub trait BSEstablisher { - /// For non-mock situations is a plain wrapper around a TcpListener - type Listener: BSListener; - /// For non-mock situations is a plain wrapper around an interface that provides a TcpStream - type Connector: BSConnector; - /// Generates an instance of the assonciated type - fn get_listener(&mut self, addr: &SocketAddr) -> io::Result; - /// Generates an instance of the assonciated type - fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result; -} - /// The listener we are using #[derive(Debug)] pub struct DefaultListener(TcpListener); @@ -73,16 +61,11 @@ impl DefaultEstablisher { pub fn new() -> Self { DefaultEstablisher {} } -} - -impl BSEstablisher for DefaultEstablisher { - type Connector = DefaultConnector; - type Listener = DefaultListener; /// Gets the associated listener /// /// # Argument /// * `addr`: `SocketAddr` we want to bind to. - fn get_listener(&mut self, addr: &SocketAddr) -> io::Result { + pub fn get_listener(&mut self, addr: &SocketAddr) -> io::Result { // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag // This is needed to get the same behavior on all OS // However, if IPv6 is disabled system-wide, you may need to bind to an IPv4 address instead. @@ -107,7 +90,7 @@ impl BSEstablisher for DefaultEstablisher { /// /// # Argument /// * `timeout_duration`: timeout duration in milliseconds - fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result { + pub fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result { Ok(DefaultConnector(timeout_duration)) } } diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index 0121fb7abb0..d666fb7877c 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -29,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::{BSEstablisher, DefaultEstablisher}; +pub use establisher::DefaultEstablisher; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index fc43eead2e4..c2a46039248 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -8,7 +8,7 @@ use std::net::{SocketAddr, TcpListener, TcpStream}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::Arc; -use crate::establisher::{BSConnector, BSEstablisher, BSListener}; +use crate::establisher::{BSConnector, BSListener}; pub fn new() -> (MockEstablisher, MockEstablisherInterface) { let (connection_listener_tx, connection_listener_rx) = @@ -114,11 +114,8 @@ pub struct MockEstablisher { connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, } -impl BSEstablisher for MockEstablisher { - type Listener = MockListener; - type Connector = MockConnector; - - fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { +impl MockEstablisher { + pub fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { Ok(MockListener { connection_listener_rx: self .connection_listener_rx @@ -127,12 +124,12 @@ impl BSEstablisher for MockEstablisher { }) } - fn get_connector(&mut self, _timeout_duration: MassaTime) -> std::io::Result { + pub fn get_connector(&mut self) -> MockConnector { // create connector stream - Ok(MockConnector { + MockConnector { connection_connector_tx: self.connection_connector_tx.clone(), - }) + } } } diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 1af477b35a5..29711306441 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -7,17 +7,14 @@ use super::{ get_random_ledger_changes, wait_network_command, }, }; +use crate::tests::tools::{ + get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, +}; use crate::BootstrapConfig; use crate::{ get_state, start_bootstrap_server, tests::tools::{assert_eq_bootstrap_graph, get_bootstrap_config}, }; -use crate::{ - tests::tools::{ - get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, - }, - BSEstablisher, -}; use massa_async_pool::AsyncPoolConfig; use massa_consensus_exports::{ bootstrapable_graph::BootstrapableGraph, @@ -172,12 +169,12 @@ async fn test_bootstrap_server() { dbg!("test: launching get state"); // launch the get_state process - let (remote_establisher, mut remote_interface) = mock_establisher::new(); + let (mut remote_establisher, mut remote_interface) = mock_establisher::new(); let get_state_h = tokio::spawn(async move { get_state( bootstrap_config, final_state_client_clone, - remote_establisher, + remote_establisher.get_connector(), Version::from_str("TEST.1.10").unwrap(), MassaTime::now().unwrap().saturating_sub(1000.into()), None, diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index e5097e2cb8b..2085eaac6c0 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -11,9 +11,7 @@ use dialoguer::Password; use massa_api::{ApiServer, ApiV2, Private, Public, RpcServer, StopHandle, API}; use massa_api_exports::config::APIConfig; use massa_async_pool::AsyncPoolConfig; -use massa_bootstrap::{ - get_state, start_bootstrap_server, BSEstablisher, BootstrapConfig, BootstrapManager, -}; +use massa_bootstrap::{get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager}; use massa_consensus_exports::events::ConsensusEvent; use massa_consensus_exports::{ConsensusChannels, ConsensusConfig, ConsensusManager}; use massa_consensus_worker::start_consensus_worker; @@ -239,7 +237,7 @@ async fn launch( res = get_state( &bootstrap_config, final_state.clone(), - massa_bootstrap::DefaultEstablisher::default(), + massa_bootstrap::DefaultEstablisher::default().get_connector(bootstrap_config.connect_timeout).unwrap(), *VERSION, *GENESIS_TIMESTAMP, *END_TIMESTAMP, From e0a30893901210dbe363a15d753ec0309eb8c0cf Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 14:50:41 +0200 Subject: [PATCH 29/83] Move the default get-listener into a struct method --- massa-bootstrap/src/establisher.rs | 23 +++++++++++++++++++++++ massa-bootstrap/src/lib.rs | 2 +- massa-node/src/main.rs | 8 ++++---- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 86c085779f5..0f5365678af 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -27,6 +27,29 @@ pub trait BSConnector { /// The listener we are using #[derive(Debug)] pub struct DefaultListener(TcpListener); +impl DefaultListener { + /// Provides a standard TcpListener + pub fn new(addr: &SocketAddr) -> io::Result { + // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag + // This is needed to get the same behavior on all OS + // However, if IPv6 is disabled system-wide, you may need to bind to an IPv4 address instead. + let domain = match addr.is_ipv4() { + true => socket2::Domain::IPV4, + _ => socket2::Domain::IPV6, + }; + + let socket = socket2::Socket::new(domain, socket2::Type::STREAM, None)?; + + if addr.is_ipv6() { + socket.set_only_v6(false)?; + } + socket.bind(&(*addr).into())?; + + // Number of connections to queue, set to the hardcoded value used by tokio + socket.listen(1024)?; + Ok(DefaultListener(socket.into())) + } +} impl BSListener for DefaultListener { /// Accepts a new incoming connection from this listener. diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index d666fb7877c..9d0e109d498 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -29,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::DefaultEstablisher; +pub use establisher::{DefaultEstablisher, DefaultListener}; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 2085eaac6c0..10f7f81ac3c 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -11,7 +11,9 @@ use dialoguer::Password; use massa_api::{ApiServer, ApiV2, Private, Public, RpcServer, StopHandle, API}; use massa_api_exports::config::APIConfig; use massa_async_pool::AsyncPoolConfig; -use massa_bootstrap::{get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager}; +use massa_bootstrap::{ + get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager, DefaultListener, +}; use massa_consensus_exports::events::ConsensusEvent; use massa_consensus_exports::{ConsensusChannels, ConsensusConfig, ConsensusManager}; use massa_consensus_worker::start_consensus_worker; @@ -511,9 +513,7 @@ async fn launch( network_command_sender.clone(), final_state.clone(), bootstrap_config, - massa_bootstrap::DefaultEstablisher::new() - .get_listener(addr) - .unwrap(), + DefaultListener::new(addr).unwrap(), private_key, *VERSION, ) From 17783adae78e7c87ce53dc4a4329f9cfa68503a2 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 14:51:46 +0200 Subject: [PATCH 30/83] Give mock setup variables clearer names --- massa-bootstrap/src/tests/scenarios.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 29711306441..061e8f939bb 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -152,13 +152,13 @@ async fn test_bootstrap_server() { let final_state_server_clone = final_state_server.clone(); // start bootstrap server - let (mut bootstrap_establisher, bootstrap_interface) = mock_establisher::new(); + let (mut mock_bs_listener, bootstrap_interface) = mock_establisher::new(); let bootstrap_manager = start_bootstrap_server::( consensus_controller, NetworkCommandSender(network_cmd_tx), final_state_server.clone(), bootstrap_config.clone(), - bootstrap_establisher + mock_bs_listener .get_listener(&bootstrap_config.listen_addr.unwrap()) .unwrap(), keypair.clone(), @@ -169,12 +169,12 @@ async fn test_bootstrap_server() { dbg!("test: launching get state"); // launch the get_state process - let (mut remote_establisher, mut remote_interface) = mock_establisher::new(); + let (mut mock_remote_connector, mut remote_interface) = mock_establisher::new(); let get_state_h = tokio::spawn(async move { get_state( bootstrap_config, final_state_client_clone, - remote_establisher.get_connector(), + mock_remote_connector.get_connector(), Version::from_str("TEST.1.10").unwrap(), MassaTime::now().unwrap().saturating_sub(1000.into()), None, @@ -185,7 +185,7 @@ async fn test_bootstrap_server() { dbg!("test: get state launched, waiting on connection"); // accept connection attempt from remote - let remote_rw = std::thread::spawn(move || { + let remote_bridge = std::thread::spawn(move || { dbg!("conn wait thread"); let (remote_rw, conn_addr, waker) = remote_interface .wait_connection_attempt_from_controller() @@ -202,7 +202,7 @@ async fn test_bootstrap_server() { // connect to bootstrap let remote_addr = std::net::SocketAddr::from_str("82.245.72.98:10000").unwrap(); // not checked - let bootstrap_rw = tokio::time::timeout( + let bootstrap_bridge = tokio::time::timeout( std::time::Duration::from_millis(1000), bootstrap_interface.connect_to_controller(&remote_addr), ) @@ -211,13 +211,13 @@ async fn test_bootstrap_server() { .expect("could not connect to bootstrap"); // launch bridge - bootstrap_rw.set_nonblocking(true).unwrap(); - let bootstrap_rw = TcpStream::from_std(bootstrap_rw).unwrap(); + bootstrap_bridge.set_nonblocking(true).unwrap(); + let bootstrap_bridge = TcpStream::from_std(bootstrap_bridge).unwrap(); let bridge = tokio::spawn(async move { - let remote_rw = remote_rw.join().unwrap(); - remote_rw.set_nonblocking(true).unwrap(); - let remote_rw = TcpStream::from_std(remote_rw).unwrap(); - bridge_mock_streams(remote_rw, bootstrap_rw).await; + let remote_bridge = remote_bridge.join().unwrap(); + remote_bridge.set_nonblocking(true).unwrap(); + let remote_bridge = TcpStream::from_std(remote_bridge).unwrap(); + bridge_mock_streams(remote_bridge, bootstrap_bridge).await; }); // intercept peers being asked From 660e7e2bdb0988bc3bd0c6c44ddc71e4fb6f4f12 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 15:16:14 +0200 Subject: [PATCH 31/83] Add mocking todos --- massa-bootstrap/src/server.rs | 2 ++ massa-bootstrap/src/tests/scenarios.rs | 2 ++ 2 files changed, 4 insertions(+) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index eb5df8e71e7..d8e69236055 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -823,6 +823,8 @@ async fn manage_bootstrap( .send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { + // TODO: pass in an impl that can be mocked so that this will be hardcoded + // where the mock is defined peers: network_command_sender.get_bootstrap_peers().await?, }, ) diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 061e8f939bb..8911ad27753 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -221,6 +221,8 @@ async fn test_bootstrap_server() { }); // intercept peers being asked + // TODO: This would ideally be mocked such that the bootstrap server takes an impl of the network controller. + // and the impl is mocked such that it will just return the sent peers let wait_peers = async move || { // wait for bootstrap to ask network for peers, send them let response = From 48025584efc72a601e4a386872d7b8dc68a6b99d Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 15:45:47 +0200 Subject: [PATCH 32/83] Remove `dbg!`s --- massa-bootstrap/src/tests/mock_establisher.rs | 17 ----------------- massa-bootstrap/src/tests/scenarios.rs | 4 ---- 2 files changed, 21 deletions(-) diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index c2a46039248..ad99be7b2ca 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -36,14 +36,12 @@ pub struct MockListener { impl BSListener for MockListener { fn accept(&mut self) -> std::io::Result<(TcpStream, SocketAddr)> { - dbg!("accept recving"); let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener accept channel from Establisher closed".to_string(), ) })?; - dbg!("accept received"); let duplex_controller = TcpListener::bind("localhost:0").unwrap(); let duplex_mock = TcpStream::connect(duplex_controller.local_addr().unwrap()).unwrap(); let duplex_controller = duplex_controller.accept().unwrap(); @@ -51,15 +49,12 @@ impl BSListener for MockListener { // Tokio `from_std` have non-blocking Tcp objects as a requirement duplex_mock.set_nonblocking(true).unwrap(); - dbg!("accept sending mock", &duplex_mock); sender.send(duplex_mock).map_err(|_| { io::Error::new( io::ErrorKind::Other, "MockListener accept return \"oneshot\" channel to Establisher closed".to_string(), ) })?; - dbg!("accept mock sent"); - dbg!("accept returning", &duplex_controller); Ok(duplex_controller) } @@ -72,7 +67,6 @@ pub struct MockConnector { impl BSConnector for MockConnector { fn connect(&mut self, addr: SocketAddr) -> std::io::Result { - dbg!("connect"); let duplex_mock = TcpListener::bind(addr).unwrap(); let duplex_controller = TcpStream::connect(addr).unwrap(); let duplex_mock = duplex_mock.accept().unwrap(); @@ -84,7 +78,6 @@ impl BSConnector for MockConnector { let sender = self.connection_connector_tx.clone(); let send = std::thread::spawn(move || { // send new connection to mock - dbg!("connect thread sending"); sender .send((duplex_mock.0, addr, provided_waker)) .map_err(|_err| { @@ -94,16 +87,12 @@ impl BSConnector for MockConnector { ) }) .unwrap(); - dbg!("connect thread sent"); }); - dbg!("sent spun up"); while !waker.load(Ordering::Relaxed) { std::thread::yield_now(); } - dbg!("flag is true"); send.join().unwrap(); - dbg!("joined"); Ok(duplex_controller) } } @@ -140,7 +129,6 @@ pub struct MockEstablisherInterface { impl MockEstablisherInterface { pub async fn connect_to_controller(&self, addr: &SocketAddr) -> io::Result { - dbg!("conn ctrl"); let sender = self.connection_listener_tx.as_ref().ok_or_else(|| { io::Error::new( io::ErrorKind::Other, @@ -148,15 +136,12 @@ impl MockEstablisherInterface { ) })?; let (response_tx, response_rx) = bounded::(1); - dbg!("conn ctrl: sending resp_tx", addr); sender.send((*addr, response_tx)).map_err(|_err| { io::Error::new( io::ErrorKind::Other, "mock connect_to_controller_listener channel to listener closed".to_string(), ) })?; - dbg!("conn ctrl: sent resp_tx"); - dbg!("conn ctrl: getting duplex mokc"); let duplex_mock = response_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, @@ -164,14 +149,12 @@ impl MockEstablisherInterface { .to_string(), ) })?; - dbg!("conn ctrl: got", &duplex_mock); Ok(duplex_mock) } pub fn wait_connection_attempt_from_controller( &mut self, ) -> io::Result<(TcpStream, SocketAddr, Arc)> { - dbg!("wait conn attempt"); self.connection_connector_rx.recv().map_err(|_| { io::Error::new( io::ErrorKind::Other, diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 8911ad27753..ebdfa03de9f 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -167,7 +167,6 @@ async fn test_bootstrap_server() { .unwrap() .unwrap(); - dbg!("test: launching get state"); // launch the get_state process let (mut mock_remote_connector, mut remote_interface) = mock_establisher::new(); let get_state_h = tokio::spawn(async move { @@ -182,15 +181,12 @@ async fn test_bootstrap_server() { .await .unwrap() }); - dbg!("test: get state launched, waiting on connection"); // accept connection attempt from remote let remote_bridge = std::thread::spawn(move || { - dbg!("conn wait thread"); let (remote_rw, conn_addr, waker) = remote_interface .wait_connection_attempt_from_controller() .expect("timeout waiting for connection attempt from remote"); - dbg!("test: got conn", &conn_addr); let expect_conn_addr = bootstrap_config.bootstrap_list[0].0; assert_eq!( conn_addr, expect_conn_addr, From e990892c2ad5c25e17d6f751a429f464f2656250 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 16:01:22 +0200 Subject: [PATCH 33/83] Self review --- massa-bootstrap/src/establisher.rs | 2 +- massa-bootstrap/src/server.rs | 1 + massa-bootstrap/src/tests/mock_establisher.rs | 5 ++--- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 0f5365678af..8f8106801bf 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -8,7 +8,7 @@ use std::{ /// duplex connection pub trait Duplex: // static because need to send between threads :( - 'static + Send + Sync + tokio::io::AsyncReadExt + tokio::io::AsyncWriteExt + std::marker::Unpin + 'static + Send + tokio::io::AsyncReadExt + tokio::io::AsyncWriteExt + std::marker::Unpin { } diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index d8e69236055..2e0eedd5e04 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -72,6 +72,7 @@ type BsConn = (D, SocketAddr); pub struct BootstrapManager { update_handle: thread::JoinHandle>>, // need to preserve the listener handle up to here to prevent it being destroyed + #[allow(clippy::type_complexity)] _listen_handle: thread::JoinHandle>, Box>>, main_handle: thread::JoinHandle>>, listen_stopper_tx: crossbeam::channel::Sender<()>, diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs index ad99be7b2ca..802647dfd5a 100644 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ b/massa-bootstrap/src/tests/mock_establisher.rs @@ -1,7 +1,6 @@ // Copyright (c) 2022 MASSA LABS use crossbeam::channel::{bounded, Receiver, Sender}; use massa_models::config::CHANNEL_SIZE; -use massa_time::MassaTime; use socket2 as _; use std::io; use std::net::{SocketAddr, TcpListener, TcpStream}; @@ -104,7 +103,7 @@ pub struct MockEstablisher { } impl MockEstablisher { - pub fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { + pub(crate) fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { Ok(MockListener { connection_listener_rx: self .connection_listener_rx @@ -113,7 +112,7 @@ impl MockEstablisher { }) } - pub fn get_connector(&mut self) -> MockConnector { + pub(crate) fn get_connector(&mut self) -> MockConnector { // create connector stream MockConnector { From c53a842c209aade3e9d12bf94ad1fa9286db1056 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 15:43:10 +0200 Subject: [PATCH 34/83] Make the network command sender mockable --- Cargo.lock | 305 ++++++++++++------ massa-bootstrap/src/server.rs | 2 +- massa-network-exports/Cargo.toml | 2 + massa-network-exports/src/lib.rs | 4 +- .../src/network_controller.rs | 166 +++++++--- 5 files changed, 330 insertions(+), 149 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 572c8f573af..80c4a206da6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,13 +187,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.66" +version = "0.1.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b84f9ebcc6c1f5b8cb160f6990096a5c127f423fcb6e1ccc46c370cbdfb75dfc" +checksum = "b9ccdd8f2a161be9bd5c023df56f1b2a0bd1d83872ae53b71a84a12c9bf6e842" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", - "syn 1.0.109", + "proc-macro2 1.0.54", + "quote 1.0.26", + "syn 2.0.10", ] [[package]] @@ -267,8 +267,8 @@ dependencies = [ "lazy_static", "lazycell", "peeking_take_while", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "regex", "rustc-hash", "shlex", @@ -345,7 +345,7 @@ dependencies = [ "borsh-derive-internal", "borsh-schema-derive-internal", "proc-macro-crate 0.1.5", - "proc-macro2 1.0.51", + "proc-macro2 1.0.54", "syn 1.0.109", ] @@ -355,8 +355,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "186b734fa1c9f6743e90c95d7233c9faab6360d1a96d4ffa19d9cfd1e9350f8a" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -366,8 +366,8 @@ version = "0.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99b7ff1008316626f485991b960ade129253d4034014616b94f309a15366cc49" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -434,8 +434,8 @@ version = "0.6.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e31225543cb46f81a7e224762764f4a6a0f097b1db0b175f69e8065efaa42de5" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -666,8 +666,8 @@ version = "0.2.29" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d897f47bf7270cf70d370f8f98c1abb6d2d4cf60a6845d30e05bfb90c6568650" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "unicode-xid 0.2.4", ] @@ -910,7 +910,7 @@ version = "0.1.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096" dependencies = [ - "quote 1.0.23", + "quote 1.0.26", "syn 1.0.109", ] @@ -957,8 +957,8 @@ dependencies = [ "cc", "codespan-reporting", "once_cell", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "scratch", "syn 1.0.109", ] @@ -975,8 +975,8 @@ version = "1.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b75aed41bb2e6367cae39e6326ef817a851db13c13e4f3263714ca3cfb8de56" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -998,8 +998,8 @@ checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" dependencies = [ "fnv", "ident_case", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "strsim 0.10.0", "syn 1.0.109", ] @@ -1011,7 +1011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" dependencies = [ "darling_core", - "quote 1.0.23", + "quote 1.0.26", "syn 1.0.109", ] @@ -1046,6 +1046,12 @@ version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "56254986775e3233ffa9c4d7d3faaf6d36a2c09d30b20687e9f88bc8bafc16c8" +[[package]] +name = "difflib" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6184e33543162437515c2e2b48714794e37845ec9851711914eec9d308f6ebe8" + [[package]] name = "digest" version = "0.9.0" @@ -1113,8 +1119,8 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bf95dc3f046b9da4f2d51833c0d3547d8564ef6910f5c1ed130306a75b92886" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -1124,6 +1130,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0688c2a7f92e427f44895cd63841bff7b29f8d7a1648b9e7e07a4a365b2e1257" +[[package]] +name = "downcast" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1435fa1053d8b2fbbe9be7e97eca7f33d37b28409959813daefc1446a14247f1" + [[package]] name = "dynasm" version = "1.2.3" @@ -1134,8 +1146,8 @@ dependencies = [ "byteorder", "lazy_static", "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -1207,8 +1219,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c134c37760b27a871ba422106eedbb8247da973a09e82558bf26d619c882b159" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -1228,8 +1240,8 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a4da76b3b6116d758c7ba93f7ec6a35d2e2cf24feda76c6e38a375f4d5c59f2" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -1249,8 +1261,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "03e7b551eba279bf0fa88b83a46330168c1560a52a94f5126f892f0b364ab3e0" dependencies = [ "darling", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -1342,12 +1354,27 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "float-cmp" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98de4bbd547a563b716d8dfa9aad1cb19bfab00f4fa09a6a4ed21dbcf44ce9c4" +dependencies = [ + "num-traits", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "fragile" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c2141d6d6c8512188a7891b4b01590a45f6dac67afb4f255c4124dbb86d4eaa" + [[package]] name = "function_name" version = "0.3.0" @@ -1423,8 +1450,8 @@ version = "0.3.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "95a73af87da33b5acf53acfebdc339fe592ecf5357ac7c0a7734ab9d8c876a70" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -2013,8 +2040,8 @@ checksum = "baa6da1e4199c10d7b1d0a6e5e8bd8e55f351163b6f4b3cbb044672a69bd4c1c" dependencies = [ "heck 0.4.1", "proc-macro-crate 1.3.1", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -2209,7 +2236,7 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0fbfc88337168279f2e9ae06e157cfed4efd3316e14dc96ed074d4f2e6c5952" dependencies = [ - "quote 1.0.23", + "quote 1.0.26", "syn 1.0.109", ] @@ -2755,6 +2782,7 @@ dependencies = [ name = "massa_network_exports" version = "0.1.0" dependencies = [ + "async-trait", "displaydoc", "enum-map", "massa_hash 0.1.0", @@ -2762,6 +2790,7 @@ dependencies = [ "massa_serialization 0.1.0", "massa_signature", "massa_time", + "mockall", "nom", "serde", "serde_json", @@ -3109,6 +3138,33 @@ dependencies = [ "windows-sys 0.45.0", ] +[[package]] +name = "mockall" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c84490118f2ee2d74570d114f3d0493cbf02790df303d2707606c3e14e07c96" +dependencies = [ + "cfg-if", + "downcast", + "fragile", + "lazy_static", + "mockall_derive", + "predicates", + "predicates-tree", +] + +[[package]] +name = "mockall_derive" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" +dependencies = [ + "cfg-if", + "proc-macro2 1.0.54", + "quote 1.0.26", + "syn 1.0.109", +] + [[package]] name = "more-asserts" version = "0.2.2" @@ -3152,6 +3208,12 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "normalize-line-endings" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61807f77802ff30975e01f4f071c8ba10c022052f98b3294119f3e615d13e5be" + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -3268,8 +3330,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbff9bc912032c62bf65ef1d5aea88983b420f4f839db1e9b0c281a25c9c799" dependencies = [ "proc-macro-crate 1.3.1", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -3402,8 +3464,8 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f35583365be5d148e959284f42526841917b7bfa09e2d1a7ad5dde2cf0eaa39" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -3465,8 +3527,8 @@ checksum = "75a1ef20bf3193c15ac345acb32e26b3dc3223aff4d77ae4fc5359567683796b" dependencies = [ "pest", "pest_meta", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -3506,8 +3568,8 @@ version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "069bdb1e05adc7a8990dce9cc75370895fbe4e3d58b9b73bf1aee56359344a55" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -3575,6 +3637,36 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "predicates" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59230a63c37f3e18569bdb90e4a89cbf5bf8b06fea0b84e65ea10cc4df47addd" +dependencies = [ + "difflib", + "float-cmp", + "itertools", + "normalize-line-endings", + "predicates-core", + "regex", +] + +[[package]] +name = "predicates-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b794032607612e7abeb4db69adb4e33590fa6cf1149e95fd7cb00e634b92f174" + +[[package]] +name = "predicates-tree" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "368ba315fb8c5052ab692e68a0eefec6ec57b23a36959c14496f0b0df2c0cecf" +dependencies = [ + "predicates-core", + "termtree", +] + [[package]] name = "pretty_assertions" version = "1.3.0" @@ -3613,8 +3705,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" dependencies = [ "proc-macro-error-attr", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", "version_check", ] @@ -3625,8 +3717,8 @@ version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "version_check", ] @@ -3641,9 +3733,9 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.51" +version = "1.0.54" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6" +checksum = "e472a104799c74b514a57226160104aa483546de37e839ec50e3c2e41dd87534" dependencies = [ "unicode-ident", ] @@ -3663,8 +3755,8 @@ version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -3679,11 +3771,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.23" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b" +checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc" dependencies = [ - "proc-macro2 1.0.51", + "proc-macro2 1.0.54", ] [[package]] @@ -3920,8 +4012,8 @@ version = "0.7.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ff26ed6c7c4dfc2aa9480b86a60e3c7233543a270a680e10758a507c5a4ce476" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4068,8 +4160,8 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "107c3d5d7f370ac09efa62a78375f94d94b8a33c61d8c278b96683fb4dbf2d8d" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4191,8 +4283,8 @@ version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fc80d722935453bcafdc2c9a73cd6fac4dc1938f0346035d84bf99fa9e33217" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4241,8 +4333,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9e6b7e52858f9f06c25e1c566bbb4ab428200cb3b30053ea09dc50837de7538b" dependencies = [ "darling", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4294,8 +4386,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1b95bb2f4f624565e8fe8140c789af7e2082c0e0561b5a82a1b678baa9703dc" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "rustversion", "syn 1.0.109", ] @@ -4306,8 +4398,8 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b64f9e531ce97c88b4778aad0ceee079216071cffec6ac9b904277f8f92e7fe3" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4317,8 +4409,8 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "079a83df15f85d89a68d64ae1238f142f172b1fa915d0d76b26a7cba1b659a69" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4498,8 +4590,8 @@ checksum = "dcb5ae327f9cc13b68763b5749770cb9e048a99bd9dfdfa58d0cf05d5f64afe0" dependencies = [ "heck 0.3.3", "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4519,8 +4611,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1e385be0d24f186b4ce2f9982191e7101bb737312ad61c1f2f984f34bcf85d59" dependencies = [ "heck 0.4.1", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "rustversion", "syn 1.0.109", ] @@ -4530,7 +4622,7 @@ name = "substruct" version = "0.1.0" source = "git+https://github.com/sydhds/substruct#2fb3ae0dc9d913a0566ce6415eaa7a7ca1690fe1" dependencies = [ - "quote 1.0.23", + "quote 1.0.26", "syn 1.0.109", ] @@ -4557,8 +4649,19 @@ version = "1.0.109" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aad1363ed6d37b84299588d62d3a7d95b5a5c2d9aad5c85609fda12afaa1f40" +dependencies = [ + "proc-macro2 1.0.54", + "quote 1.0.26", "unicode-ident", ] @@ -4568,8 +4671,8 @@ version = "0.12.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f36bdaa60a83aca3921b5259d5400cbf5e90fc51931376a9bd4a0eb79aa7210f" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", "unicode-xid 0.2.4", ] @@ -4608,6 +4711,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "termtree" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3369f5ac52d5eb6ab48c6b4ffdc8efbcad6b89c765749064ba298f2c68a16a76" + [[package]] name = "textwrap" version = "0.11.0" @@ -4638,8 +4747,8 @@ version = "1.0.39" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5420d42e90af0c38c3290abcca25b9b3bdf379fc9f55c528f53a269d9c9a267e" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4738,8 +4847,8 @@ version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d266c00fde287f55d3f1c3e96c500c362a2b8c695076ec180f27918820bc6df8" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -4889,8 +4998,8 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4017f8f45139870ca7e672686113917c71c7a6e02d4924eda67186083c03081a" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -5107,8 +5216,8 @@ dependencies = [ "bumpalo", "log", "once_cell", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", "wasm-bindgen-shared", ] @@ -5131,8 +5240,8 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c5020cfa87c7cecefef118055d44e3c1fc122c7ec25701d528ee458a0b45f38f" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -5154,7 +5263,7 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c21f77c0bedc37fd5dc21f897894a5ca01e7bb159884559461862ae90c0b4c5" dependencies = [ - "quote 1.0.23", + "quote 1.0.26", "wasm-bindgen-macro-support", ] @@ -5164,8 +5273,8 @@ version = "0.2.84" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2aff81306fcac3c7515ad4e177f521b5c9a15f2b08f4e32d823066102f35a5f6" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", "wasm-bindgen-backend", "wasm-bindgen-shared", @@ -5281,8 +5390,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ff577b7c1cfcd3d7c5b3a09fe1a499b73f7c17084845ff71225c8250a6a63a9" dependencies = [ "proc-macro-error", - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", ] @@ -5595,8 +5704,8 @@ version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44bf07cb3e50ea2003396695d58bf46bc9887a1f362260446fad6bc4e79bd36c" dependencies = [ - "proc-macro2 1.0.51", - "quote 1.0.23", + "proc-macro2 1.0.54", + "quote 1.0.26", "syn 1.0.109", "synstructure", ] diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 2e0eedd5e04..4144b85673b 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -39,7 +39,7 @@ use massa_models::{ block_id::BlockId, prehash::PreHashSet, slot::Slot, streaming_step::StreamingStep, version::Version, }; -use massa_network_exports::NetworkCommandSender; +use massa_network_exports::{NetworkCommandSender, NetworkCommandSenderTrait}; use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; diff --git a/massa-network-exports/Cargo.toml b/massa-network-exports/Cargo.toml index 3fd982f3400..69081bc7b64 100644 --- a/massa-network-exports/Cargo.toml +++ b/massa-network-exports/Cargo.toml @@ -25,6 +25,8 @@ tracing = { version = "0.1", features = [ "max_level_debug", "release_max_level_debug", ] } +async-trait = "0.1.68" +mockall = "0.11.4" [dev-dependencies] massa_models = { path = "../massa-models", features = ["testing"] } diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index bf468586ef1..68080eee805 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -12,7 +12,9 @@ pub use commands::{ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; -pub use network_controller::{NetworkCommandSender, NetworkEventReceiver, NetworkManager}; +pub use network_controller::{ + NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, +}; pub use peers::{ BootstrapPeers, BootstrapPeersDeserializer, BootstrapPeersSerializer, ConnectionCount, Peer, PeerInfo, PeerType, Peers, diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index 188018df383..6527a4716e9 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -5,6 +5,7 @@ use crate::{ error::NetworkError, BlockInfoReply, BootstrapPeers, NetworkCommand, NetworkEvent, Peers, }; +use async_trait::async_trait; use massa_models::{ block_header::SecuredHeader, block_id::BlockId, @@ -14,6 +15,7 @@ use massa_models::{ operation::{OperationPrefixIds, SecureShareOperation}, stats::NetworkStats, }; +use mockall::automock; use std::{ collections::{HashMap, VecDeque}, net::IpAddr, @@ -31,9 +33,106 @@ use tracing::{info, warn}; #[derive(Clone)] pub struct NetworkCommandSender(pub mpsc::Sender); -impl NetworkCommandSender { +#[automock] +#[async_trait] +/// Network command sender interface. Can be mocked for testing +pub trait NetworkCommandSenderTrait { /// ban node(s) by id(s) - pub async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError> { + async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; + + /// ban node(s) by ip(s) + async fn node_ban_by_ips(&self, ips: Vec) -> Result<(), NetworkError>; + + /// add ip to whitelist + async fn add_to_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; + + /// remove ip from whitelist + async fn remove_from_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; + + /// remove from banned node(s) by id(s) + async fn node_unban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; + + /// remove from banned node(s) by ip(s) + async fn node_unban_ips(&self, ips: Vec) -> Result<(), NetworkError>; + + /// Send info about the contents of a block. + async fn send_block_info( + &self, + node: NodeId, + info: Vec<(BlockId, BlockInfoReply)>, + ) -> Result<(), NetworkError>; + + /// Send the order to ask for a block. + async fn ask_for_block_list( + &self, + list: HashMap>, + ) -> Result<(), NetworkError>; + + /// Send the order to send block header. + /// + /// Note: with the current use of shared storage, + /// sending a header requires having the block stored. + /// This matches the current use of `send_block_header`, + /// which is only used after a block has been integrated in the graph. + async fn send_block_header( + &self, + node: NodeId, + header: SecuredHeader, + ) -> Result<(), NetworkError>; + + /// Send the order to get peers. + async fn get_peers(&self) -> Result; + + /// get network stats + async fn get_network_stats(&self) -> Result; + + /// Send the order to get bootstrap peers. + async fn get_bootstrap_peers(&self) -> Result; + + /// send operations to node + async fn send_operations( + &self, + node: NodeId, + operations: Vec, + ) -> Result<(), NetworkError>; + + /// Create a new call to the network, sending a announcement of operation ID prefixes to a + /// target node (`to_node`) + /// + /// # Returns + /// Can return a `[NetworkError::ChannelError]` that must be managed by the direct caller of the + /// function. + async fn announce_operations( + &self, + to_node: NodeId, + batch: OperationPrefixIds, + ) -> Result<(), NetworkError>; + + /// Create a new call to the network, sending a `wishlist` of `operationIds` to a + /// target node (`to_node`) in order to receive the full operations in the future. + /// + /// # Returns + /// Can return a `[NetworkError::ChannelError]` that must be managed by the direct caller of the + /// function. + async fn send_ask_for_operations( + &self, + to_node: NodeId, + wishlist: OperationPrefixIds, + ) -> Result<(), NetworkError>; + + /// send endorsements to node id + async fn send_endorsements( + &self, + node: NodeId, + endorsements: Vec, + ) -> Result<(), NetworkError>; + + /// Sign a message using the node's keypair + async fn node_sign_message(&self, msg: Vec) -> Result; +} +#[async_trait] +impl NetworkCommandSenderTrait for NetworkCommandSender { + async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::NodeBanByIds(ids)) .await @@ -41,8 +140,7 @@ impl NetworkCommandSender { Ok(()) } - /// ban node(s) by ip(s) - pub async fn node_ban_by_ips(&self, ips: Vec) -> Result<(), NetworkError> { + async fn node_ban_by_ips(&self, ips: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::NodeBanByIps(ips)) .await @@ -50,8 +148,7 @@ impl NetworkCommandSender { Ok(()) } - /// add ip to whitelist - pub async fn add_to_whitelist(&self, ips: Vec) -> Result<(), NetworkError> { + async fn add_to_whitelist(&self, ips: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::Whitelist(ips)) .await @@ -59,8 +156,7 @@ impl NetworkCommandSender { Ok(()) } - /// remove ip from whitelist - pub async fn remove_from_whitelist(&self, ips: Vec) -> Result<(), NetworkError> { + async fn remove_from_whitelist(&self, ips: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::RemoveFromWhitelist(ips)) .await @@ -70,8 +166,7 @@ impl NetworkCommandSender { Ok(()) } - /// remove from banned node(s) by id(s) - pub async fn node_unban_by_ids(&self, ids: Vec) -> Result<(), NetworkError> { + async fn node_unban_by_ids(&self, ids: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::NodeUnbanByIds(ids)) .await @@ -79,8 +174,7 @@ impl NetworkCommandSender { Ok(()) } - /// remove from banned node(s) by ip(s) - pub async fn node_unban_ips(&self, ips: Vec) -> Result<(), NetworkError> { + async fn node_unban_ips(&self, ips: Vec) -> Result<(), NetworkError> { self.0 .send(NetworkCommand::NodeUnbanByIps(ips)) .await @@ -88,8 +182,7 @@ impl NetworkCommandSender { Ok(()) } - /// Send info about the contents of a block. - pub async fn send_block_info( + async fn send_block_info( &self, node: NodeId, info: Vec<(BlockId, BlockInfoReply)>, @@ -103,8 +196,7 @@ impl NetworkCommandSender { Ok(()) } - /// Send the order to ask for a block. - pub async fn ask_for_block_list( + async fn ask_for_block_list( &self, list: HashMap>, ) -> Result<(), NetworkError> { @@ -115,13 +207,7 @@ impl NetworkCommandSender { Ok(()) } - /// Send the order to send block header. - /// - /// Note: with the current use of shared storage, - /// sending a header requires having the block stored. - /// This matches the current use of `send_block_header`, - /// which is only used after a block has been integrated in the graph. - pub async fn send_block_header( + async fn send_block_header( &self, node: NodeId, header: SecuredHeader, @@ -135,8 +221,7 @@ impl NetworkCommandSender { Ok(()) } - /// Send the order to get peers. - pub async fn get_peers(&self) -> Result { + async fn get_peers(&self) -> Result { let (response_tx, response_rx) = oneshot::channel(); self.0 .send(NetworkCommand::GetPeers(response_tx)) @@ -149,8 +234,7 @@ impl NetworkCommandSender { }) } - /// get network stats - pub async fn get_network_stats(&self) -> Result { + async fn get_network_stats(&self) -> Result { let (response_tx, response_rx) = oneshot::channel(); self.0 .send(NetworkCommand::GetStats { response_tx }) @@ -161,8 +245,7 @@ impl NetworkCommandSender { .map_err(|_| NetworkError::ChannelError("could not send GetStats upstream".into())) } - /// Send the order to get bootstrap peers. - pub async fn get_bootstrap_peers(&self) -> Result { + async fn get_bootstrap_peers(&self) -> Result { let (response_tx, response_rx) = oneshot::channel::(); self.0 .send(NetworkCommand::GetBootstrapPeers(response_tx)) @@ -175,8 +258,7 @@ impl NetworkCommandSender { }) } - /// send operations to node - pub async fn send_operations( + async fn send_operations( &self, node: NodeId, operations: Vec, @@ -190,13 +272,7 @@ impl NetworkCommandSender { Ok(()) } - /// Create a new call to the network, sending a announcement of operation ID prefixes to a - /// target node (`to_node`) - /// - /// # Returns - /// Can return a `[NetworkError::ChannelError]` that must be managed by the direct caller of the - /// function. - pub async fn announce_operations( + async fn announce_operations( &self, to_node: NodeId, batch: OperationPrefixIds, @@ -218,13 +294,7 @@ impl NetworkCommandSender { Ok(()) } - /// Create a new call to the network, sending a `wishlist` of `operationIds` to a - /// target node (`to_node`) in order to receive the full operations in the future. - /// - /// # Returns - /// Can return a `[NetworkError::ChannelError]` that must be managed by the direct caller of the - /// function. - pub async fn send_ask_for_operations( + async fn send_ask_for_operations( &self, to_node: NodeId, wishlist: OperationPrefixIds, @@ -238,8 +308,7 @@ impl NetworkCommandSender { Ok(()) } - /// send endorsements to node id - pub async fn send_endorsements( + async fn send_endorsements( &self, node: NodeId, endorsements: Vec, @@ -253,8 +322,7 @@ impl NetworkCommandSender { Ok(()) } - /// Sign a message using the node's keypair - pub async fn node_sign_message(&self, msg: Vec) -> Result { + async fn node_sign_message(&self, msg: Vec) -> Result { let (response_tx, response_rx) = oneshot::channel(); self.0 .send(NetworkCommand::NodeSignMessage { msg, response_tx }) From 40e8f6094f4784fb3ecb5dbc99562df0da348095 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 20:31:32 +0200 Subject: [PATCH 35/83] Use auto-mock to get the bootstrap peers --- massa-bootstrap/src/server.rs | 26 +++---- massa-bootstrap/src/tests/scenarios.rs | 48 +++++------- massa-network-exports/Cargo.toml | 2 +- massa-network-exports/src/lib.rs | 3 +- .../src/network_controller.rs | 75 ++++++++++++++++++- 5 files changed, 105 insertions(+), 49 deletions(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 4144b85673b..82ff64884d0 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -39,7 +39,7 @@ use massa_models::{ block_id::BlockId, prehash::PreHashSet, slot::Slot, streaming_step::StreamingStep, version::Version, }; -use massa_network_exports::{NetworkCommandSender, NetworkCommandSenderTrait}; +use massa_network_exports::NetworkCommandSenderTrait; use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; @@ -102,9 +102,9 @@ impl BootstrapManager { } /// See module level documentation for details -pub fn start_bootstrap_server( +pub fn start_bootstrap_server( consensus_controller: Box, - network_command_sender: NetworkCommandSender, + network_command_sender: C, final_state: Arc>, config: BootstrapConfig, listener: impl BSListener + Send + 'static, @@ -145,7 +145,7 @@ pub fn start_bootstrap_server( let update_handle = thread::Builder::new() .name("wb_list_updater".to_string()) .spawn(move || { - let res = BootstrapServer::::run_updater( + let res = BootstrapServer::::run_updater( updater_lists, config.cache_duration.into(), update_stopper_rx, @@ -166,7 +166,7 @@ pub fn start_bootstrap_server( // GAT lifetime is likely to remedy this, however. .spawn(move || { let res = listen_rt_handle - .block_on(BootstrapServer::::run_listener(listener, listener_tx)); + .block_on(BootstrapServer::::run_listener(listener, listener_tx)); res }) // the non-builder spawn doesn't return a Result, and documentation states that @@ -205,9 +205,9 @@ pub fn start_bootstrap_server( })) } -struct BootstrapServer<'a, D: Duplex> { +struct BootstrapServer<'a, D: Duplex, C: NetworkCommandSenderTrait> { consensus_controller: Box, - network_command_sender: NetworkCommandSender, + network_command_sender: C, final_state: Arc>, listener_rx: crossbeam::channel::Receiver>, listen_stopper_rx: crossbeam::channel::Receiver<()>, @@ -219,7 +219,7 @@ struct BootstrapServer<'a, D: Duplex> { bs_server_runtime: Runtime, } -impl BootstrapServer<'_, D> { +impl BootstrapServer<'_, D, C> { fn run_updater( mut list: SharedWhiteBlackList<'_>, interval: Duration, @@ -332,7 +332,7 @@ impl BootstrapServer<'_, D> { } // check IP's bootstrap attempt history - if let Err(msg) = BootstrapServer::::greedy_client_check( + if let Err(msg) = BootstrapServer::::greedy_client_check( &mut self.ip_hist_map, remote_addr, now, @@ -486,7 +486,7 @@ impl BootstrapServer<'_, D> { /// The arc_counter variable is used as a proxy to keep track the number of active bootstrap /// sessions. #[allow(clippy::too_many_arguments)] -fn run_bootstrap_session( +fn run_bootstrap_session( mut server: BootstrapServerBinder, arc_counter: Arc<()>, config: BootstrapConfig, @@ -494,7 +494,7 @@ fn run_bootstrap_session( data_execution: Arc>, version: Version, consensus_command_sender: Box, - network_command_sender: NetworkCommandSender, + network_command_sender: C, bs_loop_rt_handle: Handle, ) { debug!("running bootstrap for peer {}", remote_addr); @@ -753,13 +753,13 @@ pub async fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -async fn manage_bootstrap( +async fn manage_bootstrap( bootstrap_config: &BootstrapConfig, server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, - network_command_sender: NetworkCommandSender, + network_command_sender: C, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index ebdfa03de9f..0736f9f70f6 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -4,11 +4,12 @@ use super::{ mock_establisher, tools::{ bridge_mock_streams, get_boot_state, get_peers, get_random_final_state_bootstrap, - get_random_ledger_changes, wait_network_command, + get_random_ledger_changes, }, }; use crate::tests::tools::{ get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, + wait_network_command, }; use crate::BootstrapConfig; use crate::{ @@ -37,7 +38,7 @@ use massa_models::{ }, prehash::PreHashSet, }; -use massa_network_exports::{NetworkCommand, NetworkCommandSender}; +use massa_network_exports::{MockNetworkCommandSender, NetworkCommand}; use massa_pos_exports::{ test_exports::assert_eq_pos_selection, PoSConfig, PoSFinalState, SelectorConfig, }; @@ -53,7 +54,7 @@ use std::{ time::Duration, }; use tempfile::TempDir; -use tokio::{net::TcpStream, sync::mpsc}; +use tokio::net::TcpStream; lazy_static::lazy_static! { pub static ref BOOTSTRAP_CONFIG_KEYPAIR: (BootstrapConfig, KeyPair) = { @@ -73,7 +74,6 @@ async fn test_bootstrap_server() { let (consensus_controller, mut consensus_event_receiver) = MockConsensusController::new_with_receiver(); - let (network_cmd_tx, mut network_cmd_rx) = mpsc::channel::(5); // setup final state local config let temp_dir = TempDir::new().unwrap(); @@ -151,11 +151,20 @@ async fn test_bootstrap_server() { let final_state_client_clone = final_state_client.clone(); let final_state_server_clone = final_state_server.clone(); - // start bootstrap server + // Setup bootstrap mock-duplex let (mut mock_bs_listener, bootstrap_interface) = mock_establisher::new(); - let bootstrap_manager = start_bootstrap_server::( + // Setup network command mock-story: hard-code the result of getting bootstrap peers + let mut mocked1 = MockNetworkCommandSender::new(); + let mut mocked2 = MockNetworkCommandSender::new(); + mocked2 + .expect_get_bootstrap_peers() + .times(1) + .returning(|| Ok(get_peers())); + + mocked1.expect_clone().return_once(move || mocked2); + let bootstrap_manager = start_bootstrap_server::( consensus_controller, - NetworkCommandSender(network_cmd_tx), + mocked1, final_state_server.clone(), bootstrap_config.clone(), mock_bs_listener @@ -216,26 +225,6 @@ async fn test_bootstrap_server() { bridge_mock_streams(remote_bridge, bootstrap_bridge).await; }); - // intercept peers being asked - // TODO: This would ideally be mocked such that the bootstrap server takes an impl of the network controller. - // and the impl is mocked such that it will just return the sent peers - let wait_peers = async move || { - // wait for bootstrap to ask network for peers, send them - let response = - match wait_network_command(&mut network_cmd_rx, 20_000.into(), |cmd| match cmd { - NetworkCommand::GetBootstrapPeers(resp) => Some(resp), - _ => None, - }) - .await - { - Some(resp) => resp, - None => panic!("timeout waiting for get peers command"), - }; - let sent_peers = get_peers(); - response.send(sent_peers.clone()).unwrap(); - sent_peers - }; - // intercept consensus parts being asked let sent_graph = get_boot_state(); let sent_graph_clone = sent_graph.clone(); @@ -301,9 +290,6 @@ async fn test_bootstrap_server() { } }); - // wait for peers and graph - let sent_peers = wait_peers().await; - // wait for get_state let bootstrap_res = get_state_h .await @@ -349,7 +335,7 @@ async fn test_bootstrap_server() { // check peers assert_eq!( - sent_peers.0, + get_peers().0, bootstrap_res.peers.unwrap().0, "mismatch between sent and received peers" ); diff --git a/massa-network-exports/Cargo.toml b/massa-network-exports/Cargo.toml index 69081bc7b64..010884b4953 100644 --- a/massa-network-exports/Cargo.toml +++ b/massa-network-exports/Cargo.toml @@ -26,7 +26,7 @@ tracing = { version = "0.1", features = [ "release_max_level_debug", ] } async-trait = "0.1.68" -mockall = "0.11.4" +mockall = {version = "0.11.4", features = ["nightly"] } [dev-dependencies] massa_models = { path = "../massa-models", features = ["testing"] } diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index 68080eee805..452d1f9d26c 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -13,7 +13,8 @@ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; pub use network_controller::{ - NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, + MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, + NetworkEventReceiver, NetworkManager, }; pub use peers::{ BootstrapPeers, BootstrapPeersDeserializer, BootstrapPeersSerializer, ConnectionCount, Peer, diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index 6527a4716e9..db8bb7a26f4 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -15,7 +15,7 @@ use massa_models::{ operation::{OperationPrefixIds, SecureShareOperation}, stats::NetworkStats, }; -use mockall::automock; +use mockall::{automock, mock}; use std::{ collections::{HashMap, VecDeque}, net::IpAddr, @@ -30,13 +30,82 @@ use tokio::{ use tracing::{info, warn}; /// Network command sender -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct NetworkCommandSender(pub mpsc::Sender); +mock! { + pub NetworkCommandSender{} + impl Clone for NetworkCommandSender { + fn clone(&self) -> Self; + } + #[async_trait] + impl NetworkCommandSenderTrait for NetworkCommandSender { + async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; + + async fn node_ban_by_ips(&self, ips: Vec) -> Result<(), NetworkError>; + + async fn add_to_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; + + async fn remove_from_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; + + async fn node_unban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; + + async fn node_unban_ips(&self, ips: Vec) -> Result<(), NetworkError>; + + async fn send_block_info( + &self, + node: NodeId, + info: Vec<(BlockId, BlockInfoReply)>, + ) -> Result<(), NetworkError>; + + async fn ask_for_block_list( + &self, + list: HashMap>, + ) -> Result<(), NetworkError>; + + async fn send_block_header( + &self, + node: NodeId, + header: SecuredHeader, + ) -> Result<(), NetworkError>; + + async fn get_peers(&self) -> Result; + + async fn get_network_stats(&self) -> Result; + + async fn get_bootstrap_peers(&self) -> Result; + + async fn send_operations( + &self, + node: NodeId, + operations: Vec, + ) -> Result<(), NetworkError>; + + async fn announce_operations( + &self, + to_node: NodeId, + batch: OperationPrefixIds, + ) -> Result<(), NetworkError>; + + async fn send_ask_for_operations( + &self, + to_node: NodeId, + wishlist: OperationPrefixIds, + ) -> Result<(), NetworkError>; + + async fn send_endorsements( + &self, + node: NodeId, + endorsements: Vec, + ) -> Result<(), NetworkError>; + + async fn node_sign_message(&self, msg: Vec) -> Result; + } +} #[automock] #[async_trait] /// Network command sender interface. Can be mocked for testing -pub trait NetworkCommandSenderTrait { +pub trait NetworkCommandSenderTrait: Send + 'static { /// ban node(s) by id(s) async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; From e2243566d629d7502405a3a867d5b01be15ce327 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 27 Mar 2023 22:35:21 +0200 Subject: [PATCH 36/83] Mock the consensus controller implementation when testing bootstrap --- Cargo.lock | 2 + massa-api/src/private.rs | 2 +- massa-api/src/public.rs | 2 +- massa-bootstrap/Cargo.toml | 1 + massa-bootstrap/src/tests/scenarios.rs | 87 ++++++++----------- massa-bootstrap/src/tests/tools.rs | 21 ----- massa-consensus-exports/Cargo.toml | 3 +- .../src/test_exports/mock.rs | 55 +++++++++++- massa-factory-worker/src/tests/tools.rs | 4 +- massa-network-worker/src/tests/scenarios.rs | 2 +- massa-node/src/main.rs | 4 +- massa-protocol-worker/src/protocol_network.rs | 4 +- massa-protocol-worker/src/protocol_worker.rs | 4 +- massa-protocol-worker/src/tests/tools.rs | 6 +- .../src/worker_operations_impl.rs | 1 + 15 files changed, 110 insertions(+), 88 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80c4a206da6..08d75bc5512 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2478,6 +2478,7 @@ dependencies = [ "massa_serialization 0.1.0", "massa_signature", "massa_time", + "mockall", "nom", "num_enum", "parking_lot", @@ -2526,6 +2527,7 @@ dependencies = [ "massa_signature", "massa_storage", "massa_time", + "mockall", "nom", "serde", "serde_json", diff --git a/massa-api/src/private.rs b/massa-api/src/private.rs index 7eafb34d80b..887f5283123 100644 --- a/massa-api/src/private.rs +++ b/massa-api/src/private.rs @@ -28,7 +28,7 @@ use massa_models::{ address::Address, block::Block, block_id::BlockId, endorsement::EndorsementId, execution::EventFilter, operation::OperationId, slot::Slot, }; -use massa_network_exports::NetworkCommandSender; +use massa_network_exports::{NetworkCommandSender, NetworkCommandSenderTrait}; use massa_signature::KeyPair; use massa_wallet::Wallet; diff --git a/massa-api/src/public.rs b/massa-api/src/public.rs index eead454b8d4..de7e6878348 100644 --- a/massa-api/src/public.rs +++ b/massa-api/src/public.rs @@ -54,7 +54,7 @@ use massa_models::{ timeslots::{get_latest_block_slot_at_timestamp, time_range_to_slot_range}, version::Version, }; -use massa_network_exports::{NetworkCommandSender, NetworkConfig}; +use massa_network_exports::{NetworkCommandSender, NetworkCommandSenderTrait, NetworkConfig}; use massa_pool_exports::PoolController; use massa_signature::KeyPair; use massa_storage::Storage; diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 90a89c272fd..bc7e397bdc2 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -42,6 +42,7 @@ massa_pos_exports = { path = "../massa-pos-exports" } massa_time = { path = "../massa-time" } [dev-dependencies] +mockall = "0.11.4" bitvec = { version = "1.0", features = ["serde"] } serial_test = "1.0" massa_final_state = { path = "../massa-final-state", features = ["testing"] } diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 0736f9f70f6..08b7355c037 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -9,7 +9,6 @@ use super::{ }; use crate::tests::tools::{ get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, - wait_network_command, }; use crate::BootstrapConfig; use crate::{ @@ -18,8 +17,7 @@ use crate::{ }; use massa_async_pool::AsyncPoolConfig; use massa_consensus_exports::{ - bootstrapable_graph::BootstrapableGraph, - test_exports::{MockConsensusController, MockConsensusControllerMessage}, + bootstrapable_graph::BootstrapableGraph, test_exports::MockConsensusControllerImpl, }; use massa_executed_ops::ExecutedOpsConfig; use massa_final_state::{ @@ -38,7 +36,7 @@ use massa_models::{ }, prehash::PreHashSet, }; -use massa_network_exports::{MockNetworkCommandSender, NetworkCommand}; +use massa_network_exports::MockNetworkCommandSender; use massa_pos_exports::{ test_exports::assert_eq_pos_selection, PoSConfig, PoSFinalState, SelectorConfig, }; @@ -72,9 +70,6 @@ async fn test_bootstrap_server() { let rolls_path = PathBuf::from_str("../massa-node/base_config/initial_rolls.json").unwrap(); let genesis_address = Address::from_public_key(&KeyPair::generate().get_public_key()); - let (consensus_controller, mut consensus_event_receiver) = - MockConsensusController::new_with_receiver(); - // setup final state local config let temp_dir = TempDir::new().unwrap(); let final_state_local_config = FinalStateConfig { @@ -162,8 +157,42 @@ async fn test_bootstrap_server() { .returning(|| Ok(get_peers())); mocked1.expect_clone().return_once(move || mocked2); + let mut stream_mock1 = Box::new(MockConsensusControllerImpl::new()); + let mut stream_mock2 = Box::new(MockConsensusControllerImpl::new()); + let mut stream_mock3 = Box::new(MockConsensusControllerImpl::new()); + let mut seq = mockall::Sequence::new(); + + let sent_graph = get_boot_state(); + let sent_graph_clone = sent_graph.clone(); + stream_mock3 + .expect_get_bootstrap_part() + .times(10) + .in_sequence(&mut seq) + .returning(move |_, slot| { + if StreamingStep::Ongoing(Slot::new(1, 1)) == slot { + Ok(( + sent_graph_clone.clone(), + PreHashSet::default(), + StreamingStep::Started, + )) + } else { + Ok(( + BootstrapableGraph { + final_blocks: vec![], + }, + PreHashSet::default(), + StreamingStep::Finished(None), + )) + } + }); + stream_mock2 + .expect_clone_box() + .return_once(move || stream_mock3); + stream_mock1 + .expect_clone_box() + .return_once(move || stream_mock2); let bootstrap_manager = start_bootstrap_server::( - consensus_controller, + stream_mock1, mocked1, final_state_server.clone(), bootstrap_config.clone(), @@ -225,48 +254,6 @@ async fn test_bootstrap_server() { bridge_mock_streams(remote_bridge, bootstrap_bridge).await; }); - // intercept consensus parts being asked - let sent_graph = get_boot_state(); - let sent_graph_clone = sent_graph.clone(); - std::thread::spawn(move || loop { - consensus_event_receiver.wait_command(MassaTime::from_millis(20_000), |cmd| match &cmd { - MockConsensusControllerMessage::GetBootstrapableGraph { - execution_cursor, - response_tx, - .. - } => { - // send the consensus blocks at the 4th slot (1 for startup + 3 for safety) - // give an empty answer for any other call - if execution_cursor - == &StreamingStep::Ongoing(Slot { - period: 1, - thread: 1, - }) - { - response_tx - .send(Ok(( - sent_graph_clone.clone(), - PreHashSet::default(), - StreamingStep::Started, - ))) - .unwrap(); - } else { - response_tx - .send(Ok(( - BootstrapableGraph { - final_blocks: Vec::new(), - }, - PreHashSet::default(), - StreamingStep::Finished(None), - ))) - .unwrap(); - } - Some(()) - } - _ => None, - }); - }); - // launch the modifier thread let list_changes: Arc>> = Arc::new(RwLock::new(Vec::new())); let list_changes_clone = list_changes.clone(); diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index a06c680b956..5bdad2c7e0b 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -339,27 +339,6 @@ pub fn get_bootstrap_config(bootstrap_public_key: NodeId) -> BootstrapConfig { } } -pub async fn wait_network_command( - network_command_receiver: &mut Receiver, - timeout: MassaTime, - filter_map: F, -) -> Option -where - F: Fn(NetworkCommand) -> Option, -{ - let timer = sleep(timeout.into()); - tokio::pin!(timer); - loop { - tokio::select! { - cmd = network_command_receiver.recv() => match cmd { - Some(orig_evt) => if let Some(res_evt) = filter_map(orig_evt) { return Some(res_evt); }, - _ => panic!("network event channel died") - }, - _ = &mut timer => return None - } - } -} - /// asserts that two `BootstrapableGraph` are equal pub fn assert_eq_bootstrap_graph(v1: &BootstrapableGraph, v2: &BootstrapableGraph) { assert_eq!( diff --git a/massa-consensus-exports/Cargo.toml b/massa-consensus-exports/Cargo.toml index 3996748a415..5f6f8915e30 100644 --- a/massa-consensus-exports/Cargo.toml +++ b/massa-consensus-exports/Cargo.toml @@ -15,6 +15,7 @@ serde_json = "1.0" thiserror = "1.0" jsonrpsee = { version = "0.16.2", features = ["server"] } tokio = { version = "1.23", features = ["sync"] } +mockall = {version = "0.11.4", features = ["nightly"]} #custom modules massa_hash = { path = "../massa-hash"} massa_execution_exports = { path = "../massa-execution-exports" } @@ -28,4 +29,4 @@ massa_time = { path = "../massa-time" } massa_signature = { path = "../massa-signature" } [features] -testing = ["massa_models/testing", "massa_execution_exports/testing", "massa_pool_exports/testing", "massa_pos_exports/testing", "massa_protocol_exports/testing", "massa_storage/testing"] \ No newline at end of file +testing = ["massa_models/testing", "massa_execution_exports/testing", "massa_pool_exports/testing", "massa_pos_exports/testing", "massa_protocol_exports/testing", "massa_storage/testing"] diff --git a/massa-consensus-exports/src/test_exports/mock.rs b/massa-consensus-exports/src/test_exports/mock.rs index 70b1c256607..2a10908d635 100644 --- a/massa-consensus-exports/src/test_exports/mock.rs +++ b/massa-consensus-exports/src/test_exports/mock.rs @@ -88,15 +88,62 @@ pub enum MockConsensusControllerMessage { /// For messages with a `response_tx` field, the mock will await a response through their `response_tx` channel /// in order to simulate returning this value at the end of the call. #[derive(Clone)] -pub struct MockConsensusController(Arc>>); +pub struct ConsensusControllerImpl(Arc>>); -impl MockConsensusController { +mockall::mock! { + pub ConsensusControllerImpl {} + impl Clone for ConsensusControllerImpl { + fn clone(&self) -> Self; + } + impl ConsensusController for ConsensusControllerImpl { + fn get_block_graph_status( + &self, + start_slot: Option, + end_slot: Option, + ) -> Result; + + fn get_block_statuses(&self, ids: &[BlockId]) -> Vec; + + fn get_cliques(&self) -> Vec; + + fn get_bootstrap_part( + &self, + cursor: StreamingStep>, + execution_cursor: StreamingStep, + ) -> Result< + ( + BootstrapableGraph, + PreHashSet, + StreamingStep>, + ), + ConsensusError, + >; + + fn get_stats(&self) -> Result; + + fn get_best_parents(&self) -> Vec<(BlockId, u64)>; + + fn get_blockclique_block_at_slot(&self, slot: Slot) -> Option; + + fn get_latest_blockclique_block_at_slot(&self, slot: Slot) -> BlockId; + + fn register_block(&self, block_id: BlockId, slot: Slot, block_storage: Storage, created: bool); + + fn register_block_header(&self, block_id: BlockId, header: SecureShare); + + fn mark_invalid_block(&self, block_id: BlockId, header: SecureShare); + + fn clone_box(&self) -> Box; + } +} + +impl ConsensusControllerImpl { /// Create a new pair (mock graph controller, mpsc receiver for emitted messages) /// Note that unbounded mpsc channels are used pub fn new_with_receiver() -> (Box, ConsensusEventReceiver) { let (tx, rx) = mpsc::channel(); ( - Box::new(MockConsensusController(Arc::new(Mutex::new(tx)))), + Box::new(ConsensusControllerImpl(Arc::new(Mutex::new(tx)))), ConsensusEventReceiver(rx), ) } @@ -120,7 +167,7 @@ impl ConsensusEventReceiver { /// If the message contains a `response_tx`, /// a response from that channel is read and returned as return value. /// See the documentation of `ConsensusController` for details on each function. -impl ConsensusController for MockConsensusController { +impl ConsensusController for ConsensusControllerImpl { fn get_block_graph_status( &self, start_slot: Option, diff --git a/massa-factory-worker/src/tests/tools.rs b/massa-factory-worker/src/tests/tools.rs index afb1336d9de..5e9a4b33d66 100644 --- a/massa-factory-worker/src/tests/tools.rs +++ b/massa-factory-worker/src/tests/tools.rs @@ -1,5 +1,5 @@ use massa_consensus_exports::test_exports::{ - ConsensusEventReceiver, MockConsensusController, MockConsensusControllerMessage, + ConsensusControllerImpl, ConsensusEventReceiver, MockConsensusControllerMessage, }; use parking_lot::RwLock; use std::{ @@ -56,7 +56,7 @@ impl TestFactory { pub fn new(default_keypair: &KeyPair) -> TestFactory { let (selector_controller, selector_receiver) = MockSelectorController::new_with_receiver(); let (consensus_controller, consensus_event_receiver) = - MockConsensusController::new_with_receiver(); + ConsensusControllerImpl::new_with_receiver(); let (pool_controller, pool_receiver) = MockPoolController::new_with_receiver(); let mut storage = Storage::create_root(); let mut factory_config = FactoryConfig::default(); diff --git a/massa-network-worker/src/tests/scenarios.rs b/massa-network-worker/src/tests/scenarios.rs index bf9f03b37e7..cad74104881 100644 --- a/massa-network-worker/src/tests/scenarios.rs +++ b/massa-network-worker/src/tests/scenarios.rs @@ -31,7 +31,7 @@ use massa_models::{ use massa_network_exports::{settings::PeerTypeConnectionConfig, NodeCommand, NodeEvent}; use massa_network_exports::{ AskForBlocksInfo, BlockInfoReply, ConnectionClosureReason, ConnectionId, HandshakeErrorType, - PeerInfo, PeerType, + NetworkCommandSenderTrait, PeerInfo, PeerType, }; use massa_signature::KeyPair; use massa_time::MassaTime; diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 10f7f81ac3c..e564f82f7d4 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -49,7 +49,7 @@ use massa_models::config::constants::{ T0, THREAD_COUNT, VERSION, }; use massa_models::config::CONSENSUS_BOOTSTRAP_PART_SIZE; -use massa_network_exports::{Establisher, NetworkConfig, NetworkManager}; +use massa_network_exports::{Establisher, NetworkCommandSender, NetworkConfig, NetworkManager}; use massa_network_worker::start_network_controller; use massa_pool_exports::{PoolChannels, PoolConfig, PoolManager}; use massa_pool_worker::start_pool_controller; @@ -508,7 +508,7 @@ async fn launch( // launch bootstrap server let addr = &bootstrap_config.listen_addr.unwrap(); // TODO: use std::net::TcpStream - let bootstrap_manager = start_bootstrap_server::( + let bootstrap_manager = start_bootstrap_server::( consensus_controller.clone(), network_command_sender.clone(), final_state.clone(), diff --git a/massa-protocol-worker/src/protocol_network.rs b/massa-protocol-worker/src/protocol_network.rs index 52acc97cb07..b5962f00cc2 100644 --- a/massa-protocol-worker/src/protocol_network.rs +++ b/massa-protocol-worker/src/protocol_network.rs @@ -17,7 +17,9 @@ use massa_models::{ prehash::{CapacityAllocator, PreHashSet}, secure_share::{Id, SecureShare}, }; -use massa_network_exports::{AskForBlocksInfo, BlockInfoReply, NetworkEvent}; +use massa_network_exports::{ + AskForBlocksInfo, BlockInfoReply, NetworkCommandSenderTrait, NetworkEvent, +}; use massa_protocol_exports::ProtocolError; use massa_serialization::Serializer; use massa_storage::Storage; diff --git a/massa-protocol-worker/src/protocol_worker.rs b/massa-protocol-worker/src/protocol_worker.rs index e63924a0df4..057b8334645 100644 --- a/massa-protocol-worker/src/protocol_worker.rs +++ b/massa-protocol-worker/src/protocol_worker.rs @@ -20,7 +20,9 @@ use massa_models::{ operation::{OperationId, SecureShareOperation}, prehash::{CapacityAllocator, PreHashMap, PreHashSet}, }; -use massa_network_exports::{AskForBlocksInfo, NetworkCommandSender, NetworkEventReceiver}; +use massa_network_exports::{ + AskForBlocksInfo, NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, +}; use massa_pool_exports::PoolController; use massa_protocol_exports::{ ProtocolCommand, ProtocolConfig, ProtocolError, ProtocolManagementCommand, ProtocolManager, diff --git a/massa-protocol-worker/src/tests/tools.rs b/massa-protocol-worker/src/tests/tools.rs index 47491208e5d..8fb66b160c1 100644 --- a/massa-protocol-worker/src/tests/tools.rs +++ b/massa-protocol-worker/src/tests/tools.rs @@ -1,6 +1,6 @@ use crate::start_protocol_controller; use futures::Future; -use massa_consensus_exports::test_exports::{ConsensusEventReceiver, MockConsensusController}; +use massa_consensus_exports::test_exports::{ConsensusControllerImpl, ConsensusEventReceiver}; use massa_models::{ block::SecureShareBlock, block_id::BlockId, node::NodeId, operation::SecureShareOperation, prehash::PreHashSet, @@ -38,7 +38,7 @@ where let (pool_controller, pool_event_receiver) = MockPoolController::new_with_receiver(); let (consensus_controller, consensus_event_receiver) = - MockConsensusController::new_with_receiver(); + ConsensusControllerImpl::new_with_receiver(); // start protocol controller let (protocol_command_sender, protocol_command_receiver) = mpsc::channel(protocol_config.controller_channel_size); @@ -107,7 +107,7 @@ where MockNetworkController::new(); let (pool_controller, mock_pool_receiver) = MockPoolController::new_with_receiver(); let (consensus_controller, mock_consensus_receiver) = - MockConsensusController::new_with_receiver(); + ConsensusControllerImpl::new_with_receiver(); let storage = Storage::create_root(); // start protocol controller let (protocol_command_sender, protocol_command_receiver) = diff --git a/massa-protocol-worker/src/worker_operations_impl.rs b/massa-protocol-worker/src/worker_operations_impl.rs index e9c3dd74c41..68571a2ecd6 100644 --- a/massa-protocol-worker/src/worker_operations_impl.rs +++ b/massa-protocol-worker/src/worker_operations_impl.rs @@ -17,6 +17,7 @@ use massa_models::{ operation::{OperationPrefixIds, SecureShareOperation}, prehash::CapacityAllocator, }; +use massa_network_exports::NetworkCommandSenderTrait; use massa_protocol_exports::ProtocolError; use massa_time::TimeError; use std::pin::Pin; From ed056e810cab31ca8ef4a5129e4db63c17ee40b9 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 12:56:24 +0200 Subject: [PATCH 37/83] Replace the mock establisher with `automock` of listener and connector --- massa-bootstrap/src/establisher.rs | 2 + massa-bootstrap/src/tests/mod.rs | 1 - massa-bootstrap/src/tests/scenarios.rs | 88 ++++++++++---------------- massa-bootstrap/src/tests/tools.rs | 39 +----------- 4 files changed, 35 insertions(+), 95 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 8f8106801bf..7a0f32204ed 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -15,11 +15,13 @@ pub trait Duplex: impl Duplex for tokio::net::TcpStream {} /// Specifies a common interface that can be used by standard, or mockers +#[cfg_attr(test, mockall::automock)] pub trait BSListener { fn accept(&mut self) -> io::Result<(TcpStream, SocketAddr)>; } /// Specifies a common interface that can be used by standard, or mockers +#[cfg_attr(test, mockall::automock)] pub trait BSConnector { fn connect(&mut self, addr: SocketAddr) -> io::Result; } diff --git a/massa-bootstrap/src/tests/mod.rs b/massa-bootstrap/src/tests/mod.rs index 42f01eae9dc..9f9e1914cae 100644 --- a/massa-bootstrap/src/tests/mod.rs +++ b/massa-bootstrap/src/tests/mod.rs @@ -1,6 +1,5 @@ // Copyright (c) 2022 MASSA LABS mod binders; -pub(crate) mod mock_establisher; mod scenarios; pub(crate) mod tools; diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 08b7355c037..decce5dbf01 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -1,17 +1,14 @@ // Copyright (c) 2022 MASSA LABS -use super::{ - mock_establisher, - tools::{ - bridge_mock_streams, get_boot_state, get_peers, get_random_final_state_bootstrap, - get_random_ledger_changes, - }, +use super::tools::{ + get_boot_state, get_peers, get_random_final_state_bootstrap, get_random_ledger_changes, }; use crate::tests::tools::{ get_random_async_pool_changes, get_random_executed_ops_changes, get_random_pos_changes, }; use crate::BootstrapConfig; use crate::{ + establisher::{MockBSConnector, MockBSListener}, get_state, start_bootstrap_server, tests::tools::{assert_eq_bootstrap_graph, get_bootstrap_config}, }; @@ -45,12 +42,7 @@ use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; use serial_test::serial; -use std::{ - path::PathBuf, - str::FromStr, - sync::{atomic::Ordering, Arc}, - time::Duration, -}; +use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use tempfile::TempDir; use tokio::net::TcpStream; @@ -146,8 +138,8 @@ async fn test_bootstrap_server() { let final_state_client_clone = final_state_client.clone(); let final_state_server_clone = final_state_server.clone(); - // Setup bootstrap mock-duplex - let (mut mock_bs_listener, bootstrap_interface) = mock_establisher::new(); + let (mock_bs_listener, mock_remote_connector) = conn_establishment_mocks(); + // Setup network command mock-story: hard-code the result of getting bootstrap peers let mut mocked1 = MockNetworkCommandSender::new(); let mut mocked2 = MockNetworkCommandSender::new(); @@ -196,9 +188,7 @@ async fn test_bootstrap_server() { mocked1, final_state_server.clone(), bootstrap_config.clone(), - mock_bs_listener - .get_listener(&bootstrap_config.listen_addr.unwrap()) - .unwrap(), + mock_bs_listener, keypair.clone(), Version::from_str("TEST.1.10").unwrap(), ) @@ -206,12 +196,11 @@ async fn test_bootstrap_server() { .unwrap(); // launch the get_state process - let (mut mock_remote_connector, mut remote_interface) = mock_establisher::new(); let get_state_h = tokio::spawn(async move { get_state( bootstrap_config, final_state_client_clone, - mock_remote_connector.get_connector(), + mock_remote_connector, Version::from_str("TEST.1.10").unwrap(), MassaTime::now().unwrap().saturating_sub(1000.into()), None, @@ -220,40 +209,6 @@ async fn test_bootstrap_server() { .unwrap() }); - // accept connection attempt from remote - let remote_bridge = std::thread::spawn(move || { - let (remote_rw, conn_addr, waker) = remote_interface - .wait_connection_attempt_from_controller() - .expect("timeout waiting for connection attempt from remote"); - let expect_conn_addr = bootstrap_config.bootstrap_list[0].0; - assert_eq!( - conn_addr, expect_conn_addr, - "client connected to wrong bootstrap ip" - ); - waker.store(true, Ordering::Relaxed); - remote_rw - }); - - // connect to bootstrap - let remote_addr = std::net::SocketAddr::from_str("82.245.72.98:10000").unwrap(); // not checked - let bootstrap_bridge = tokio::time::timeout( - std::time::Duration::from_millis(1000), - bootstrap_interface.connect_to_controller(&remote_addr), - ) - .await - .expect("timeout while connecting to bootstrap") - .expect("could not connect to bootstrap"); - - // launch bridge - bootstrap_bridge.set_nonblocking(true).unwrap(); - let bootstrap_bridge = TcpStream::from_std(bootstrap_bridge).unwrap(); - let bridge = tokio::spawn(async move { - let remote_bridge = remote_bridge.join().unwrap(); - remote_bridge.set_nonblocking(true).unwrap(); - let remote_bridge = TcpStream::from_std(remote_bridge).unwrap(); - bridge_mock_streams(remote_bridge, bootstrap_bridge).await; - }); - // launch the modifier thread let list_changes: Arc>> = Arc::new(RwLock::new(Vec::new())); let list_changes_clone = list_changes.clone(); @@ -282,9 +237,6 @@ async fn test_bootstrap_server() { .await .expect("error while waiting for get_state to finish"); - // wait for bridge - bridge.await.expect("bridge join failed"); - // apply the changes to the server state before matching with the client { let mut final_state_server_write = final_state_server.write(); @@ -340,3 +292,27 @@ async fn test_bootstrap_server() { server_selector_manager.stop(); client_selector_manager.stop(); } + +fn conn_establishment_mocks() -> (MockBSListener, MockBSConnector) { + // Setup the server/client connection + // Bind a TcpListener to localhost on a specific port + let listener = std::net::TcpListener::bind("127.0.0.1:8069").unwrap(); + let (conn_tx, conn_rx) = std::sync::mpsc::sync_channel(100); + let conn = std::thread::spawn(|| std::net::TcpStream::connect("127.0.0.1:8069").unwrap()); + std::thread::spawn(move || loop { + conn_tx.send(listener.accept().unwrap()).unwrap() + }); + + // Mock the connection setups + let mut mock_bs_listener = MockBSListener::new(); + mock_bs_listener + .expect_accept() + .times(2) + .returning(move || Ok(conn_rx.recv().unwrap())); + let mut mock_remote_connector = MockBSConnector::new(); + mock_remote_connector + .expect_connect() + .times(1) + .return_once(move |_| Ok(conn.join().unwrap())); + (mock_bs_listener, mock_remote_connector) +} diff --git a/massa-bootstrap/src/tests/tools.rs b/massa-bootstrap/src/tests/tools.rs index 5bdad2c7e0b..17fd027a0db 100644 --- a/massa-bootstrap/src/tests/tools.rs +++ b/massa-bootstrap/src/tests/tools.rs @@ -1,6 +1,5 @@ // Copyright (c) 2022 MASSA LABS -use crate::establisher::Duplex; use crate::settings::{BootstrapConfig, IpType}; use bitvec::vec::BitVec; use massa_async_pool::test_exports::{create_async_pool, get_random_message}; @@ -47,7 +46,7 @@ use massa_models::{ secure_share::SecureShareContent, slot::Slot, }; -use massa_network_exports::{BootstrapPeers, NetworkCommand}; +use massa_network_exports::BootstrapPeers; use massa_pos_exports::{CycleInfo, DeferredCredits, PoSChanges, PoSFinalState, ProductionStats}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::KeyPair; @@ -60,7 +59,6 @@ use std::{ net::{IpAddr, Ipv4Addr, SocketAddr}, path::PathBuf, }; -use tokio::{sync::mpsc::Receiver, time::sleep}; // Use loop-back address. use port 0 to auto-assign a port pub const BASE_BOOTSTRAP_IP: IpAddr = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1)); @@ -441,38 +439,3 @@ pub fn get_peers() -> BootstrapPeers { "82.220.123.78".parse().unwrap(), ]) } - -pub async fn bridge_mock_streams(mut side1: D, mut side2: D) { - let mut buf1 = vec![0u8; 1024]; - let mut buf2 = vec![0u8; 1024]; - loop { - tokio::select! { - res1 = side1.read(&mut buf1) => match res1 { - Ok(n1) => { - if n1 == 0 { - return; - } - if side2.write_all(&buf1[..n1]).await.is_err() { - return; - } - }, - Err(_err) => { - return; - } - }, - res2 = side2.read(&mut buf2) => match res2 { - Ok(n2) => { - if n2 == 0 { - return; - } - if side1.write_all(&buf2[..n2]).await.is_err() { - return; - } - }, - Err(_err) => { - return; - } - }, - } - } -} From 877fb00516041f3792e9918c9e78188ba5a08f54 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 13:27:49 +0200 Subject: [PATCH 38/83] Clarify the mock-connection setup --- massa-bootstrap/src/tests/scenarios.rs | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index decce5dbf01..86a68c54977 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -297,18 +297,27 @@ fn conn_establishment_mocks() -> (MockBSListener, MockBSConnector) { // Setup the server/client connection // Bind a TcpListener to localhost on a specific port let listener = std::net::TcpListener::bind("127.0.0.1:8069").unwrap(); + + // Due to the limitations of the mocking system, the listener must loop-accept in a dedicated + // thread. We use a channel to make the connection available in the mocked `accept` method let (conn_tx, conn_rx) = std::sync::mpsc::sync_channel(100); let conn = std::thread::spawn(|| std::net::TcpStream::connect("127.0.0.1:8069").unwrap()); - std::thread::spawn(move || loop { - conn_tx.send(listener.accept().unwrap()).unwrap() - }); + std::thread::Builder::new() + .name("mock-listen-loop".to_string()) + .spawn(move || loop { + conn_tx.send(listener.accept().unwrap()).unwrap() + }) + .unwrap(); // Mock the connection setups + // TODO: Why is it twice, and not just once? let mut mock_bs_listener = MockBSListener::new(); mock_bs_listener .expect_accept() .times(2) + // Mock the `accept` method here by receiving from the listen-loop thread .returning(move || Ok(conn_rx.recv().unwrap())); + let mut mock_remote_connector = MockBSConnector::new(); mock_remote_connector .expect_connect() From 512a4ff8948e3e2029e8f830b0e0d17e2f9fff8e Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 16:42:15 +0200 Subject: [PATCH 39/83] Remove the vestigial mock_establisher.rs file --- massa-bootstrap/src/tests/mock_establisher.rs | 164 ------------------ 1 file changed, 164 deletions(-) delete mode 100644 massa-bootstrap/src/tests/mock_establisher.rs diff --git a/massa-bootstrap/src/tests/mock_establisher.rs b/massa-bootstrap/src/tests/mock_establisher.rs deleted file mode 100644 index 802647dfd5a..00000000000 --- a/massa-bootstrap/src/tests/mock_establisher.rs +++ /dev/null @@ -1,164 +0,0 @@ -// Copyright (c) 2022 MASSA LABS -use crossbeam::channel::{bounded, Receiver, Sender}; -use massa_models::config::CHANNEL_SIZE; -use socket2 as _; -use std::io; -use std::net::{SocketAddr, TcpListener, TcpStream}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::Arc; - -use crate::establisher::{BSConnector, BSListener}; - -pub fn new() -> (MockEstablisher, MockEstablisherInterface) { - let (connection_listener_tx, connection_listener_rx) = - bounded::<(SocketAddr, Sender)>(CHANNEL_SIZE); - - let (connection_connector_tx, connection_connector_rx) = - bounded::<(TcpStream, SocketAddr, Arc)>(CHANNEL_SIZE); - - ( - MockEstablisher { - connection_listener_rx: Some(connection_listener_rx), - connection_connector_tx, - }, - MockEstablisherInterface { - connection_listener_tx: Some(connection_listener_tx), - connection_connector_rx, - }, - ) -} - -#[derive(Debug)] -pub struct MockListener { - connection_listener_rx: crossbeam::channel::Receiver<(SocketAddr, Sender)>, // (controller, mock) -} - -impl BSListener for MockListener { - fn accept(&mut self) -> std::io::Result<(TcpStream, SocketAddr)> { - let (_addr, sender) = self.connection_listener_rx.recv().map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener accept channel from Establisher closed".to_string(), - ) - })?; - let duplex_controller = TcpListener::bind("localhost:0").unwrap(); - let duplex_mock = TcpStream::connect(duplex_controller.local_addr().unwrap()).unwrap(); - let duplex_controller = duplex_controller.accept().unwrap(); - - // Tokio `from_std` have non-blocking Tcp objects as a requirement - duplex_mock.set_nonblocking(true).unwrap(); - - sender.send(duplex_mock).map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener accept return \"oneshot\" channel to Establisher closed".to_string(), - ) - })?; - - Ok(duplex_controller) - } -} - -#[derive(Debug)] -pub struct MockConnector { - connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, -} - -impl BSConnector for MockConnector { - fn connect(&mut self, addr: SocketAddr) -> std::io::Result { - let duplex_mock = TcpListener::bind(addr).unwrap(); - let duplex_controller = TcpStream::connect(addr).unwrap(); - let duplex_mock = duplex_mock.accept().unwrap(); - - // Used to see if the connection is accepted - let waker = Arc::new(AtomicBool::from(false)); - let provided_waker = Arc::clone(&waker); - - let sender = self.connection_connector_tx.clone(); - let send = std::thread::spawn(move || { - // send new connection to mock - sender - .send((duplex_mock.0, addr, provided_waker)) - .map_err(|_err| { - io::Error::new( - io::ErrorKind::Other, - "MockConnector connect channel to Establisher closed".to_string(), - ) - }) - .unwrap(); - }); - - while !waker.load(Ordering::Relaxed) { - std::thread::yield_now(); - } - send.join().unwrap(); - Ok(duplex_controller) - } -} - -#[derive(Debug)] -pub struct MockEstablisher { - connection_listener_rx: Option)>>, - connection_connector_tx: Sender<(TcpStream, SocketAddr, Arc)>, -} - -impl MockEstablisher { - pub(crate) fn get_listener(&mut self, _addr: &SocketAddr) -> io::Result { - Ok(MockListener { - connection_listener_rx: self - .connection_listener_rx - .take() - .expect("MockEstablisher get_listener called more than once"), - }) - } - - pub(crate) fn get_connector(&mut self) -> MockConnector { - // create connector stream - - MockConnector { - connection_connector_tx: self.connection_connector_tx.clone(), - } - } -} - -pub struct MockEstablisherInterface { - connection_listener_tx: Option)>>, - connection_connector_rx: Receiver<(TcpStream, SocketAddr, Arc)>, -} - -impl MockEstablisherInterface { - pub async fn connect_to_controller(&self, addr: &SocketAddr) -> io::Result { - let sender = self.connection_listener_tx.as_ref().ok_or_else(|| { - io::Error::new( - io::ErrorKind::Other, - "mock connect_to_controller_listener channel not initialized".to_string(), - ) - })?; - let (response_tx, response_rx) = bounded::(1); - sender.send((*addr, response_tx)).map_err(|_err| { - io::Error::new( - io::ErrorKind::Other, - "mock connect_to_controller_listener channel to listener closed".to_string(), - ) - })?; - let duplex_mock = response_rx.recv().map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener connect_to_controller_listener channel from listener closed" - .to_string(), - ) - })?; - Ok(duplex_mock) - } - - pub fn wait_connection_attempt_from_controller( - &mut self, - ) -> io::Result<(TcpStream, SocketAddr, Arc)> { - self.connection_connector_rx.recv().map_err(|_| { - io::Error::new( - io::ErrorKind::Other, - "MockListener get_connect_stream channel from connector closed".to_string(), - ) - }) - } -} From 337d3fdfdacefd36cd48c571ac86d3504334f8e1 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 16:42:58 +0200 Subject: [PATCH 40/83] Move some of `get_stat` pre-loop code to helper functions --- massa-bootstrap/src/client.rs | 84 ++++++++++++++++++++--------------- 1 file changed, 48 insertions(+), 36 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index b63c0e108de..e86009e30eb 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -424,46 +424,13 @@ pub async fn get_state( end_timestamp: Option, ) -> Result { massa_trace!("bootstrap.lib.get_state", {}); - let now = MassaTime::now()?; // if we are before genesis, do not bootstrap - if now < genesis_timestamp { - massa_trace!("bootstrap.lib.get_state.init_from_scratch", {}); - // init final state - { - let mut final_state_guard = final_state.write(); - // load ledger from initial ledger file - final_state_guard - .ledger - .load_initial_ledger() - .map_err(|err| { - BootstrapError::GeneralError(format!("could not load initial ledger: {}", err)) - })?; - // create the initial cycle of PoS cycle_history - final_state_guard.pos_state.create_initial_cycle(); - } - return Ok(GlobalBootstrapState::new(final_state)); + if MassaTime::now()? < genesis_timestamp { + return get_genesis_state(final_state); } // we filter the bootstrap list to keep only the ip addresses we are compatible with - let mut filtered_bootstrap_list = filter_bootstrap_list( - bootstrap_config.bootstrap_list.clone(), - bootstrap_config.bootstrap_protocol, - ); - - // we are after genesis => bootstrap - massa_trace!("bootstrap.lib.get_state.init_from_others", {}); - if filtered_bootstrap_list.is_empty() { - return Err(BootstrapError::GeneralError( - "no bootstrap nodes found in list".into(), - )); - } - - // we shuffle the list - filtered_bootstrap_list.shuffle(&mut StdRng::from_entropy()); - - // we remove the duplicated node ids (if a bootstrap server appears both with its IPv4 and IPv6 address) - let mut unique_node_ids: HashSet = HashSet::new(); - filtered_bootstrap_list.retain(|e| unique_node_ids.insert(e.1)); + let filtered_bootstrap_list = get_bootstrap_list_iter(bootstrap_config)?; let mut next_bootstrap_message: BootstrapClientMessage = BootstrapClientMessage::AskBootstrapPart { @@ -516,3 +483,48 @@ pub async fn get_state( } } } + +fn get_genesis_state( + final_state: Arc>, +) -> Result { + massa_trace!("bootstrap.lib.get_state.init_from_scratch", {}); + // init final state + { + let mut final_state_guard = final_state.write(); + // load ledger from initial ledger file + final_state_guard + .ledger + .load_initial_ledger() + .map_err(|err| { + BootstrapError::GeneralError(format!("could not load initial ledger: {}", err)) + })?; + // create the initial cycle of PoS cycle_history + final_state_guard.pos_state.create_initial_cycle(); + } + return Ok(GlobalBootstrapState::new(final_state)); +} + +fn get_bootstrap_list_iter( + bootstrap_config: &BootstrapConfig, +) -> Result, BootstrapError> { + let mut filtered_bootstrap_list = filter_bootstrap_list( + bootstrap_config.bootstrap_list.clone(), + bootstrap_config.bootstrap_protocol, + ); + + // we are after genesis => bootstrap + massa_trace!("bootstrap.lib.get_state.init_from_others", {}); + if filtered_bootstrap_list.is_empty() { + return Err(BootstrapError::GeneralError( + "no bootstrap nodes found in list".into(), + )); + } + + // we shuffle the list + filtered_bootstrap_list.shuffle(&mut StdRng::from_entropy()); + + // we remove the duplicated node ids (if a bootstrap server appears both with its IPv4 and IPv6 address) + let mut unique_node_ids: HashSet = HashSet::new(); + filtered_bootstrap_list.retain(|e| unique_node_ids.insert(e.1)); + Ok(filtered_bootstrap_list) +} From 4bc6611ee7e4c0e1bcf204dd0cc4cab66d33c3f7 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 16:51:48 +0200 Subject: [PATCH 41/83] Move everything from tokio to std::net::TcpStream --- Cargo.lock | 13 ------------- massa-bootstrap/Cargo.toml | 4 ---- massa-bootstrap/src/client.rs | 4 ++-- massa-bootstrap/src/client_binder.rs | 25 ++++++++++--------------- massa-bootstrap/src/establisher.rs | 4 ++-- massa-bootstrap/src/server.rs | 9 ++------- massa-bootstrap/src/server_binder.rs | 23 ++++++++++------------- massa-bootstrap/src/tests/binders.rs | 20 +++++++++----------- massa-bootstrap/src/tests/scenarios.rs | 3 +-- 9 files changed, 36 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 08d75bc5512..7280b9d9680 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,18 +173,6 @@ dependencies = [ "event-listener", ] -[[package]] -name = "async-speed-limit" -version = "0.4.0" -source = "git+https://github.com/adrien-zinger/async-speed-limit?rev=36d79e0#36d79e063144cf0cdfc258c9e127c84f10a53b57" -dependencies = [ - "futures-core", - "futures-io", - "futures-timer", - "pin-project-lite", - "tokio", -] - [[package]] name = "async-trait" version = "0.1.68" @@ -2457,7 +2445,6 @@ dependencies = [ name = "massa_bootstrap" version = "0.1.0" dependencies = [ - "async-speed-limit", "bitvec", "crossbeam", "displaydoc", diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index bc7e397bdc2..28533e6a320 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -7,10 +7,6 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -async-speed-limit = { git = "https://github.com/adrien-zinger/async-speed-limit", rev = "36d79e0", features = [ - "default", - "tokio", -] } displaydoc = "0.2" num_enum = "0.5" nom = "7.1" diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index e86009e30eb..59b81500855 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -376,12 +376,12 @@ fn connect_to_server( bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, -) -> Result, Box> { +) -> Result, Box> { let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; socket.set_nonblocking(true).unwrap(); Ok(BootstrapClientBinder::new( // this from_std will panic if this method doesn't exist within an async runtime... - tokio::net::TcpStream::from_std(socket).unwrap(), + socket, *pub_key, bootstrap_config.into(), )) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index a34a4de94cc..725f09a91a5 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -7,23 +7,19 @@ use crate::messages::{ BootstrapServerMessageDeserializer, }; use crate::settings::BootstrapClientConfig; -use async_speed_limit::clock::StandardClock; -use async_speed_limit::{Limiter, Resource}; use massa_hash::{Hash, HASH_SIZE_BYTES}; use massa_models::serialization::{DeserializeMinBEInt, SerializeMinBEInt}; use massa_models::version::{Version, VersionSerializer}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::{PublicKey, Signature, SIGNATURE_SIZE_BYTES}; use rand::{rngs::StdRng, RngCore, SeedableRng}; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWriteExt; /// Bootstrap client binder pub struct BootstrapClientBinder { // max_bootstrap_message_size: u32, size_field_len: usize, remote_pubkey: PublicKey, - duplex: Resource, + duplex: D, prev_message: Option, version_serializer: VersionSerializer, cfg: BootstrapClientConfig, @@ -41,7 +37,7 @@ impl BootstrapClientBinder { BootstrapClientBinder { size_field_len, remote_pubkey, - duplex: ::new(cfg.max_bytes_read_write).limit(duplex), + duplex, prev_message: None, version_serializer: VersionSerializer::new(), cfg, @@ -60,7 +56,7 @@ impl BootstrapClientBinder { vec![0u8; version_ser.len() + self.cfg.randomness_size_bytes]; version_random_bytes[..version_ser.len()].clone_from_slice(&version_ser); StdRng::from_entropy().fill_bytes(&mut version_random_bytes[version_ser.len()..]); - self.duplex.write_all(&version_random_bytes).await?; + self.duplex.write_all(&version_random_bytes)?; Hash::compute_from(&version_random_bytes) }; @@ -74,14 +70,14 @@ impl BootstrapClientBinder { // read signature let sig = { let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; - self.duplex.read_exact(&mut sig_bytes).await?; + self.duplex.read_exact(&mut sig_bytes)?; Signature::from_bytes(&sig_bytes)? }; // read message length let msg_len = { let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..]).await?; + self.duplex.read_exact(&mut msg_len_bytes[..])?; u32::from_be_bytes_min(&msg_len_bytes, self.cfg.max_bootstrap_message_size)?.0 }; @@ -93,8 +89,7 @@ impl BootstrapClientBinder { let mut sig_msg_bytes = vec![0u8; HASH_SIZE_BYTES + (msg_len as usize)]; sig_msg_bytes[..HASH_SIZE_BYTES].copy_from_slice(prev_message.to_bytes()); self.duplex - .read_exact(&mut sig_msg_bytes[HASH_SIZE_BYTES..]) - .await?; + .read_exact(&mut sig_msg_bytes[HASH_SIZE_BYTES..])?; let msg_hash = Hash::compute_from(&sig_msg_bytes); self.remote_pubkey.verify_signature(&msg_hash, &sig)?; let (_, msg) = message_deserializer @@ -104,7 +99,7 @@ impl BootstrapClientBinder { } else { self.prev_message = Some(Hash::compute_from(&sig.to_bytes())); let mut sig_msg_bytes = vec![0u8; msg_len as usize]; - self.duplex.read_exact(&mut sig_msg_bytes[..]).await?; + self.duplex.read_exact(&mut sig_msg_bytes[..])?; let msg_hash = Hash::compute_from(&sig_msg_bytes); self.remote_pubkey.verify_signature(&msg_hash, &sig)?; let (_, msg) = message_deserializer @@ -138,7 +133,7 @@ impl BootstrapClientBinder { self.prev_message = Some(Hash::compute_from(&hash_data)); // send old previous message - self.duplex.write_all(prev_message).await?; + self.duplex.write_all(prev_message)?; } else { // there was no previous message @@ -149,11 +144,11 @@ impl BootstrapClientBinder { // send message length { let msg_len_bytes = msg_len.to_be_bytes_min(self.cfg.max_bootstrap_message_size)?; - self.duplex.write_all(&msg_len_bytes).await?; + self.duplex.write_all(&msg_len_bytes)?; } // send message - self.duplex.write_all(&msg_bytes).await?; + self.duplex.write_all(&msg_bytes)?; Ok(()) } } diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 7a0f32204ed..2969f3edb57 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -8,11 +8,11 @@ use std::{ /// duplex connection pub trait Duplex: // static because need to send between threads :( - 'static + Send + tokio::io::AsyncReadExt + tokio::io::AsyncWriteExt + std::marker::Unpin + 'static + Send + io::Read + io::Write { } -impl Duplex for tokio::net::TcpStream {} +impl Duplex for std::net::TcpStream {} /// Specifies a common interface that can be used by standard, or mockers #[cfg_attr(test, mockall::automock)] diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 82ff64884d0..ef27beca2ed 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -45,15 +45,12 @@ use massa_time::MassaTime; use parking_lot::RwLock; use std::{ collections::HashMap, - net::{IpAddr, SocketAddr}, + net::{IpAddr, SocketAddr, TcpStream}, sync::Arc, thread, time::{Duration, Instant}, }; -use tokio::{ - net::TcpStream, - runtime::{self, Handle, Runtime}, -}; +use tokio::runtime::{self, Handle, Runtime}; use tracing::{debug, error, info, warn}; use crate::{ @@ -253,8 +250,6 @@ impl BootstrapServer<'_, D, C> ) -> Result>, Box> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; - msg.set_nonblocking(true).unwrap(); - let msg = TcpStream::from_std(msg).unwrap(); match listener_tx.send((msg, addr)) { Ok(_) => continue, Err(SendError((dplx, remote_addr))) => { diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index c779b97faf0..42bff5a9241 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -7,8 +7,6 @@ use crate::messages::{ BootstrapServerMessageSerializer, }; use crate::settings::BootstrapSrvBindCfg; -use async_speed_limit::clock::StandardClock; -use async_speed_limit::{Limiter, Resource}; use massa_hash::Hash; use massa_hash::HASH_SIZE_BYTES; use massa_models::serialization::{DeserializeMinBEInt, SerializeMinBEInt}; @@ -20,7 +18,6 @@ use std::convert::TryInto; use std::net::SocketAddr; use std::thread; use std::time::Duration; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::runtime::Handle; use tokio::time::error::Elapsed; use tracing::error; @@ -34,7 +31,7 @@ pub struct BootstrapServerBinder { randomness_size_bytes: usize, size_field_len: usize, local_keypair: KeyPair, - duplex: Resource, + duplex: D, prev_message: Option, version_serializer: VersionSerializer, version_deserializer: VersionDeserializer, @@ -51,7 +48,7 @@ impl BootstrapServerBinder { #[allow(clippy::too_many_arguments)] pub fn new(duplex: D, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { let BootstrapSrvBindCfg { - max_bytes_read_write: limit, + max_bytes_read_write: _limit, max_bootstrap_message_size, thread_count, max_datastore_key_length, @@ -65,7 +62,7 @@ impl BootstrapServerBinder { max_consensus_block_ids: consensus_bootstrap_part_size, size_field_len, local_keypair, - duplex: ::new(limit).limit(duplex), + duplex, prev_message: None, thread_count, max_datastore_key_length, @@ -85,7 +82,7 @@ impl BootstrapServerBinder { self.version_serializer .serialize(&version, &mut version_bytes)?; let mut msg_bytes = vec![0u8; version_bytes.len() + self.randomness_size_bytes]; - self.duplex.read_exact(&mut msg_bytes).await?; + self.duplex.read_exact(&mut msg_bytes)?; let (_, received_version) = self .version_deserializer .deserialize::(&msg_bytes[..version_bytes.len()]) @@ -182,16 +179,16 @@ impl BootstrapServerBinder { }; // send signature - self.duplex.write_all(&sig.to_bytes()).await?; + self.duplex.write_all(&sig.to_bytes())?; // send message length { let msg_len_bytes = msg_len.to_be_bytes_min(self.max_bootstrap_message_size)?; - self.duplex.write_all(&msg_len_bytes).await?; + self.duplex.write_all(&msg_len_bytes)?; } // send message - self.duplex.write_all(&msg_bytes).await?; + self.duplex.write_all(&msg_bytes)?; // save prev sig self.prev_message = Some(Hash::compute_from(&sig.to_bytes())); @@ -206,7 +203,7 @@ impl BootstrapServerBinder { let received_prev_hash = { if self.prev_message.is_some() { let mut hash_bytes = [0u8; HASH_SIZE_BYTES]; - self.duplex.read_exact(&mut hash_bytes).await?; + self.duplex.read_exact(&mut hash_bytes)?; Some(Hash::from_bytes(&hash_bytes)) } else { None @@ -216,13 +213,13 @@ impl BootstrapServerBinder { // read message length let msg_len = { let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..]).await?; + self.duplex.read_exact(&mut msg_len_bytes[..])?; u32::from_be_bytes_min(&msg_len_bytes, self.max_bootstrap_message_size)?.0 }; // read message let mut msg_bytes = vec![0u8; msg_len as usize]; - self.duplex.read_exact(&mut msg_bytes).await?; + self.duplex.read_exact(&mut msg_bytes)?; // check previous hash if received_prev_hash != self.prev_message { diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 9bb07061165..8f1adaa880a 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -65,10 +65,10 @@ impl BootstrapClientBinder { #[serial] async fn test_binders() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let server = std::net::TcpListener::bind("localhost:0").unwrap(); let addr = server.local_addr().unwrap(); - let client = tokio::net::TcpStream::connect(addr).await.unwrap(); - let server = server.accept().await.unwrap(); + let client = std::net::TcpStream::connect(addr).unwrap(); + let server = server.accept().unwrap(); // let (client, server) = duplex(1000000); let mut server = BootstrapServerBinder::new( server.0, @@ -168,11 +168,9 @@ async fn test_binders() { async fn test_binders_double_send_server_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); - let client = tokio::net::TcpStream::connect(server.local_addr().unwrap()) - .await - .unwrap(); - let server = server.accept().await.unwrap(); + let server = std::net::TcpListener::bind("localhost:0").unwrap(); + let client = std::net::TcpStream::connect(server.local_addr().unwrap()).unwrap(); + let server = server.accept().unwrap(); let mut server = BootstrapServerBinder::new( server.0, @@ -257,10 +255,10 @@ async fn test_binders_double_send_server_works() { async fn test_binders_try_double_send_client_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; - let server = tokio::net::TcpListener::bind("localhost:0").await.unwrap(); + let server = std::net::TcpListener::bind("localhost:0").unwrap(); let addr = server.local_addr().unwrap(); - let client = tokio::net::TcpStream::connect(addr).await.unwrap(); - let server = server.accept().await.unwrap(); + let client = std::net::TcpStream::connect(addr).unwrap(); + let server = server.accept().unwrap(); let mut server = BootstrapServerBinder::new( server.0, server_keypair.clone(), diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 86a68c54977..587d51775d1 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -42,9 +42,8 @@ use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; use serial_test::serial; -use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; +use std::{net::TcpStream, path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use tempfile::TempDir; -use tokio::net::TcpStream; lazy_static::lazy_static! { pub static ref BOOTSTRAP_CONFIG_KEYPAIR: (BootstrapConfig, KeyPair) = { From c9f107dc27f73c380c30622344f12e4df8037391 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 18:10:30 +0200 Subject: [PATCH 42/83] Comment out the use of tokio timeeouts --- massa-bootstrap/src/client.rs | 92 +++++++++++++++------------- massa-bootstrap/src/client_binder.rs | 6 +- 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 59b81500855..5fbde0dab49 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -1,5 +1,5 @@ use humantime::format_duration; -use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration}; +use std::{collections::HashSet, io::ErrorKind, net::SocketAddr, sync::Arc, time::Duration}; use massa_final_state::FinalState; use massa_logging::massa_trace; @@ -33,31 +33,30 @@ async fn stream_final_state_and_consensus( global_bootstrap_state: &mut GlobalBootstrapState, ) -> Result<(), BootstrapError> { if let BootstrapClientMessage::AskBootstrapPart { .. } = &next_bootstrap_message { - match tokio::time::timeout( - cfg.write_timeout.into(), - client.send(next_bootstrap_message), - ) - .await + match // tokio::time::timeout( + // cfg.write_timeout.into(), + client.send(next_bootstrap_message) { - Err(_) => Err(std::io::Error::new( + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap ask ledger part send timed out", ) .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + Err(e) => Err(e), + Ok(_) => Ok(()), }?; loop { - let msg = match tokio::time::timeout(cfg.read_timeout.into(), client.next()).await { - Err(_) => { + let msg = match // tokio::time::timeout(cfg.read_timeout.into(), + client.next() { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "final state bootstrap read timed out", ) .into()); } - Ok(Err(e)) => return Err(e), - Ok(Ok(msg)) => msg, + Err(e) => return Err(e), + Ok(msg) => msg, }; match msg { BootstrapServerMessage::BootstrapPart { @@ -205,33 +204,35 @@ async fn bootstrap_from_server( // read error (if sent by the server) // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match tokio::time::timeout(cfg.read_error_timeout.into(), client.next()).await { - Err(_) => { + match // tokio::time::timeout(cfg.read_error_timeout.into(), + client.next() { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { massa_trace!( "bootstrap.lib.bootstrap_from_server: No error sent at connection", {} ); } - Ok(Err(e)) => return Err(e), - Ok(Ok(BootstrapServerMessage::BootstrapError { error: err })) => { + Err(e) => return Err(e), + Ok(BootstrapServerMessage::BootstrapError { error: err }) => { return Err(BootstrapError::ReceivedError(err)) } - Ok(Ok(msg)) => return Err(BootstrapError::UnexpectedServerMessage(msg)), + Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(msg)), }; // handshake let send_time_uncompensated = MassaTime::now()?; // client.handshake() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match tokio::time::timeout(cfg.write_timeout.into(), client.handshake(our_version)).await { - Err(_) => { + match // tokio::time::timeout(cfg.write_timeout.into(), + client.handshake(our_version) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap handshake timed out", ) - .into()) + .into()); } - Ok(Err(e)) => return Err(e), - Ok(Ok(_)) => {} + Err(e) => return Err(e), + Ok(_) => {} } // compute ping @@ -244,19 +245,20 @@ async fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - let server_time = match tokio::time::timeout(cfg.read_timeout.into(), client.next()).await { - Err(_) => { + let server_time = match // tokio::time::timeout(cfg.read_timeout.into(), + client.next() { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap clock sync read timed out", ) .into()) } - Ok(Err(e)) => return Err(e), - Ok(Ok(BootstrapServerMessage::BootstrapTime { + Err(e) => return Err(e), + Ok(BootstrapServerMessage::BootstrapTime { server_time, version, - })) => { + }) => { if !our_version.is_compatible(&version) { return Err(BootstrapError::IncompatibleVersionError(format!( "remote is running incompatible version: {} (local node version: {})", @@ -265,10 +267,10 @@ async fn bootstrap_from_server( } server_time } - Ok(Ok(BootstrapServerMessage::BootstrapError { error })) => { + Ok(BootstrapServerMessage::BootstrapError { error }) => { return Err(BootstrapError::ReceivedError(error)) } - Ok(Ok(msg)) => return Err(BootstrapError::UnexpectedServerMessage(msg)), + Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(msg)), }; // get the time of reception @@ -331,15 +333,16 @@ async fn bootstrap_from_server( *next_bootstrap_message = BootstrapClientMessage::BootstrapSuccess; } BootstrapClientMessage::BootstrapSuccess => { - match tokio::time::timeout(write_timeout, client.send(next_bootstrap_message)).await + match // tokio::time::timeout(write_timeout, + client.send(next_bootstrap_message) { - Err(_) => Err(std::io::Error::new( + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "send bootstrap success timed out", ) .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + Err(e) => Err(e), + Ok(_)=> Ok(()), }?; break; } @@ -359,15 +362,17 @@ async fn send_client_message( read_timeout: Duration, error: &str, ) -> Result { - match tokio::time::timeout(write_timeout, client.send(message_to_send)).await { - Err(_) => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + match // tokio::time::timeout(write_timeout, + client.send(message_to_send) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), + Err(e) => Err(e), + Ok(_) => Ok(()), }?; - match tokio::time::timeout(read_timeout, client.next()).await { - Err(_) => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), - Ok(Err(e)) => Err(e), - Ok(Ok(msg)) => Ok(msg), + match // tokio::time::timeout(read_timeout, + client.next() { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), + Err(e) => Err(e), + Ok(msg) => Ok(msg), } } @@ -466,7 +471,8 @@ pub async fn get_state( Err(e) => { warn!("Error while bootstrapping: {}", e); // We allow unused result because we don't care if an error is thrown when sending the error message to the server we will close the socket anyway. - let _ = tokio::time::timeout(bootstrap_config.write_error_timeout.into(), client.send(&BootstrapClientMessage::BootstrapError { error: e.to_string() })).await; + let _ = // tokio::time::timeout(bootstrap_config.write_error_timeout.into(), + client.send(&BootstrapClientMessage::BootstrapError { error: e.to_string() }); } Ok(()) => { return Ok(global_bootstrap_state) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 725f09a91a5..5e4656ac375 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -46,7 +46,7 @@ impl BootstrapClientBinder { /// Performs a handshake. Should be called after connection /// NOT cancel-safe - pub async fn handshake(&mut self, version: Version) -> Result<(), BootstrapError> { + pub fn handshake(&mut self, version: Version) -> Result<(), BootstrapError> { // send version and randomn bytes let msg_hash = { let mut version_ser = Vec::new(); @@ -66,7 +66,7 @@ impl BootstrapClientBinder { } /// Reads the next message. NOT cancel-safe - pub async fn next(&mut self) -> Result { + pub fn next(&mut self) -> Result { // read signature let sig = { let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; @@ -113,7 +113,7 @@ impl BootstrapClientBinder { #[allow(dead_code)] /// Send a message to the bootstrap server - pub async fn send(&mut self, msg: &BootstrapClientMessage) -> Result<(), BootstrapError> { + pub fn send(&mut self, msg: &BootstrapClientMessage) -> Result<(), BootstrapError> { let mut msg_bytes = Vec::new(); let message_serializer = BootstrapClientMessageSerializer::new(); message_serializer.serialize(msg, &mut msg_bytes)?; From 210e52e368844ba0859b46d7f0224a035fed021f Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 18:20:58 +0200 Subject: [PATCH 43/83] Propogate errors instead of unwrapping --- massa-bootstrap/src/client.rs | 7 +++++-- massa-bootstrap/src/server.rs | 4 ++-- massa-network-exports/src/establisher.rs | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index b63c0e108de..3f5d9291d8f 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -378,10 +378,13 @@ fn connect_to_server( pub_key: &PublicKey, ) -> Result, Box> { let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; - socket.set_nonblocking(true).unwrap(); + socket + .set_nonblocking(true) + .map_err(|e| Box::new(BootstrapError::IoError(e)))?; Ok(BootstrapClientBinder::new( // this from_std will panic if this method doesn't exist within an async runtime... - tokio::net::TcpStream::from_std(socket).unwrap(), + tokio::net::TcpStream::from_std(socket) + .map_err(|e| Box::new(BootstrapError::IoError(e)))?, *pub_key, bootstrap_config.into(), )) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 2e0eedd5e04..6cf8f9d4011 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -253,8 +253,8 @@ impl BootstrapServer<'_, D> { ) -> Result>, Box> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; - msg.set_nonblocking(true).unwrap(); - let msg = TcpStream::from_std(msg).unwrap(); + msg.set_nonblocking(true).map_err(BootstrapError::IoError)?; + let msg = TcpStream::from_std(msg).map_err(BootstrapError::IoError)?; match listener_tx.send((msg, addr)) { Ok(_) => continue, Err(SendError((dplx, remote_addr))) => { diff --git a/massa-network-exports/src/establisher.rs b/massa-network-exports/src/establisher.rs index 783bad855f4..eb67bf7528d 100644 --- a/massa-network-exports/src/establisher.rs +++ b/massa-network-exports/src/establisher.rs @@ -103,7 +103,7 @@ mod types { socket.listen(1024)?; let socket: std::net::TcpListener = socket.into(); - socket.set_nonblocking(true).unwrap(); + socket.set_nonblocking(true)?; Ok(DefaultListener(TcpListener::from_std(socket)?)) } From 840cfa5d0346c2ca083ea5556e436e81c375ec9f Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 18:33:51 +0200 Subject: [PATCH 44/83] Don't panic if you don't have a bootstrap listen address --- massa-node/src/main.rs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 10f7f81ac3c..560354638e7 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -506,18 +506,20 @@ async fn launch( let factory_manager = start_factory(factory_config, node_wallet.clone(), factory_channels); // launch bootstrap server - let addr = &bootstrap_config.listen_addr.unwrap(); // TODO: use std::net::TcpStream - let bootstrap_manager = start_bootstrap_server::( - consensus_controller.clone(), - network_command_sender.clone(), - final_state.clone(), - bootstrap_config, - DefaultListener::new(addr).unwrap(), - private_key, - *VERSION, - ) - .unwrap(); + let bootstrap_manager = match bootstrap_config.listen_addr { + Some(addr) => start_bootstrap_server::( + consensus_controller.clone(), + network_command_sender.clone(), + final_state.clone(), + bootstrap_config, + DefaultListener::new(&addr).unwrap(), + private_key, + *VERSION, + ) + .unwrap(), + None => None, + }; let api_config: APIConfig = APIConfig { bind_private: SETTINGS.api.bind_private, From 0721d10d0ec91df983bce12e059cfcc8d62842c9 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 20:29:06 +0200 Subject: [PATCH 45/83] Remove vestigial `DefaultEstablisher` --- massa-bootstrap/src/establisher.rs | 49 ------------------------------ massa-bootstrap/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 50 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 8f8106801bf..699aa1d7693 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -74,52 +74,3 @@ impl BSConnector for DefaultConnector { TcpStream::connect_timeout(&addr, self.0.to_duration()) } } - -/// Establishes a connection -#[derive(Debug)] -pub struct DefaultEstablisher; - -impl DefaultEstablisher { - /// Creates an Establisher. - pub fn new() -> Self { - DefaultEstablisher {} - } - /// Gets the associated listener - /// - /// # Argument - /// * `addr`: `SocketAddr` we want to bind to. - pub fn get_listener(&mut self, addr: &SocketAddr) -> io::Result { - // Create a socket2 TCP listener to manually set the IPV6_V6ONLY flag - // This is needed to get the same behavior on all OS - // However, if IPv6 is disabled system-wide, you may need to bind to an IPv4 address instead. - let domain = match addr.is_ipv4() { - true => socket2::Domain::IPV4, - _ => socket2::Domain::IPV6, - }; - - let socket = socket2::Socket::new(domain, socket2::Type::STREAM, None)?; - - if addr.is_ipv6() { - socket.set_only_v6(false)?; - } - socket.bind(&(*addr).into())?; - - // Number of connections to queue, set to the hardcoded value used by tokio - socket.listen(1024)?; - Ok(DefaultListener(socket.into())) - } - - /// Get the connector with associated timeout - /// - /// # Argument - /// * `timeout_duration`: timeout duration in milliseconds - pub fn get_connector(&mut self, timeout_duration: MassaTime) -> io::Result { - Ok(DefaultConnector(timeout_duration)) - } -} - -impl Default for DefaultEstablisher { - fn default() -> Self { - Self::new() - } -} diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index 9d0e109d498..77d78c62304 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -29,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::{DefaultEstablisher, DefaultListener}; +pub use establisher::DefaultListener; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, From f128231af79d23880a03a17e459dcac6d183449d Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 21:37:37 +0200 Subject: [PATCH 46/83] Refactor the connector --- Cargo.lock | 1 + massa-bootstrap/Cargo.toml | 2 +- massa-bootstrap/src/client.rs | 7 +++---- massa-bootstrap/src/establisher.rs | 19 +++++++++++++++---- 4 files changed, 20 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c82f51cb7a4..cd90b76b17e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4831,6 +4831,7 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", + "tracing", "windows-sys 0.45.0", ] diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 28533e6a320..38a5f31af50 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1.0" humantime = "2.1.0" thiserror = "1.0" parking_lot = { version = "0.12", features = ["deadlock_detection"] } -tokio = { version = "1.23", features = ["full"] } +tokio = { version = "1.23", features = ["full", "tracing"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index f019d3eff66..6ab2ed2fb92 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -382,10 +382,9 @@ fn connect_to_server( addr: &SocketAddr, pub_key: &PublicKey, ) -> Result, Box> { - let socket = connector.connect(*addr).map_err(|e| Box::new(e.into()))?; - socket - .set_nonblocking(true) - .map_err(|e| Box::new(BootstrapError::IoError(e)))?; + let socket = connector + .connect_timeout(*addr, Some(bootstrap_config.connect_timeout)) + .map_err(|e| Box::new(e.into()))?; Ok(BootstrapClientBinder::new( // this from_std will panic if this method doesn't exist within an async runtime... socket, diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index fdc1634f39d..b9262812bd9 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -23,7 +23,11 @@ pub trait BSListener { /// Specifies a common interface that can be used by standard, or mockers #[cfg_attr(test, mockall::automock)] pub trait BSConnector { - fn connect(&mut self, addr: SocketAddr) -> io::Result; + fn connect_timeout( + &self, + addr: SocketAddr, + duration: Option, + ) -> io::Result; } /// The listener we are using @@ -65,14 +69,21 @@ impl BSListener for DefaultListener { } /// Initiates a connection with given timeout in milliseconds #[derive(Debug)] -pub struct DefaultConnector(MassaTime); +pub struct DefaultConnector; impl BSConnector for DefaultConnector { /// Tries to connect to address /// /// # Argument /// * `addr`: `SocketAddr` we are trying to connect to. - fn connect(&mut self, addr: SocketAddr) -> io::Result { - TcpStream::connect_timeout(&addr, self.0.to_duration()) + fn connect_timeout( + &self, + addr: SocketAddr, + duration: Option, + ) -> io::Result { + let Some(duration) = duration else { + return TcpStream::connect(&addr); + }; + TcpStream::connect_timeout(&addr, duration.to_duration()) } } From 36d610188e2872587fa44791f620e9354fa39907 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 29 Mar 2023 21:40:20 +0200 Subject: [PATCH 47/83] General Cleanup --- massa-bootstrap/src/tests/binders.rs | 21 +++---- massa-bootstrap/src/tests/scenarios.rs | 77 ++++++++++++++------------ 2 files changed, 52 insertions(+), 46 deletions(-) diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 8f1adaa880a..498e045e386 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -127,8 +127,8 @@ async fn test_binders() { let version: Version = Version::from_str("TEST.1.10").unwrap(); - client.handshake(version).await.unwrap(); - let message = client.next().await.unwrap(); + client.handshake(version).unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -140,7 +140,6 @@ async fn test_binders() { .send(&BootstrapClientMessage::BootstrapError { error: "test error".to_string(), }) - .await .unwrap(); // Test message 3 @@ -149,7 +148,7 @@ async fn test_binders() { bootstrap_config.bootstrap_list[0].0.ip(), bootstrap_config.bootstrap_list[0].0.ip(), ]; - let message = client.next().await.unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -221,8 +220,8 @@ async fn test_binders_double_send_server_works() { let version: Version = Version::from_str("TEST.1.10").unwrap(); - client.handshake(version).await.unwrap(); - let message = client.next().await.unwrap(); + client.handshake(version).unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -236,7 +235,7 @@ async fn test_binders_double_send_server_works() { bootstrap_config.bootstrap_list[0].0.ip(), bootstrap_config.bootstrap_list[0].0.ip(), ]; - let message = client.next().await.unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -312,8 +311,8 @@ async fn test_binders_try_double_send_client_works() { let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; let version: Version = Version::from_str("TEST.1.10").unwrap(); - client.handshake(version).await.unwrap(); - let message = client.next().await.unwrap(); + client.handshake(version).unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -325,7 +324,6 @@ async fn test_binders_try_double_send_client_works() { .send(&BootstrapClientMessage::BootstrapError { error: "test error".to_string(), }) - .await .unwrap(); // Test message 3 @@ -333,11 +331,10 @@ async fn test_binders_try_double_send_client_works() { .send(&BootstrapClientMessage::BootstrapError { error: "test error".to_string(), }) - .await .unwrap(); let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let message = client.next().await.unwrap(); + let message = client.next().unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 587d51775d1..11482f71dc4 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -195,41 +195,47 @@ async fn test_bootstrap_server() { .unwrap(); // launch the get_state process - let get_state_h = tokio::spawn(async move { - get_state( - bootstrap_config, - final_state_client_clone, - mock_remote_connector, - Version::from_str("TEST.1.10").unwrap(), - MassaTime::now().unwrap().saturating_sub(1000.into()), - None, - ) - .await - .unwrap() - }); + let get_state_h = tokio::task::Builder::new() + .name("state-getter") + .spawn(async move { + get_state( + bootstrap_config, + final_state_client_clone, + mock_remote_connector, + Version::from_str("TEST.1.10").unwrap(), + MassaTime::now().unwrap().saturating_sub(1000.into()), + None, + ) + .await + .unwrap() + }) + .unwrap(); // launch the modifier thread let list_changes: Arc>> = Arc::new(RwLock::new(Vec::new())); let list_changes_clone = list_changes.clone(); - std::thread::spawn(move || { - for _ in 0..10 { - std::thread::sleep(Duration::from_millis(500)); - let mut final_write = final_state_server_clone.write(); - let next = final_write.slot.get_next_slot(thread_count).unwrap(); - final_write.slot = next; - let changes = StateChanges { - pos_changes: get_random_pos_changes(10), - ledger_changes: get_random_ledger_changes(10), - async_pool_changes: get_random_async_pool_changes(10), - executed_ops_changes: get_random_executed_ops_changes(10), - }; - final_write - .changes_history - .push_back((next, changes.clone())); - let mut list_changes_write = list_changes_clone.write(); - list_changes_write.push((next, changes)); - } - }); + std::thread::Builder::new() + .name("modifier thread".to_string()) + .spawn(move || { + for _ in 0..10 { + std::thread::sleep(Duration::from_millis(500)); + let mut final_write = final_state_server_clone.write(); + let next = final_write.slot.get_next_slot(thread_count).unwrap(); + final_write.slot = next; + let changes = StateChanges { + pos_changes: get_random_pos_changes(10), + ledger_changes: get_random_ledger_changes(10), + async_pool_changes: get_random_async_pool_changes(10), + executed_ops_changes: get_random_executed_ops_changes(10), + }; + final_write + .changes_history + .push_back((next, changes.clone())); + let mut list_changes_write = list_changes_clone.write(); + list_changes_write.push((next, changes)); + } + }) + .unwrap(); // wait for get_state let bootstrap_res = get_state_h @@ -300,7 +306,10 @@ fn conn_establishment_mocks() -> (MockBSListener, MockBSConnector) { // Due to the limitations of the mocking system, the listener must loop-accept in a dedicated // thread. We use a channel to make the connection available in the mocked `accept` method let (conn_tx, conn_rx) = std::sync::mpsc::sync_channel(100); - let conn = std::thread::spawn(|| std::net::TcpStream::connect("127.0.0.1:8069").unwrap()); + let conn = std::thread::Builder::new() + .name("mock conn connect".to_string()) + .spawn(|| std::net::TcpStream::connect("127.0.0.1:8069").unwrap()) + .unwrap(); std::thread::Builder::new() .name("mock-listen-loop".to_string()) .spawn(move || loop { @@ -319,8 +328,8 @@ fn conn_establishment_mocks() -> (MockBSListener, MockBSConnector) { let mut mock_remote_connector = MockBSConnector::new(); mock_remote_connector - .expect_connect() + .expect_connect_timeout() .times(1) - .return_once(move |_| Ok(conn.join().unwrap())); + .return_once(move |_, _| Ok(conn.join().unwrap())); (mock_bs_listener, mock_remote_connector) } From 7cc9e459115b590e3c74ddad094e10d9390fdba7 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 13:13:41 +0200 Subject: [PATCH 48/83] De-async more methods and start working on `next` blockage --- Cargo.lock | 1 - massa-bootstrap/Cargo.toml | 2 +- massa-bootstrap/src/client.rs | 44 ++++++++++++++------------ massa-bootstrap/src/client_binder.rs | 15 ++++++--- massa-bootstrap/src/establisher.rs | 19 ++++++++++- massa-bootstrap/src/tests/binders.rs | 12 +++---- massa-bootstrap/src/tests/scenarios.rs | 9 +++--- 7 files changed, 63 insertions(+), 39 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cd90b76b17e..c82f51cb7a4 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4831,7 +4831,6 @@ dependencies = [ "signal-hook-registry", "socket2", "tokio-macros", - "tracing", "windows-sys 0.45.0", ] diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 38a5f31af50..28533e6a320 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -16,7 +16,7 @@ serde_json = "1.0" humantime = "2.1.0" thiserror = "1.0" parking_lot = { version = "0.12", features = ["deadlock_detection"] } -tokio = { version = "1.23", features = ["full", "tracing"] } +tokio = { version = "1.23", features = ["full"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 6ab2ed2fb92..388d9a5030a 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -26,7 +26,7 @@ use crate::{ /// This function will send the starting point to receive a stream of the ledger and will receive and process each part until receive a `BootstrapServerMessage::FinalStateFinished` message from the server. /// `next_bootstrap_message` passed as parameter must be `BootstrapClientMessage::AskFinalStatePart` enum variant. /// `next_bootstrap_message` will be updated after receiving each part so that in case of connection lost we can restart from the last message we processed. -async fn stream_final_state_and_consensus( +fn stream_final_state_and_consensus( cfg: &BootstrapConfig, client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, @@ -47,7 +47,7 @@ async fn stream_final_state_and_consensus( }?; loop { let msg = match // tokio::time::timeout(cfg.read_timeout.into(), - client.next() { + client.next_timeout(Some(cfg.read_timeout.to_duration())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -193,7 +193,7 @@ async fn stream_final_state_and_consensus( /// Gets the state from a bootstrap server (internal private function) /// needs to be CANCELLABLE -async fn bootstrap_from_server( +fn bootstrap_from_server( cfg: &BootstrapConfig, client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, @@ -201,22 +201,24 @@ async fn bootstrap_from_server( our_version: Version, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.bootstrap_from_server", {}); + dbg!("getting next"); // read error (if sent by the server) // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match // tokio::time::timeout(cfg.read_error_timeout.into(), - client.next() { + match // tokio::time::timeout(, + client.next_timeout(Some(cfg.read_error_timeout.to_duration())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + dbg!("no next"); massa_trace!( "bootstrap.lib.bootstrap_from_server: No error sent at connection", {} ); } - Err(e) => return Err(e), + Err(e) => return Err(dbg!(e)), Ok(BootstrapServerMessage::BootstrapError { error: err }) => { - return Err(BootstrapError::ReceivedError(err)) + return Err(BootstrapError::ReceivedError(dbg!(err))) } - Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(msg)), + Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(dbg!(msg))), }; // handshake @@ -245,8 +247,8 @@ async fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - let server_time = match // tokio::time::timeout(cfg.read_timeout.into(), - client.next() { + let server_time = match // tokio::time::timeout(, + client.next_timeout(Some(cfg.read_timeout.into())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -310,8 +312,7 @@ async fn bootstrap_from_server( client, next_bootstrap_message, global_bootstrap_state, - ) - .await?; + )?; } BootstrapClientMessage::AskBootstrapPeers => { let peers = match send_client_message( @@ -320,9 +321,7 @@ async fn bootstrap_from_server( write_timeout, cfg.read_timeout.into(), "ask bootstrap peers timed out", - ) - .await? - { + )? { BootstrapServerMessage::BootstrapPeers { peers } => peers, BootstrapServerMessage::BootstrapError { error } => { return Err(BootstrapError::ReceivedError(error)) @@ -355,7 +354,7 @@ async fn bootstrap_from_server( Ok(()) } -async fn send_client_message( +fn send_client_message( message_to_send: &BootstrapClientMessage, client: &mut BootstrapClientBinder, write_timeout: Duration, @@ -368,8 +367,8 @@ async fn send_client_message( Err(e) => Err(e), Ok(_) => Ok(()), }?; - match // tokio::time::timeout(read_timeout, - client.next() { + match // tokio::time::timeout(, + client.next_timeout(Some(read_timeout)) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), Err(e) => Err(e), Ok(msg) => Ok(msg), @@ -385,6 +384,9 @@ fn connect_to_server( let socket = connector .connect_timeout(*addr, Some(bootstrap_config.connect_timeout)) .map_err(|e| Box::new(e.into()))?; + socket + .set_nonblocking(false) + .map_err(|e| Box::new(e.into()))?; Ok(BootstrapClientBinder::new( // this from_std will panic if this method doesn't exist within an async runtime... socket, @@ -421,7 +423,7 @@ fn filter_bootstrap_list( /// Gets the state from a bootstrap server /// needs to be CANCELLABLE -pub async fn get_state( +pub fn get_state( bootstrap_config: &BootstrapConfig, final_state: Arc>, mut connector: impl BSConnector, @@ -466,7 +468,7 @@ pub async fn get_state( ) { Ok(mut client) => { match bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version) - .await // cancellable + // cancellable { Err(BootstrapError::ReceivedError(error)) => warn!("Error received from bootstrap server: {}", error), Err(e) => { @@ -486,7 +488,7 @@ pub async fn get_state( }; info!("Bootstrap from server {} failed. Your node will try to bootstrap from another server in {}.", addr, format_duration(bootstrap_config.retry_delay.to_duration()).to_string()); - sleep(bootstrap_config.retry_delay.into()).await; + std::thread::sleep(bootstrap_config.retry_delay.into()); } } } diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 5e4656ac375..0bff9010105 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -1,5 +1,7 @@ // Copyright (c) 2022 MASSA LABS +use std::time::Duration; + use crate::error::BootstrapError; use crate::establisher::Duplex; use crate::messages::{ @@ -66,18 +68,23 @@ impl BootstrapClientBinder { } /// Reads the next message. NOT cancel-safe - pub fn next(&mut self) -> Result { + pub fn next_timeout( + &mut self, + duration: Option, + ) -> Result { + self.duplex.set_read_timeout(duration).unwrap(); + self.duplex.set_nonblocking(false).unwrap(); // read signature let sig = { let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; - self.duplex.read_exact(&mut sig_bytes)?; - Signature::from_bytes(&sig_bytes)? + self.duplex.read_exact(&mut sig_bytes).unwrap(); + Signature::from_bytes(&sig_bytes).unwrap() }; // read message length let msg_len = { let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..])?; + self.duplex.read_exact(&mut msg_len_bytes[..]).unwrap(); u32::from_be_bytes_min(&msg_len_bytes, self.cfg.max_bootstrap_message_size)?.0 }; diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index b9262812bd9..5434e73f80e 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -3,6 +3,7 @@ use massa_time::MassaTime; use std::{ io, net::{SocketAddr, TcpListener, TcpStream}, + time::Duration, }; /// duplex connection @@ -10,9 +11,24 @@ pub trait Duplex: // static because need to send between threads :( 'static + Send + io::Read + io::Write { + fn set_read_timeout(&mut self, duration: Option) -> io::Result<()>; + fn set_write_timeout(&mut self, duration: Option) -> io::Result<()>; + fn set_nonblocking(&mut self, setting: bool) -> io::Result<()>; } -impl Duplex for std::net::TcpStream {} +impl Duplex for std::net::TcpStream { + fn set_read_timeout(&mut self, duration: Option) -> io::Result<()> { + Self::set_read_timeout(&self, duration) + } + + fn set_write_timeout(&mut self, duration: Option) -> io::Result<()> { + Self::set_write_timeout(&self, duration) + } + + fn set_nonblocking(&mut self, setting: bool) -> io::Result<()> { + Self::set_nonblocking(&self, setting) + } +} /// Specifies a common interface that can be used by standard, or mockers #[cfg_attr(test, mockall::automock)] @@ -53,6 +69,7 @@ impl DefaultListener { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; + socket.set_nonblocking(false); Ok(DefaultListener(socket.into())) } } diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 498e045e386..48f339b27a4 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -128,7 +128,7 @@ async fn test_binders() { let version: Version = Version::from_str("TEST.1.10").unwrap(); client.handshake(version).unwrap(); - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -148,7 +148,7 @@ async fn test_binders() { bootstrap_config.bootstrap_list[0].0.ip(), bootstrap_config.bootstrap_list[0].0.ip(), ]; - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -221,7 +221,7 @@ async fn test_binders_double_send_server_works() { let version: Version = Version::from_str("TEST.1.10").unwrap(); client.handshake(version).unwrap(); - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -235,7 +235,7 @@ async fn test_binders_double_send_server_works() { bootstrap_config.bootstrap_list[0].0.ip(), bootstrap_config.bootstrap_list[0].0.ip(), ]; - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -312,7 +312,7 @@ async fn test_binders_try_double_send_client_works() { let version: Version = Version::from_str("TEST.1.10").unwrap(); client.handshake(version).unwrap(); - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); @@ -334,7 +334,7 @@ async fn test_binders_try_double_send_client_works() { .unwrap(); let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let message = client.next().unwrap(); + let message = client.next_timeout(None).unwrap(); match message { BootstrapServerMessage::BootstrapPeers { peers } => { assert_eq!(vector_peers, peers.0); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 11482f71dc4..cf11a679076 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -195,9 +195,9 @@ async fn test_bootstrap_server() { .unwrap(); // launch the get_state process - let get_state_h = tokio::task::Builder::new() - .name("state-getter") - .spawn(async move { + let get_state_h = std::thread::Builder::new() + .name("state-getter".to_string()) + .spawn(move || { get_state( bootstrap_config, final_state_client_clone, @@ -206,7 +206,6 @@ async fn test_bootstrap_server() { MassaTime::now().unwrap().saturating_sub(1000.into()), None, ) - .await .unwrap() }) .unwrap(); @@ -239,7 +238,7 @@ async fn test_bootstrap_server() { // wait for get_state let bootstrap_res = get_state_h - .await + .join() .expect("error while waiting for get_state to finish"); // apply the changes to the server state before matching with the client From aa38179ce4b7925ffe36bc73645afa5e4c3746c3 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 13:17:47 +0200 Subject: [PATCH 49/83] Fix clippy and compilation warn/error --- massa-bootstrap/src/establisher.rs | 2 +- massa-bootstrap/src/lib.rs | 2 +- massa-execution-worker/src/execution.rs | 2 +- massa-node/src/main.rs | 5 +++-- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 699aa1d7693..640a558c51d 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -63,7 +63,7 @@ impl BSListener for DefaultListener { } /// Initiates a connection with given timeout in milliseconds #[derive(Debug)] -pub struct DefaultConnector(MassaTime); +pub struct DefaultConnector(pub MassaTime); impl BSConnector for DefaultConnector { /// Tries to connect to address diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index 77d78c62304..f4df5f91bad 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -29,7 +29,7 @@ mod server_binder; mod settings; mod tools; pub use client::get_state; -pub use establisher::DefaultListener; +pub use establisher::{DefaultConnector, DefaultListener}; pub use messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, BootstrapServerMessageSerializer, diff --git a/massa-execution-worker/src/execution.rs b/massa-execution-worker/src/execution.rs index 99b6b0adf19..f1a2d2650ad 100644 --- a/massa-execution-worker/src/execution.rs +++ b/massa-execution-worker/src/execution.rs @@ -39,7 +39,7 @@ use massa_storage::Storage; use parking_lot::{Mutex, RwLock}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::Arc; -use tracing::{debug, error, info, warn}; +use tracing::{debug, info, warn}; /// Used to acquire a lock on the execution context macro_rules! context_guard { diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index e474f162135..d3ae6c3c19e 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -12,7 +12,8 @@ use massa_api::{ApiServer, ApiV2, Private, Public, RpcServer, StopHandle, API}; use massa_api_exports::config::APIConfig; use massa_async_pool::AsyncPoolConfig; use massa_bootstrap::{ - get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager, DefaultListener, + get_state, start_bootstrap_server, BootstrapConfig, BootstrapManager, DefaultConnector, + DefaultListener, }; use massa_consensus_exports::events::ConsensusEvent; use massa_consensus_exports::{ConsensusChannels, ConsensusConfig, ConsensusManager}; @@ -242,7 +243,7 @@ async fn launch( res = get_state( &bootstrap_config, final_state.clone(), - massa_bootstrap::DefaultEstablisher::default().get_connector(bootstrap_config.connect_timeout).unwrap(), + DefaultConnector(bootstrap_config.connect_timeout), *VERSION, *GENESIS_TIMESTAMP, *END_TIMESTAMP, From 757ac3c2237ab30af0ee718a2a9ee638f6499dd7 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 13:50:02 +0200 Subject: [PATCH 50/83] Make the boostrap test sync --- massa-bootstrap/src/client.rs | 1 - massa-bootstrap/src/server.rs | 2 +- massa-bootstrap/src/tests/scenarios.rs | 7 ++----- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 388d9a5030a..e12172fd97b 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -11,7 +11,6 @@ use rand::{ prelude::{SliceRandom, StdRng}, SeedableRng, }; -use tokio::time::sleep; use tracing::{debug, info, warn}; use crate::{ diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index ef27beca2ed..cb7c5761dc6 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -78,7 +78,7 @@ pub struct BootstrapManager { impl BootstrapManager { /// stop the bootstrap server - pub async fn stop(self) -> Result<(), Box> { + pub fn stop(self) -> Result<(), Box> { massa_trace!("bootstrap.lib.stop", {}); if self.listen_stopper_tx.send(()).is_err() { warn!("bootstrap server already dropped"); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index cf11a679076..a289b07b9cf 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -41,7 +41,6 @@ use massa_pos_worker::start_selector_worker; use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; -use serial_test::serial; use std::{net::TcpStream, path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use tempfile::TempDir; @@ -52,9 +51,8 @@ lazy_static::lazy_static! { }; } -#[tokio::test] -#[serial] -async fn test_bootstrap_server() { +#[test] +fn test_bootstrap_server() { let thread_count = 2; let periods_per_cycle = 2; let (bootstrap_config, keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; @@ -289,7 +287,6 @@ async fn test_bootstrap_server() { // stop bootstrap server bootstrap_manager .stop() - .await .expect("could not stop bootstrap server"); // stop selector controllers From 6282b684809bde663f2fbde89d7b5fcb3cd5450f Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 16:53:35 +0200 Subject: [PATCH 51/83] Integrate timeouts into sending and receiving messages --- massa-bootstrap/src/client.rs | 91 ++++++++++++++++------------ massa-bootstrap/src/client_binder.rs | 14 +++-- massa-bootstrap/src/server.rs | 2 - massa-bootstrap/src/tests/binders.rs | 27 ++++++--- 4 files changed, 78 insertions(+), 56 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index e12172fd97b..0c5f0e184e9 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -32,22 +32,24 @@ fn stream_final_state_and_consensus( global_bootstrap_state: &mut GlobalBootstrapState, ) -> Result<(), BootstrapError> { if let BootstrapClientMessage::AskBootstrapPart { .. } = &next_bootstrap_message { - match // tokio::time::timeout( - // cfg.write_timeout.into(), - client.send(next_bootstrap_message) - { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), + match client.send_timeout( + next_bootstrap_message, + Some(cfg.write_timeout.to_duration()), + ) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()) + } Err(e) => Err(e), Ok(_) => Ok(()), }?; loop { - let msg = match // tokio::time::timeout(cfg.read_timeout.into(), - client.next_timeout(Some(cfg.read_timeout.to_duration())) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + dbg!(); + let msg = match client.next_timeout(Some(cfg.read_timeout.to_duration())) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "final state bootstrap read timed out", @@ -200,14 +202,14 @@ fn bootstrap_from_server( our_version: Version, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.bootstrap_from_server", {}); - dbg!("getting next"); // read error (if sent by the server) // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match // tokio::time::timeout(, - client.next_timeout(Some(cfg.read_error_timeout.to_duration())) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - dbg!("no next"); + match dbg!(client.next_timeout(Some(cfg.read_error_timeout.to_duration()))) { + Err(BootstrapError::IoError(e)) + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => + { + dbg!("no next"); massa_trace!( "bootstrap.lib.bootstrap_from_server: No error sent at connection", {} @@ -220,11 +222,12 @@ fn bootstrap_from_server( Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(dbg!(msg))), }; + dbg!("doing handshake"); // handshake let send_time_uncompensated = MassaTime::now()?; // client.handshake() is not cancel-safe but we drop the whole client object if cancelled => it's OK match // tokio::time::timeout(cfg.write_timeout.into(), - client.handshake(our_version) { + dbg!(client.handshake(our_version)) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -246,16 +249,18 @@ fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - let server_time = match // tokio::time::timeout(, - client.next_timeout(Some(cfg.read_timeout.into())) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + dbg!(); + let server_time = match dbg!(client.next_timeout(Some(cfg.read_timeout.into()))) { + Err(BootstrapError::IoError(e)) + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => + { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap clock sync read timed out", ) .into()) } - Err(e) => return Err(e), + Err(e) => return Err(dbg!(e)), Ok(BootstrapServerMessage::BootstrapTime { server_time, version, @@ -285,6 +290,7 @@ fn bootstrap_from_server( )); } + dbg!(); // compute client / server clock delta // div 2 is an approximation of the time it took the message to do server -> client // the complete ping value being client -> server -> client @@ -301,9 +307,11 @@ fn bootstrap_from_server( return Err(BootstrapError::ClockError(message)); } + dbg!(); let write_timeout: std::time::Duration = cfg.write_timeout.into(); // Loop to ask data to the server depending on the last message we sent loop { + dbg!(); match next_bootstrap_message { BootstrapClientMessage::AskBootstrapPart { .. } => { stream_final_state_and_consensus( @@ -331,16 +339,16 @@ fn bootstrap_from_server( *next_bootstrap_message = BootstrapClientMessage::BootstrapSuccess; } BootstrapClientMessage::BootstrapSuccess => { - match // tokio::time::timeout(write_timeout, - client.send(next_bootstrap_message) - { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "send bootstrap success timed out", - ) - .into()), + match client.send_timeout(next_bootstrap_message, Some(write_timeout)) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "send bootstrap success timed out", + ) + .into()) + } Err(e) => Err(e), - Ok(_)=> Ok(()), + Ok(_) => Ok(()), }?; break; } @@ -360,15 +368,18 @@ fn send_client_message( read_timeout: Duration, error: &str, ) -> Result { - match // tokio::time::timeout(write_timeout, - client.send(message_to_send) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), + match client.send_timeout(message_to_send, Some(write_timeout)) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) + } Err(e) => Err(e), Ok(_) => Ok(()), }?; - match // tokio::time::timeout(, - client.next_timeout(Some(read_timeout)) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()), + dbg!(); + match client.next_timeout(Some(read_timeout)) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) + } Err(e) => Err(e), Ok(msg) => Ok(msg), } @@ -466,15 +477,15 @@ pub fn get_state( &node_id.get_public_key(), ) { Ok(mut client) => { - match bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version) + dbg!("got the con, bootstrapping"); + match dbg!(bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version)) // cancellable { Err(BootstrapError::ReceivedError(error)) => warn!("Error received from bootstrap server: {}", error), Err(e) => { warn!("Error while bootstrapping: {}", e); // We allow unused result because we don't care if an error is thrown when sending the error message to the server we will close the socket anyway. - let _ = // tokio::time::timeout(bootstrap_config.write_error_timeout.into(), - client.send(&BootstrapClientMessage::BootstrapError { error: e.to_string() }); + let _ = client.send_timeout(&BootstrapClientMessage::BootstrapError { error: e.to_string() }, Some(bootstrap_config.write_error_timeout.into())); } Ok(()) => { return Ok(global_bootstrap_state) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 0bff9010105..90e323f22cd 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -73,18 +73,17 @@ impl BootstrapClientBinder { duration: Option, ) -> Result { self.duplex.set_read_timeout(duration).unwrap(); - self.duplex.set_nonblocking(false).unwrap(); // read signature let sig = { let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; - self.duplex.read_exact(&mut sig_bytes).unwrap(); - Signature::from_bytes(&sig_bytes).unwrap() + self.duplex.read_exact(&mut sig_bytes)?; + Signature::from_bytes(&sig_bytes)? }; // read message length let msg_len = { let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..]).unwrap(); + self.duplex.read_exact(&mut msg_len_bytes[..])?; u32::from_be_bytes_min(&msg_len_bytes, self.cfg.max_bootstrap_message_size)?.0 }; @@ -120,7 +119,11 @@ impl BootstrapClientBinder { #[allow(dead_code)] /// Send a message to the bootstrap server - pub fn send(&mut self, msg: &BootstrapClientMessage) -> Result<(), BootstrapError> { + pub fn send_timeout( + &mut self, + msg: &BootstrapClientMessage, + duration: Option, + ) -> Result<(), BootstrapError> { let mut msg_bytes = Vec::new(); let message_serializer = BootstrapClientMessageSerializer::new(); message_serializer.serialize(msg, &mut msg_bytes)?; @@ -150,6 +153,7 @@ impl BootstrapClientBinder { // send message length { + self.duplex.set_write_timeout(duration); let msg_len_bytes = msg_len.to_be_bytes_min(self.cfg.max_bootstrap_message_size)?; self.duplex.write_all(&msg_len_bytes)?; } diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index cb7c5761dc6..c5287f3b227 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -819,8 +819,6 @@ async fn manage_bootstrap( .send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { - // TODO: pass in an impl that can be mocked so that this will be hardcoded - // where the mock is defined peers: network_command_sender.get_bootstrap_peers().await?, }, ) diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 48f339b27a4..2fe982d9813 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -137,9 +137,12 @@ async fn test_binders() { } client - .send(&BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }) + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) .unwrap(); // Test message 3 @@ -321,16 +324,22 @@ async fn test_binders_try_double_send_client_works() { } client - .send(&BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }) + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) .unwrap(); // Test message 3 client - .send(&BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }) + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) .unwrap(); let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; From af20c1eb946bd006a8369c83656a4ab8404c3f70 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 16:56:11 +0200 Subject: [PATCH 52/83] Fully clear out async in the bootstrap test --- massa-bootstrap/src/server.rs | 58 +++++++++--------------- massa-bootstrap/src/tests/scenarios.rs | 62 ++++++++++++++------------ 2 files changed, 53 insertions(+), 67 deletions(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index c5287f3b227..49f64e481eb 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -118,17 +118,6 @@ pub fn start_bootstrap_server( return Err(BootstrapError::GeneralError("Fail to convert u32 to usize".to_string()).into()); }; - // We use a runtime dedicated to the bootstrap part of the system - // This should help keep it isolated from the rest of the overall binary - let Ok(bs_server_runtime) = runtime::Builder::new_multi_thread() - .enable_io() - .enable_time() - .thread_name("bootstrap-global-runtime-worker") - .thread_keep_alive(Duration::from_millis(u64::MAX)) - .build() else { - return Err(Box::new(BootstrapError::GeneralError("Failed to creato bootstrap async runtime".to_string()))); - }; - // This is the primary interface between the async-listener, and the "sync" worker let (listener_tx, listener_rx) = crossbeam::channel::bounded::>(max_bootstraps * 2); @@ -156,16 +145,11 @@ pub fn start_bootstrap_server( // the non-builder spawn doesn't return a Result, and documentation states that // it's an error at the OS level. .unwrap(); - let listen_rt_handle = bs_server_runtime.handle().clone(); let listen_handle = thread::Builder::new() .name("bs_listener".to_string()) // FIXME: The interface being used shouldn't have `: Send + 'static` as a constraint on the listener assosciated type. // GAT lifetime is likely to remedy this, however. - .spawn(move || { - let res = listen_rt_handle - .block_on(BootstrapServer::::run_listener(listener, listener_tx)); - res - }) + .spawn(move || BootstrapServer::::run_listener(listener, listener_tx)) // the non-builder spawn doesn't return a Result, and documentation states that // it's an error at the OS level. .unwrap(); @@ -184,7 +168,6 @@ pub fn start_bootstrap_server( version, ip_hist_map: HashMap::with_capacity(config.ip_list_max_size), bootstrap_config: config, - bs_server_runtime, } .run_loop(max_bootstraps) }) @@ -213,7 +196,6 @@ struct BootstrapServer<'a, D: Duplex, C: NetworkCommandSenderTrait> { bootstrap_config: BootstrapConfig, version: Version, ip_hist_map: HashMap, - bs_server_runtime: Runtime, } impl BootstrapServer<'_, D, C> { @@ -244,7 +226,7 @@ impl BootstrapServer<'_, D, C> /// Ok(Err((dplx, address))) listener accepted a connection then tried sending on a disconnected channel /// Err(..) Error accepting a connection /// TODO: Integrate the listener into the bootstrap-main-loop - async fn run_listener( + fn run_listener( mut listener: impl BSListener, listener_tx: crossbeam::channel::Sender>, ) -> Result>, Box> { @@ -294,12 +276,12 @@ impl BootstrapServer<'_, D, C> // check whether incoming peer IP is allowed. if let Err(error_msg) = self.white_black_list.is_ip_allowed(&remote_addr) { - server_binding.close_and_send_error( - self.bs_server_runtime.handle().clone(), - error_msg.to_string(), - remote_addr, - move || {}, - ); + // server_binding.close_and_send_error( + // self.bs_server_runtime.handle().clone(), + // error_msg.to_string(), + // remote_addr, + // move || {}, + // ); continue; }; @@ -344,12 +326,12 @@ impl BootstrapServer<'_, D, C> "remote_addr": remote_addr }) }; - server_binding.close_and_send_error( - self.bs_server_runtime.handle().clone(), - msg, - remote_addr, - tracer, - ); + // server_binding.close_and_send_error( + // self.bs_server_runtime.handle().clone(), + // msg, + // remote_addr, + // tracer, + // ); continue; }; // Clients Option is good, and has been updated @@ -385,12 +367,12 @@ impl BootstrapServer<'_, D, C> "active_count": Arc::strong_count(&bootstrap_sessions_counter) - 1 }); } else { - server_binding.close_and_send_error( - self.bs_server_runtime.handle().clone(), - "Bootstrap failed because the bootstrap server currently has no slots available.".to_string(), - remote_addr, - move || debug!("did not bootstrap {}: no available slots", remote_addr), - ); + // server_binding.close_and_send_error( + // self.bs_server_runtime.handle().clone(), + // "Bootstrap failed because the bootstrap server currently has no slots available.".to_string(), + // remote_addr, + // move || debug!("did not bootstrap {}: no available slots", remote_addr), + // ); } } diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index a289b07b9cf..5d3f6426d66 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -133,9 +133,10 @@ fn test_bootstrap_server() { final_state_local_config, ))); let final_state_client_clone = final_state_client.clone(); - let final_state_server_clone = final_state_server.clone(); + let final_state_server_clone1 = final_state_server.clone(); + let final_state_server_clone2 = final_state_server.clone(); - let (mock_bs_listener, mock_remote_connector) = conn_establishment_mocks(); + let (mock_bs_listener, mut mock_remote_connector) = conn_establishment_mocks(); // Setup network command mock-story: hard-code the result of getting bootstrap peers let mut mocked1 = MockNetworkCommandSender::new(); @@ -180,43 +181,33 @@ fn test_bootstrap_server() { stream_mock1 .expect_clone_box() .return_once(move || stream_mock2); - let bootstrap_manager = start_bootstrap_server::( - stream_mock1, - mocked1, - final_state_server.clone(), - bootstrap_config.clone(), - mock_bs_listener, - keypair.clone(), - Version::from_str("TEST.1.10").unwrap(), - ) - .unwrap() - .unwrap(); - // launch the get_state process - let get_state_h = std::thread::Builder::new() - .name("state-getter".to_string()) + let bootstrap_manager_thread = std::thread::Builder::new() + .name("bootstrap_thread".to_string()) .spawn(move || { - get_state( - bootstrap_config, - final_state_client_clone, - mock_remote_connector, + start_bootstrap_server::( + stream_mock1, + mocked1, + final_state_server_clone1, + bootstrap_config.clone(), + mock_bs_listener, + keypair.clone(), Version::from_str("TEST.1.10").unwrap(), - MassaTime::now().unwrap().saturating_sub(1000.into()), - None, ) .unwrap() + .unwrap() }) .unwrap(); // launch the modifier thread let list_changes: Arc>> = Arc::new(RwLock::new(Vec::new())); let list_changes_clone = list_changes.clone(); - std::thread::Builder::new() + let mod_thread = std::thread::Builder::new() .name("modifier thread".to_string()) .spawn(move || { for _ in 0..10 { std::thread::sleep(Duration::from_millis(500)); - let mut final_write = final_state_server_clone.write(); + let mut final_write = final_state_server_clone2.write(); let next = final_write.slot.get_next_slot(thread_count).unwrap(); final_write.slot = next; let changes = StateChanges { @@ -234,10 +225,20 @@ fn test_bootstrap_server() { }) .unwrap(); - // wait for get_state - let bootstrap_res = get_state_h - .join() - .expect("error while waiting for get_state to finish"); + // mock_remote_connector + // .expect_connect_timeout() + // .times(1) + // .returning(|_, _| todo!()); + // launch the get_state process + let bootstrap_res = get_state( + bootstrap_config, + final_state_client_clone, + mock_remote_connector, + Version::from_str("TEST.1.10").unwrap(), + MassaTime::now().unwrap().saturating_sub(1000.into()), + None, + ) + .unwrap(); // apply the changes to the server state before matching with the client { @@ -284,8 +285,11 @@ fn test_bootstrap_server() { // check graphs assert_eq_bootstrap_graph(&sent_graph, &bootstrap_res.graph.unwrap()); + dbg!("now stop! (hammer time)"); // stop bootstrap server - bootstrap_manager + bootstrap_manager_thread + .join() + .unwrap() .stop() .expect("could not stop bootstrap server"); From 2bbd6acd84366b335f24df976b49451328c61ffa Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 17:26:46 +0200 Subject: [PATCH 53/83] Get bootstrap test passing! --- massa-bootstrap/src/client.rs | 1 - massa-bootstrap/src/client_binder.rs | 2 +- massa-bootstrap/src/establisher.rs | 2 +- massa-bootstrap/src/server.rs | 65 +++++++++++++++----------- massa-bootstrap/src/server_binder.rs | 8 +++- massa-bootstrap/src/tests/binders.rs | 6 +-- massa-bootstrap/src/tests/scenarios.rs | 4 +- 7 files changed, 52 insertions(+), 36 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 0c5f0e184e9..ab7c2e39a07 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -398,7 +398,6 @@ fn connect_to_server( .set_nonblocking(false) .map_err(|e| Box::new(e.into()))?; Ok(BootstrapClientBinder::new( - // this from_std will panic if this method doesn't exist within an async runtime... socket, *pub_key, bootstrap_config.into(), diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 90e323f22cd..2d0d3d3b988 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -153,7 +153,7 @@ impl BootstrapClientBinder { // send message length { - self.duplex.set_write_timeout(duration); + self.duplex.set_write_timeout(duration)?; let msg_len_bytes = msg_len.to_be_bytes_min(self.cfg.max_bootstrap_message_size)?; self.duplex.write_all(&msg_len_bytes)?; } diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index 5434e73f80e..ce41a5cb0b1 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -69,7 +69,7 @@ impl DefaultListener { // Number of connections to queue, set to the hardcoded value used by tokio socket.listen(1024)?; - socket.set_nonblocking(false); + socket.set_nonblocking(false)?; Ok(DefaultListener(socket.into())) } } diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 49f64e481eb..0200a68be7b 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -45,12 +45,13 @@ use massa_time::MassaTime; use parking_lot::RwLock; use std::{ collections::HashMap, + io::ErrorKind, net::{IpAddr, SocketAddr, TcpStream}, sync::Arc, thread, time::{Duration, Instant}, }; -use tokio::runtime::{self, Handle, Runtime}; +use tokio::runtime::{self, Handle}; use tracing::{debug, error, info, warn}; use crate::{ @@ -118,7 +119,7 @@ pub fn start_bootstrap_server( return Err(BootstrapError::GeneralError("Fail to convert u32 to usize".to_string()).into()); }; - // This is the primary interface between the async-listener, and the "sync" worker + // This is needed for now, as there is no ergonomic to "select!" on both a channel and a blocking std::net::TcpStream let (listener_tx, listener_rx) = crossbeam::channel::bounded::>(max_bootstraps * 2); @@ -741,12 +742,13 @@ async fn manage_bootstrap( massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - match tokio::time::timeout( - bootstrap_config.read_timeout.into(), - server.handshake(version), - ) - .await - { + match dbg!( + tokio::time::timeout( + bootstrap_config.read_timeout.into(), + server.handshake(version), + ) + .await + ) { Err(_) => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -758,13 +760,17 @@ async fn manage_bootstrap( Ok(Ok(_)) => (), }; - match tokio::time::timeout(read_error_timeout, server.next()).await { - Err(_) => (), - Ok(Err(e)) => return Err(e), - Ok(Ok(BootstrapClientMessage::BootstrapError { error })) => { + match dbg!(server.next_timeout(Some(read_error_timeout))) { + Err(BootstrapError::IoError(e)) + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => + { + () + } + Err(e) => return Err(e), + Ok(BootstrapClientMessage::BootstrapError { error }) => { return Err(BootstrapError::GeneralError(error)); } - Ok(Ok(msg)) => return Err(BootstrapError::UnexpectedClientMessage(msg)), + Ok(msg) => return Err(BootstrapError::UnexpectedClientMessage(msg)), }; let write_timeout: Duration = bootstrap_config.write_timeout.into(); @@ -772,16 +778,17 @@ async fn manage_bootstrap( // Sync clocks. let server_time = MassaTime::now()?; - match server - .send_msg( - write_timeout, - BootstrapServerMessage::BootstrapTime { - server_time, - version, - }, - ) - .await - { + match dbg!( + server + .send_msg( + write_timeout, + BootstrapServerMessage::BootstrapTime { + server_time, + version, + }, + ) + .await + ) { Err(_) => Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap clock send timed out", @@ -792,10 +799,14 @@ async fn manage_bootstrap( }?; loop { - match tokio::time::timeout(bootstrap_config.read_timeout.into(), server.next()).await { - Err(_) => break Ok(()), - Ok(Err(e)) => break Err(e), - Ok(Ok(msg)) => match msg { + match dbg!(server.next_timeout(Some(bootstrap_config.read_timeout.into()))) { + Err(BootstrapError::IoError(e)) + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => + { + break Ok(()) + } + Err(e) => break Err(e), + Ok(msg) => match msg { BootstrapClientMessage::AskBootstrapPeers => { match server .send_msg( diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 42bff5a9241..29dbdc37906 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -114,7 +114,7 @@ impl BootstrapServerBinder { /// 5. runs the passed in closure (typically a custom logging msg) /// /// consumes the binding in the process - pub(crate) fn close_and_send_error( + pub(crate) fn _close_and_send_error( mut self, server_outer_rt_hnd: Handle, msg: String, @@ -198,7 +198,11 @@ impl BootstrapServerBinder { #[allow(dead_code)] /// Read a message sent from the client (not signed). NOT cancel-safe - pub async fn next(&mut self) -> Result { + pub fn next_timeout( + &mut self, + duration: Option, + ) -> Result { + self.duplex.set_read_timeout(duration)?; // read prev hash let received_prev_hash = { if self.prev_message.is_some() { diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 2fe982d9813..779b83621a0 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -100,7 +100,7 @@ async fn test_binders() { server.handshake(version).await.unwrap(); server.send(test_peers_message.clone()).await.unwrap(); - let message = server.next().await.unwrap(); + let message = server.next_timeout(None).unwrap(); match message { BootstrapClientMessage::BootstrapError { error } => { assert_eq!(error, "test error"); @@ -290,7 +290,7 @@ async fn test_binders_try_double_send_client_works() { server.handshake(version).await.unwrap(); server.send(test_peers_message.clone()).await.unwrap(); - let message = server.next().await.unwrap(); + let message = server.next_timeout(None).unwrap(); match message { BootstrapClientMessage::BootstrapError { error } => { assert_eq!(error, "test error"); @@ -298,7 +298,7 @@ async fn test_binders_try_double_send_client_works() { _ => panic!("Bad message receive: Expected a peers list message"), } - let message = server.next().await.unwrap(); + let message = server.next_timeout(None).unwrap(); match message { BootstrapClientMessage::BootstrapError { error } => { assert_eq!(error, "test error"); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 5d3f6426d66..15ae7ae277f 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -136,7 +136,7 @@ fn test_bootstrap_server() { let final_state_server_clone1 = final_state_server.clone(); let final_state_server_clone2 = final_state_server.clone(); - let (mock_bs_listener, mut mock_remote_connector) = conn_establishment_mocks(); + let (mock_bs_listener, mock_remote_connector) = conn_establishment_mocks(); // Setup network command mock-story: hard-code the result of getting bootstrap peers let mut mocked1 = MockNetworkCommandSender::new(); @@ -261,6 +261,8 @@ fn test_bootstrap_server() { .apply_changes(change.executed_ops_changes.clone(), *slot); } } + // Make sure the modifier thread has done its job + mod_thread.join().unwrap(); // check final states assert_eq_final_state(&final_state_server.read(), &final_state_client.read()); From 3b73bdaf434436f595dae5d2392ac4643d06ad32 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 17:51:34 +0200 Subject: [PATCH 54/83] Make message sending from server sync --- massa-bootstrap/src/server.rs | 192 ++++++++++++--------------- massa-bootstrap/src/server_binder.rs | 50 +++---- massa-bootstrap/src/tests/binders.rs | 24 +++- 3 files changed, 130 insertions(+), 136 deletions(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 0200a68be7b..389f32383a7 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -277,12 +277,7 @@ impl BootstrapServer<'_, D, C> // check whether incoming peer IP is allowed. if let Err(error_msg) = self.white_black_list.is_ip_allowed(&remote_addr) { - // server_binding.close_and_send_error( - // self.bs_server_runtime.handle().clone(), - // error_msg.to_string(), - // remote_addr, - // move || {}, - // ); + server_binding.close_and_send_error(error_msg.to_string(), remote_addr, move || {}); continue; }; @@ -327,15 +322,11 @@ impl BootstrapServer<'_, D, C> "remote_addr": remote_addr }) }; - // server_binding.close_and_send_error( - // self.bs_server_runtime.handle().clone(), - // msg, - // remote_addr, - // tracer, - // ); + server_binding.close_and_send_error(msg, remote_addr, tracer); continue; - }; // Clients Option is good, and has been updated + }; + // Clients Option is good, and has been updated massa_trace!("bootstrap.lib.run.select.accept.cache_available", {}); // launch bootstrap @@ -368,12 +359,11 @@ impl BootstrapServer<'_, D, C> "active_count": Arc::strong_count(&bootstrap_sessions_counter) - 1 }); } else { - // server_binding.close_and_send_error( - // self.bs_server_runtime.handle().clone(), - // "Bootstrap failed because the bootstrap server currently has no slots available.".to_string(), - // remote_addr, - // move || debug!("did not bootstrap {}: no available slots", remote_addr), - // ); + server_binding.close_and_send_error( + "Bootstrap failed because the bootstrap server currently has no slots available.".to_string(), + remote_addr, + move || debug!("did not bootstrap {}: no available slots", remote_addr), + ); } } @@ -508,19 +498,17 @@ fn run_bootstrap_session( debug!("bootstrap serving error for peer {}: {}", remote_addr, err); // We allow unused result because we don't care if an error is thrown when // sending the error message to the server we will close the socket anyway. - let _ = server.send_error(err.to_string()).await; + let _ = server.send_error_timeout(err.to_string()); } }, Err(_timeout) => { debug!("bootstrap timeout for peer {}", remote_addr); // We allow unused result because we don't care if an error is thrown when // sending the error message to the server we will close the socket anyway. - let _ = server - .send_error(format!( - "Bootstrap process timedout ({})", - format_duration(config.bootstrap_timeout.to_duration()) - )) - .await; + let _ = server.send_error_timeout(format!( + "Bootstrap process timedout ({})", + format_duration(config.bootstrap_timeout.to_duration()) + )); } } }); @@ -620,17 +608,16 @@ pub async fn stream_bootstrap_information( } if slot_too_old { - match server - .send_msg(write_timeout, BootstrapServerMessage::SlotTooOld) - .await - { - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "SlotTooOld message send timed out", - ) - .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + match server.send_msg(write_timeout, BootstrapServerMessage::SlotTooOld) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "SlotTooOld message send timed out", + ) + .into()) + } + Err(e) => Err(e), + Ok(_) => Ok(()), }?; return Ok(()); } @@ -685,46 +672,44 @@ pub async fn stream_bootstrap_information( && final_state_changes_step.finished() && last_consensus_step.finished() { - match server - .send_msg(write_timeout, BootstrapServerMessage::BootstrapFinished) - .await - { - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + match server.send_msg(write_timeout, BootstrapServerMessage::BootstrapFinished) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()) + } + Err(e) => Err(e), + Ok(_) => Ok(()), }?; break; } // At this point we know that consensus, final state or both are not finished - match server - .send_msg( - write_timeout, - BootstrapServerMessage::BootstrapPart { - slot: current_slot, - ledger_part, - async_pool_part, - pos_cycle_part, - pos_credits_part, - exec_ops_part, - final_state_changes, - consensus_part, - consensus_outdated_ids, - }, - ) - .await - { - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + match server.send_msg( + write_timeout, + BootstrapServerMessage::BootstrapPart { + slot: current_slot, + ledger_part, + async_pool_part, + pos_cycle_part, + pos_credits_part, + exec_ops_part, + final_state_changes, + consensus_part, + consensus_outdated_ids, + }, + ) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()) + } + Err(e) => Err(e), + Ok(_) => Ok(()), }?; } Ok(()) @@ -778,24 +763,22 @@ async fn manage_bootstrap( // Sync clocks. let server_time = MassaTime::now()?; - match dbg!( - server - .send_msg( - write_timeout, - BootstrapServerMessage::BootstrapTime { - server_time, - version, - }, + match dbg!(server.send_msg( + write_timeout, + BootstrapServerMessage::BootstrapTime { + server_time, + version, + }, + )) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap clock send timed out", ) - .await - ) { - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap clock send timed out", - ) - .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + .into()) + } + Err(e) => Err(e), + Ok(_) => Ok(()), }?; loop { @@ -808,22 +791,21 @@ async fn manage_bootstrap( Err(e) => break Err(e), Ok(msg) => match msg { BootstrapClientMessage::AskBootstrapPeers => { - match server - .send_msg( - write_timeout, - BootstrapServerMessage::BootstrapPeers { - peers: network_command_sender.get_bootstrap_peers().await?, - }, - ) - .await - { - Err(_) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap peers send timed out", - ) - .into()), - Ok(Err(e)) => Err(e), - Ok(Ok(_)) => Ok(()), + match server.send_msg( + write_timeout, + BootstrapServerMessage::BootstrapPeers { + peers: network_command_sender.get_bootstrap_peers().await?, + }, + ) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap peers send timed out", + ) + .into()) + } + Err(e) => Err(e), + Ok(_) => Ok(()), }?; } BootstrapClientMessage::AskBootstrapPart { diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 29dbdc37906..6b1f9ba0cb5 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -15,11 +15,11 @@ use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::KeyPair; use massa_time::MassaTime; use std::convert::TryInto; +use std::io::ErrorKind; use std::net::SocketAddr; use std::thread; use std::time::Duration; use tokio::runtime::Handle; -use tokio::time::error::Elapsed; use tracing::error; /// Bootstrap server binder @@ -99,12 +99,12 @@ impl BootstrapServerBinder { Ok(()) } - pub async fn send_msg( + pub fn send_msg( &mut self, timeout: Duration, msg: BootstrapServerMessage, - ) -> Result, Elapsed> { - tokio::time::timeout(timeout, self.send(msg)).await + ) -> Result<(), BootstrapError> { + self.send_timeout(msg, Some(timeout)) } /// 1. Spawns a thread @@ -114,28 +114,22 @@ impl BootstrapServerBinder { /// 5. runs the passed in closure (typically a custom logging msg) /// /// consumes the binding in the process - pub(crate) fn _close_and_send_error( - mut self, - server_outer_rt_hnd: Handle, - msg: String, - addr: SocketAddr, - close_fn: F, - ) where + pub(crate) fn close_and_send_error(mut self, msg: String, addr: SocketAddr, close_fn: F) + where F: FnOnce() + Send + 'static, { thread::Builder::new() .name("bootstrap-error-send".to_string()) .spawn(move || { let msg_cloned = msg.clone(); - let err_send = - server_outer_rt_hnd.block_on(async move { self.send_error(msg_cloned).await }); + let err_send = self.send_error_timeout(msg_cloned); match err_send { - Err(_) => error!( + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => error!( "bootstrap server timed out sending error '{}' to addr {}", msg, addr ), - Ok(Err(e)) => error!("{}", e), - Ok(Ok(_)) => {} + Err(e) => error!("{}", e), + Ok(_) => {} } close_fn(); }) @@ -143,19 +137,25 @@ impl BootstrapServerBinder { // it's an error at the OS level. .unwrap(); } - pub async fn send_error( - &mut self, - error: String, - ) -> Result, Elapsed> { - tokio::time::timeout( - self.write_error_timeout.into(), - self.send(BootstrapServerMessage::BootstrapError { error }), + pub fn send_error_timeout(&mut self, error: String) -> Result<(), BootstrapError> { + self.send_timeout( + BootstrapServerMessage::BootstrapError { error }, + Some(self.write_error_timeout.to_duration()), ) - .await + .map_err(|e| match e { + BootstrapError::IoError(e) if e.kind() == ErrorKind::WouldBlock => { + BootstrapError::IoError(ErrorKind::TimedOut.into()) + } + e => e, + }) } /// Writes the next message. NOT cancel-safe - pub async fn send(&mut self, msg: BootstrapServerMessage) -> Result<(), BootstrapError> { + pub fn send_timeout( + &mut self, + msg: BootstrapServerMessage, + duration: Option, + ) -> Result<(), BootstrapError> { // serialize message let mut msg_bytes = Vec::new(); BootstrapServerMessageSerializer::new().serialize(&msg, &mut msg_bytes)?; diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 779b83621a0..b803d7bee80 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -98,7 +98,9 @@ async fn test_binders() { let version: Version = Version::from_str("TEST.1.10").unwrap(); server.handshake(version).await.unwrap(); - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); let message = server.next_timeout(None).unwrap(); match message { @@ -118,7 +120,9 @@ async fn test_binders() { peers: BootstrapPeers(vector_peers.clone()), }; - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); }); let client_thread = tokio::spawn(async move { @@ -202,7 +206,9 @@ async fn test_binders_double_send_server_works() { let version: Version = Version::from_str("TEST.1.10").unwrap(); server.handshake(version).await.unwrap(); - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); // Test message 2 let vector_peers = vec![ @@ -214,7 +220,9 @@ async fn test_binders_double_send_server_works() { peers: BootstrapPeers(vector_peers.clone()), }; - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); }); let client_thread = tokio::spawn(async move { @@ -288,7 +296,9 @@ async fn test_binders_try_double_send_client_works() { let version: Version = Version::from_str("TEST.1.10").unwrap(); server.handshake(version).await.unwrap(); - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); let message = server.next_timeout(None).unwrap(); match message { @@ -306,7 +316,9 @@ async fn test_binders_try_double_send_client_works() { _ => panic!("Bad message receive: Expected a peers list message"), } - server.send(test_peers_message.clone()).await.unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); }); let client_thread = tokio::spawn(async move { From 7c5c3d6751ba2b7afe64f405d66ac858593e9abd Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 18:12:00 +0200 Subject: [PATCH 55/83] Get binder tests working --- massa-bootstrap/src/server.rs | 14 +- massa-bootstrap/src/server_binder.rs | 9 +- massa-bootstrap/src/tests/binders.rs | 443 ++++++++++++++------------- 3 files changed, 240 insertions(+), 226 deletions(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 389f32383a7..0342dd14f27 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -727,22 +727,16 @@ async fn manage_bootstrap( massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - match dbg!( - tokio::time::timeout( - bootstrap_config.read_timeout.into(), - server.handshake(version), - ) - .await - ) { - Err(_) => { + match dbg!(server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))) { + Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap handshake send timed out", ) .into()); } - Ok(Err(e)) => return Err(e), - Ok(Ok(_)) => (), + Err(e) => return Err(e), + Ok(_) => (), }; match dbg!(server.next_timeout(Some(read_error_timeout))) { diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 6b1f9ba0cb5..606ba7741be 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -19,7 +19,6 @@ use std::io::ErrorKind; use std::net::SocketAddr; use std::thread; use std::time::Duration; -use tokio::runtime::Handle; use tracing::error; /// Bootstrap server binder @@ -75,13 +74,18 @@ impl BootstrapServerBinder { /// Performs a handshake. Should be called after connection /// NOT cancel-safe /// MUST always be followed by a send of the `BootstrapMessage::BootstrapTime` - pub async fn handshake(&mut self, version: Version) -> Result<(), BootstrapError> { + pub fn handshake_timeout( + &mut self, + version: Version, + duration: Option, + ) -> Result<(), BootstrapError> { // read version and random bytes, send signature let msg_hash = { let mut version_bytes = Vec::new(); self.version_serializer .serialize(&version, &mut version_bytes)?; let mut msg_bytes = vec![0u8; version_bytes.len() + self.randomness_size_bytes]; + self.duplex.set_read_timeout(duration)?; self.duplex.read_exact(&mut msg_bytes)?; let (_, received_version) = self .version_deserializer @@ -179,6 +183,7 @@ impl BootstrapServerBinder { }; // send signature + self.duplex.set_write_timeout(duration)?; self.duplex.write_all(&sig.to_bytes())?; // send message length diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index b803d7bee80..4cf4384546d 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -19,7 +19,6 @@ use massa_models::node::NodeId; use massa_models::version::Version; use massa_signature::{KeyPair, PublicKey}; use massa_time::MassaTime; -use serial_test::serial; use std::str::FromStr; lazy_static::lazy_static! { @@ -61,9 +60,8 @@ impl BootstrapClientBinder { } /// The server and the client will handshake and then send message in both ways in order -#[tokio::test] -#[serial] -async fn test_binders() { +#[test] +fn test_binders() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; let server = std::net::TcpListener::bind("localhost:0").unwrap(); let addr = server.local_addr().unwrap(); @@ -88,90 +86,96 @@ async fn test_binders() { bootstrap_config.bootstrap_list[0].1.get_public_key(), ); - let server_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let test_peers_message = BootstrapServerMessage::BootstrapPeers { - peers: BootstrapPeers(vector_peers.clone()), - }; - - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - server.handshake(version).await.unwrap(); - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - - let message = server.next_timeout(None).unwrap(); - match message { - BootstrapClientMessage::BootstrapError { error } => { - assert_eq!(error, "test error"); + let server_thread = std::thread::Builder::new() + .name("test_binders::server_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + let test_peers_message = BootstrapServerMessage::BootstrapPeers { + peers: BootstrapPeers(vector_peers.clone()), + }; + + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + server.handshake_timeout(version, None).unwrap(); + + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + + let message = server.next_timeout(None).unwrap(); + match message { + BootstrapClientMessage::BootstrapError { error } => { + assert_eq!(error, "test error"); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - - // Test message 3 - let vector_peers = vec![ - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - ]; - let test_peers_message = BootstrapServerMessage::BootstrapPeers { - peers: BootstrapPeers(vector_peers.clone()), - }; - - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - }); - let client_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - client.handshake(version).unwrap(); - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + // Test message 3 + let vector_peers = vec![ + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + ]; + let test_peers_message = BootstrapServerMessage::BootstrapPeers { + peers: BootstrapPeers(vector_peers.clone()), + }; + + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + }) + .unwrap(); + + let client_thread = std::thread::Builder::new() + .name("test_binders::server_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + client.handshake(version).unwrap(); + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - - client - .send_timeout( - &BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }, - None, - ) - .unwrap(); - - // Test message 3 - let vector_peers = vec![ - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - ]; - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + + client + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) + .unwrap(); + + // Test message 3 + let vector_peers = vec![ + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + ]; + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - }); + }) + .unwrap(); - server_thread.await.unwrap(); - client_thread.await.unwrap(); + server_thread.join().unwrap(); + client_thread.join().unwrap(); } /// The server and the client will handshake and then send message only from server to client -#[tokio::test] -#[serial] -async fn test_binders_double_send_server_works() { +#[test] +fn test_binders_double_send_server_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; let server = std::net::TcpListener::bind("localhost:0").unwrap(); @@ -196,73 +200,78 @@ async fn test_binders_double_send_server_works() { bootstrap_config.bootstrap_list[0].1.get_public_key(), ); - let server_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let test_peers_message = BootstrapServerMessage::BootstrapPeers { - peers: BootstrapPeers(vector_peers.clone()), - }; - - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - server.handshake(version).await.unwrap(); - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - - // Test message 2 - let vector_peers = vec![ - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - ]; - let test_peers_message = BootstrapServerMessage::BootstrapPeers { - peers: BootstrapPeers(vector_peers.clone()), - }; - - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - }); - - let client_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - client.handshake(version).unwrap(); - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + let server_thread = std::thread::Builder::new() + .name("test_buinders_double_send_server_works::server_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + let test_peers_message = BootstrapServerMessage::BootstrapPeers { + peers: BootstrapPeers(vector_peers.clone()), + }; + + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + server.handshake_timeout(version, None).unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + + // Test message 2 + let vector_peers = vec![ + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + ]; + let test_peers_message = BootstrapServerMessage::BootstrapPeers { + peers: BootstrapPeers(vector_peers.clone()), + }; + + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + }) + .unwrap(); + + let client_thread = std::thread::Builder::new() + .name("test_buinders_double_send_server_works::client_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + client.handshake(version).unwrap(); + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - - // Test message 2 - let vector_peers = vec![ - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - bootstrap_config.bootstrap_list[0].0.ip(), - ]; - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + + // Test message 2 + let vector_peers = vec![ + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + bootstrap_config.bootstrap_list[0].0.ip(), + ]; + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - }); + }) + .unwrap(); - server_thread.await.unwrap(); - client_thread.await.unwrap(); + server_thread.join().unwrap(); + client_thread.join().unwrap(); } /// The server and the client will handshake and then send message in both ways but the client will try to send two messages without answer -#[tokio::test] -#[serial] -async fn test_binders_try_double_send_client_works() { +#[test] +fn test_binders_try_double_send_client_works() { let (bootstrap_config, server_keypair): &(BootstrapConfig, KeyPair) = &BOOTSTRAP_CONFIG_KEYPAIR; let server = std::net::TcpListener::bind("localhost:0").unwrap(); @@ -287,83 +296,89 @@ async fn test_binders_try_double_send_client_works() { bootstrap_config.bootstrap_list[0].1.get_public_key(), ); - let server_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let test_peers_message = BootstrapServerMessage::BootstrapPeers { - peers: BootstrapPeers(vector_peers.clone()), - }; - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - server.handshake(version).await.unwrap(); - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - - let message = server.next_timeout(None).unwrap(); - match message { - BootstrapClientMessage::BootstrapError { error } => { - assert_eq!(error, "test error"); + let server_thread = std::thread::Builder::new() + .name("test_buinders_double_send_client_works::server_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + let test_peers_message = BootstrapServerMessage::BootstrapPeers { + peers: BootstrapPeers(vector_peers.clone()), + }; + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + server.handshake_timeout(version, None).unwrap(); + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + + let message = server.next_timeout(None).unwrap(); + match message { + BootstrapClientMessage::BootstrapError { error } => { + assert_eq!(error, "test error"); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - let message = server.next_timeout(None).unwrap(); - match message { - BootstrapClientMessage::BootstrapError { error } => { - assert_eq!(error, "test error"); + let message = server.next_timeout(None).unwrap(); + match message { + BootstrapClientMessage::BootstrapError { error } => { + assert_eq!(error, "test error"); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - - server - .send_timeout(test_peers_message.clone(), None) - .unwrap(); - }); - - let client_thread = tokio::spawn(async move { - // Test message 1 - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let version: Version = Version::from_str("TEST.1.10").unwrap(); - - client.handshake(version).unwrap(); - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + + server + .send_timeout(test_peers_message.clone(), None) + .unwrap(); + }) + .unwrap(); + + let client_thread = std::thread::Builder::new() + .name("test_buinders_double_send_client_works::client_thread".to_string()) + .spawn(move || { + // Test message 1 + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + let version: Version = Version::from_str("TEST.1.10").unwrap(); + + client.handshake(version).unwrap(); + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - - client - .send_timeout( - &BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }, - None, - ) - .unwrap(); - - // Test message 3 - client - .send_timeout( - &BootstrapClientMessage::BootstrapError { - error: "test error".to_string(), - }, - None, - ) - .unwrap(); - - let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; - let message = client.next_timeout(None).unwrap(); - match message { - BootstrapServerMessage::BootstrapPeers { peers } => { - assert_eq!(vector_peers, peers.0); + + client + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) + .unwrap(); + + // Test message 3 + client + .send_timeout( + &BootstrapClientMessage::BootstrapError { + error: "test error".to_string(), + }, + None, + ) + .unwrap(); + + let vector_peers = vec![bootstrap_config.bootstrap_list[0].0.ip()]; + let message = client.next_timeout(None).unwrap(); + match message { + BootstrapServerMessage::BootstrapPeers { peers } => { + assert_eq!(vector_peers, peers.0); + } + _ => panic!("Bad message receive: Expected a peers list message"), } - _ => panic!("Bad message receive: Expected a peers list message"), - } - }); + }) + .unwrap(); - server_thread.await.unwrap(); - client_thread.await.unwrap(); + server_thread.join().unwrap(); + client_thread.join().unwrap(); } From 38d5fc255ecd5cf5d80b37de9cc03e12cc2992a3 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 18:25:14 +0200 Subject: [PATCH 56/83] Fix issues created by merge --- massa-bootstrap/src/client.rs | 2 +- massa-bootstrap/src/tests/scenarios.rs | 20 +++++++++++--------- massa-node/src/main.rs | 7 +++---- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index ab7c2e39a07..e39a744421f 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -432,7 +432,7 @@ fn filter_bootstrap_list( /// Gets the state from a bootstrap server /// needs to be CANCELLABLE -pub fn get_state( +pub async fn get_state( bootstrap_config: &BootstrapConfig, final_state: Arc>, mut connector: impl BSConnector, diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 15ae7ae277f..92ec898b2ca 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -230,15 +230,17 @@ fn test_bootstrap_server() { // .times(1) // .returning(|_, _| todo!()); // launch the get_state process - let bootstrap_res = get_state( - bootstrap_config, - final_state_client_clone, - mock_remote_connector, - Version::from_str("TEST.1.10").unwrap(), - MassaTime::now().unwrap().saturating_sub(1000.into()), - None, - ) - .unwrap(); + let bootstrap_res = tokio::runtime::Runtime::new() + .unwrap() + .block_on(get_state( + bootstrap_config, + final_state_client_clone, + mock_remote_connector, + Version::from_str("TEST.1.10").unwrap(), + MassaTime::now().unwrap().saturating_sub(1000.into()), + None, + )) + .unwrap(); // apply the changes to the server state before matching with the client { diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index 4085c7f9192..15f3cabc933 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -65,13 +65,13 @@ use massa_storage::Storage; use massa_time::MassaTime; use massa_wallet::Wallet; use parking_lot::RwLock; +use std::net::TcpStream; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread::sleep; use std::time::Duration; use std::{path::Path, process, sync::Arc}; use structopt::StructOpt; -use tokio::net::TcpStream; use tokio::signal; use tokio::sync::{broadcast, mpsc}; use tracing::{error, info, warn}; @@ -243,7 +243,7 @@ async fn launch( res = get_state( &bootstrap_config, final_state.clone(), - DefaultConnector(bootstrap_config.connect_timeout), + DefaultConnector, *VERSION, *GENESIS_TIMESTAMP, *END_TIMESTAMP, @@ -514,7 +514,7 @@ async fn launch( // launch bootstrap server // TODO: use std::net::TcpStream let bootstrap_manager = match bootstrap_config.listen_addr { - Some(addr) => start_bootstrap_server::( + Some(addr) => start_bootstrap_server::( consensus_controller.clone(), network_command_sender.clone(), final_state.clone(), @@ -684,7 +684,6 @@ async fn stop( if let Some(bootstrap_manager) = bootstrap_manager { bootstrap_manager .stop() - .await .expect("bootstrap server shutdown failed") } From a8e5d571295e7b0025a8d2f40a999b97d542b1d9 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 19:02:10 +0200 Subject: [PATCH 57/83] clear clippy lints --- massa-bootstrap/src/client.rs | 4 ++-- massa-bootstrap/src/establisher.rs | 8 ++++---- massa-bootstrap/src/lib.rs | 2 ++ massa-bootstrap/src/server.rs | 5 +---- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index e39a744421f..01037da7229 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -459,7 +459,7 @@ pub async fn get_state( last_ops_step: StreamingStep::Started, last_consensus_step: StreamingStep::Started, }; - let mut global_bootstrap_state = GlobalBootstrapState::new(final_state.clone()); + let mut global_bootstrap_state = GlobalBootstrapState::new(final_state); loop { for (addr, node_id) in filtered_bootstrap_list.iter() { @@ -519,7 +519,7 @@ fn get_genesis_state( // create the initial cycle of PoS cycle_history final_state_guard.pos_state.create_initial_cycle(); } - return Ok(GlobalBootstrapState::new(final_state)); + Ok(GlobalBootstrapState::new(final_state)) } fn get_bootstrap_list_iter( diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index ce41a5cb0b1..ee3cb6b6616 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -18,15 +18,15 @@ pub trait Duplex: impl Duplex for std::net::TcpStream { fn set_read_timeout(&mut self, duration: Option) -> io::Result<()> { - Self::set_read_timeout(&self, duration) + Self::set_read_timeout(self, duration) } fn set_write_timeout(&mut self, duration: Option) -> io::Result<()> { - Self::set_write_timeout(&self, duration) + Self::set_write_timeout(self, duration) } fn set_nonblocking(&mut self, setting: bool) -> io::Result<()> { - Self::set_nonblocking(&self, setting) + Self::set_nonblocking(self, setting) } } @@ -99,7 +99,7 @@ impl BSConnector for DefaultConnector { duration: Option, ) -> io::Result { let Some(duration) = duration else { - return TcpStream::connect(&addr); + return TcpStream::connect(addr); }; TcpStream::connect_timeout(&addr, duration.to_duration()) } diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index f4df5f91bad..89b89f60e25 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -12,6 +12,8 @@ #![warn(unused_crate_dependencies)] #![feature(ip)] #![feature(let_chains)] +// TODO: Box large variants of BootstrapError +#![allow(clippy::result_large_err)] use massa_consensus_exports::bootstrapable_graph::BootstrapableGraph; use massa_final_state::FinalState; diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index f61a6f9d37a..2b539662472 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -737,10 +737,7 @@ async fn manage_bootstrap( match dbg!(server.next_timeout(Some(read_error_timeout))) { Err(BootstrapError::IoError(e)) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => - { - () - } + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => {} Err(e) => return Err(e), Ok(BootstrapClientMessage::BootstrapError { error }) => { return Err(BootstrapError::GeneralError(error)); From b1a4e35418eeecd7cdcf982a98a0d836ea424c2d Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 19:03:19 +0200 Subject: [PATCH 58/83] Remove unused dependency --- Cargo.lock | 1 - massa-bootstrap/Cargo.toml | 1 - 2 files changed, 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c82f51cb7a4..0a3b828fc46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2479,7 +2479,6 @@ dependencies = [ "rand 0.8.5", "serde", "serde_json", - "serial_test 1.0.0", "socket2", "substruct", "tempfile", diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index 28533e6a320..8684b1820a8 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -40,7 +40,6 @@ massa_time = { path = "../massa-time" } [dev-dependencies] mockall = "0.11.4" bitvec = { version = "1.0", features = ["serde"] } -serial_test = "1.0" massa_final_state = { path = "../massa-final-state", features = ["testing"] } massa_async_pool = { path = "../massa-async-pool", features = ["testing"] } massa_ledger_worker = { path = "../massa-ledger-worker", features = [ From 68c1efeafd721cf78f1287f2c6f1f10c65bd9d05 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Thu, 30 Mar 2023 19:11:11 +0200 Subject: [PATCH 59/83] Remove `dbg!`s --- massa-bootstrap/src/client.rs | 25 ++++++++----------------- massa-bootstrap/src/server.rs | 10 +++++----- massa-bootstrap/src/tests/scenarios.rs | 1 - 3 files changed, 13 insertions(+), 23 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 01037da7229..5ad178fc459 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -47,7 +47,6 @@ fn stream_final_state_and_consensus( Ok(_) => Ok(()), }?; loop { - dbg!(); let msg = match client.next_timeout(Some(cfg.read_timeout.to_duration())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( @@ -205,29 +204,27 @@ fn bootstrap_from_server( // read error (if sent by the server) // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match dbg!(client.next_timeout(Some(cfg.read_error_timeout.to_duration()))) { + match client.next_timeout(Some(cfg.read_error_timeout.to_duration())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => { - dbg!("no next"); massa_trace!( "bootstrap.lib.bootstrap_from_server: No error sent at connection", {} ); } - Err(e) => return Err(dbg!(e)), + Err(e) => return Err(e), Ok(BootstrapServerMessage::BootstrapError { error: err }) => { - return Err(BootstrapError::ReceivedError(dbg!(err))) + return Err(BootstrapError::ReceivedError(err)) } - Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(dbg!(msg))), + Ok(msg) => return Err(BootstrapError::UnexpectedServerMessage(msg)), }; - dbg!("doing handshake"); // handshake let send_time_uncompensated = MassaTime::now()?; // client.handshake() is not cancel-safe but we drop the whole client object if cancelled => it's OK match // tokio::time::timeout(cfg.write_timeout.into(), - dbg!(client.handshake(our_version)) { + client.handshake(our_version) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -249,8 +246,7 @@ fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK - dbg!(); - let server_time = match dbg!(client.next_timeout(Some(cfg.read_timeout.into()))) { + let server_time = match client.next_timeout(Some(cfg.read_timeout.into())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => { @@ -260,7 +256,7 @@ fn bootstrap_from_server( ) .into()) } - Err(e) => return Err(dbg!(e)), + Err(e) => return Err(e), Ok(BootstrapServerMessage::BootstrapTime { server_time, version, @@ -290,7 +286,6 @@ fn bootstrap_from_server( )); } - dbg!(); // compute client / server clock delta // div 2 is an approximation of the time it took the message to do server -> client // the complete ping value being client -> server -> client @@ -307,11 +302,9 @@ fn bootstrap_from_server( return Err(BootstrapError::ClockError(message)); } - dbg!(); let write_timeout: std::time::Duration = cfg.write_timeout.into(); // Loop to ask data to the server depending on the last message we sent loop { - dbg!(); match next_bootstrap_message { BootstrapClientMessage::AskBootstrapPart { .. } => { stream_final_state_and_consensus( @@ -375,7 +368,6 @@ fn send_client_message( Err(e) => Err(e), Ok(_) => Ok(()), }?; - dbg!(); match client.next_timeout(Some(read_timeout)) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) @@ -476,8 +468,7 @@ pub async fn get_state( &node_id.get_public_key(), ) { Ok(mut client) => { - dbg!("got the con, bootstrapping"); - match dbg!(bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version)) + match bootstrap_from_server(bootstrap_config, &mut client, &mut next_bootstrap_message, &mut global_bootstrap_state,version) // cancellable { Err(BootstrapError::ReceivedError(error)) => warn!("Error received from bootstrap server: {}", error), diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 2b539662472..082e9c926c1 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -723,7 +723,7 @@ async fn manage_bootstrap( massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - match dbg!(server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))) { + match server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -735,7 +735,7 @@ async fn manage_bootstrap( Ok(_) => (), }; - match dbg!(server.next_timeout(Some(read_error_timeout))) { + match server.next_timeout(Some(read_error_timeout)) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => {} Err(e) => return Err(e), @@ -750,13 +750,13 @@ async fn manage_bootstrap( // Sync clocks. let server_time = MassaTime::now()?; - match dbg!(server.send_msg( + match server.send_msg( write_timeout, BootstrapServerMessage::BootstrapTime { server_time, version, }, - )) { + ) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { Err(std::io::Error::new( std::io::ErrorKind::TimedOut, @@ -769,7 +769,7 @@ async fn manage_bootstrap( }?; loop { - match dbg!(server.next_timeout(Some(bootstrap_config.read_timeout.into()))) { + match server.next_timeout(Some(bootstrap_config.read_timeout.into())) { Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => { diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 92ec898b2ca..0fd597c65b4 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -289,7 +289,6 @@ fn test_bootstrap_server() { // check graphs assert_eq_bootstrap_graph(&sent_graph, &bootstrap_res.graph.unwrap()); - dbg!("now stop! (hammer time)"); // stop bootstrap server bootstrap_manager_thread .join() From cd8392ce35bfec784c5ac512c2afa72ec6cadf16 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Fri, 31 Mar 2023 15:02:29 +0200 Subject: [PATCH 60/83] Add timed-out variant to bootstrap error --- massa-bootstrap/src/client.rs | 42 +++++++++----------- massa-bootstrap/src/error.rs | 23 ++++++++++- massa-bootstrap/src/server.rs | 72 ++++++++++++++--------------------- 3 files changed, 66 insertions(+), 71 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 5ad178fc459..f7336448781 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -1,5 +1,5 @@ use humantime::format_duration; -use std::{collections::HashSet, io::ErrorKind, net::SocketAddr, sync::Arc, time::Duration}; +use std::{collections::HashSet, net::SocketAddr, sync::Arc, time::Duration}; use massa_final_state::FinalState; use massa_logging::massa_trace; @@ -36,19 +36,17 @@ fn stream_final_state_and_consensus( next_bootstrap_message, Some(cfg.write_timeout.to_duration()), ) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; loop { let msg = match client.next_timeout(Some(cfg.read_timeout.to_duration())) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(BootstrapError::TimedOut(_)) => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "final state bootstrap read timed out", @@ -205,9 +203,7 @@ fn bootstrap_from_server( // read error (if sent by the server) // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK match client.next_timeout(Some(cfg.read_error_timeout.to_duration())) { - Err(BootstrapError::IoError(e)) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => - { + Err(BootstrapError::TimedOut(_)) => { massa_trace!( "bootstrap.lib.bootstrap_from_server: No error sent at connection", {} @@ -225,7 +221,7 @@ fn bootstrap_from_server( // client.handshake() is not cancel-safe but we drop the whole client object if cancelled => it's OK match // tokio::time::timeout(cfg.write_timeout.into(), client.handshake(our_version) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(BootstrapError::TimedOut(_)) => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap handshake timed out", @@ -247,9 +243,7 @@ fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK let server_time = match client.next_timeout(Some(cfg.read_timeout.into())) { - Err(BootstrapError::IoError(e)) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => - { + Err(BootstrapError::TimedOut(_)) => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap clock sync read timed out", @@ -333,13 +327,11 @@ fn bootstrap_from_server( } BootstrapClientMessage::BootstrapSuccess => { match client.send_timeout(next_bootstrap_message, Some(write_timeout)) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "send bootstrap success timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "send bootstrap success timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; @@ -362,14 +354,14 @@ fn send_client_message( error: &str, ) -> Result { match client.send_timeout(message_to_send, Some(write_timeout)) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(BootstrapError::TimedOut(_)) => { Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) } Err(e) => Err(e), Ok(_) => Ok(()), }?; match client.next_timeout(Some(read_timeout)) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(BootstrapError::TimedOut(_)) => { Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) } Err(e) => Err(e), diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index d8f24d68630..2dc480eaa69 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -1,5 +1,7 @@ // Copyright (c) 2022 MASSA LABS +use std::io::ErrorKind; + use crate::messages::{BootstrapClientMessage, BootstrapServerMessage}; use displaydoc::Display; use massa_consensus_exports::error::ConsensusError; @@ -14,8 +16,10 @@ use thiserror::Error; #[non_exhaustive] #[derive(Display, Error, Debug)] pub enum BootstrapError { - /// io error: {0} - IoError(#[from] std::io::Error), + /// all io errors except for Timedout, and would-block (unix error when timed out) + IoError(std::io::Error), + /// We want to handle Timedout and WouldBlock + TimedOut(std::io::Error), /// general bootstrap error: {0} GeneralError(String), /// models error: {0} @@ -59,3 +63,18 @@ pub enum BootstrapError { /// IP {0} is not in the whitelist WhiteListed(String), } + +/// # Platform-specific behavior +/// +/// Platforms may return a different error code whenever a read times out as +/// a result of setting this option. For example Unix typically returns an +/// error of the kind [`WouldBlock`], but Windows may return [`TimedOut`].) +impl From for BootstrapError { + fn from(e: std::io::Error) -> Self { + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock { + BootstrapError::TimedOut(e) + } else { + BootstrapError::IoError(e) + } + } +} diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 082e9c926c1..8ba6891c5e0 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -45,7 +45,6 @@ use massa_time::MassaTime; use parking_lot::RwLock; use std::{ collections::HashMap, - io::ErrorKind, net::{IpAddr, SocketAddr, TcpStream}, sync::Arc, thread, @@ -605,13 +604,11 @@ pub async fn stream_bootstrap_information( if slot_too_old { match server.send_msg(write_timeout, BootstrapServerMessage::SlotTooOld) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "SlotTooOld message send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "SlotTooOld message send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; @@ -669,13 +666,11 @@ pub async fn stream_bootstrap_information( && last_consensus_step.finished() { match server.send_msg(write_timeout, BootstrapServerMessage::BootstrapFinished) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; @@ -697,13 +692,11 @@ pub async fn stream_bootstrap_information( consensus_outdated_ids, }, ) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap ask ledger part send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; @@ -724,7 +717,7 @@ async fn manage_bootstrap( let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); match server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into())) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { + Err(BootstrapError::TimedOut(_)) => { return Err(std::io::Error::new( std::io::ErrorKind::TimedOut, "bootstrap handshake send timed out", @@ -736,8 +729,7 @@ async fn manage_bootstrap( }; match server.next_timeout(Some(read_error_timeout)) { - Err(BootstrapError::IoError(e)) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => {} + Err(BootstrapError::TimedOut(_)) => {} Err(e) => return Err(e), Ok(BootstrapClientMessage::BootstrapError { error }) => { return Err(BootstrapError::GeneralError(error)); @@ -757,24 +749,18 @@ async fn manage_bootstrap( version, }, ) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap clock send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap clock send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; loop { match server.next_timeout(Some(bootstrap_config.read_timeout.into())) { - Err(BootstrapError::IoError(e)) - if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => - { - break Ok(()) - } + Err(BootstrapError::TimedOut(_)) => break Ok(()), Err(e) => break Err(e), Ok(msg) => match msg { BootstrapClientMessage::AskBootstrapPeers => { @@ -784,13 +770,11 @@ async fn manage_bootstrap( peers: network_command_sender.get_bootstrap_peers().await?, }, ) { - Err(BootstrapError::IoError(e)) if e.kind() == ErrorKind::TimedOut => { - Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap peers send timed out", - ) - .into()) - } + Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( + std::io::ErrorKind::TimedOut, + "bootstrap peers send timed out", + ) + .into()), Err(e) => Err(e), Ok(_) => Ok(()), }?; From f706b85731086a7c19c8c6c3bac1ce187062a6e8 Mon Sep 17 00:00:00 2001 From: Modship Date: Mon, 3 Apr 2023 13:38:40 +0200 Subject: [PATCH 61/83] remove clippy::result_large_err (#3751) --- massa-bootstrap/src/client.rs | 10 +++----- massa-bootstrap/src/error.rs | 2 +- massa-bootstrap/src/lib.rs | 2 -- massa-bootstrap/src/server.rs | 24 +++++++++---------- .../src/server/white_black_list.rs | 22 +++++++---------- 5 files changed, 24 insertions(+), 36 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index f7336448781..09a3057139f 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -374,13 +374,9 @@ fn connect_to_server( bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, -) -> Result, Box> { - let socket = connector - .connect_timeout(*addr, Some(bootstrap_config.connect_timeout)) - .map_err(|e| Box::new(e.into()))?; - socket - .set_nonblocking(false) - .map_err(|e| Box::new(e.into()))?; +) -> Result, BootstrapError> { + let socket = connector.connect_timeout(*addr, Some(bootstrap_config.connect_timeout))?; + socket.set_nonblocking(false)?; Ok(BootstrapClientBinder::new( socket, *pub_key, diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index 2dc480eaa69..8b326d53990 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -29,7 +29,7 @@ pub enum BootstrapError { /// unexpected message received from server: {0:?} UnexpectedServerMessage(BootstrapServerMessage), /// unexpected message received from client: {0:?} - UnexpectedClientMessage(BootstrapClientMessage), + UnexpectedClientMessage(Box), /// connection with bootstrap node dropped UnexpectedConnectionDrop, /// `massa_hash` error: {0} diff --git a/massa-bootstrap/src/lib.rs b/massa-bootstrap/src/lib.rs index 89b89f60e25..f4df5f91bad 100644 --- a/massa-bootstrap/src/lib.rs +++ b/massa-bootstrap/src/lib.rs @@ -12,8 +12,6 @@ #![warn(unused_crate_dependencies)] #![feature(ip)] #![feature(let_chains)] -// TODO: Box large variants of BootstrapError -#![allow(clippy::result_large_err)] use massa_consensus_exports::bootstrapable_graph::BootstrapableGraph; use massa_final_state::FinalState; diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 8ba6891c5e0..921656249d7 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -67,18 +67,18 @@ type BsConn = (D, SocketAddr); /// handle on the bootstrap server pub struct BootstrapManager { - update_handle: thread::JoinHandle>>, + update_handle: thread::JoinHandle>, // need to preserve the listener handle up to here to prevent it being destroyed #[allow(clippy::type_complexity)] - _listen_handle: thread::JoinHandle>, Box>>, - main_handle: thread::JoinHandle>>, + _listen_handle: thread::JoinHandle>, BootstrapError>>, + main_handle: thread::JoinHandle>, listen_stopper_tx: crossbeam::channel::Sender<()>, update_stopper_tx: crossbeam::channel::Sender<()>, } impl BootstrapManager { /// stop the bootstrap server - pub fn stop(self) -> Result<(), Box> { + pub fn stop(self) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.stop", {}); if self.listen_stopper_tx.send(()).is_err() { warn!("bootstrap server already dropped"); @@ -109,7 +109,7 @@ pub fn start_bootstrap_server( listener: impl BSListener + Send + 'static, keypair: KeyPair, version: Version, -) -> Result>, Box> { +) -> Result>, BootstrapError> { massa_trace!("bootstrap.lib.start_bootstrap_server", {}); // TODO(low prio): See if a zero capacity channel model can work @@ -117,7 +117,7 @@ pub fn start_bootstrap_server( let (update_stopper_tx, update_stopper_rx) = crossbeam::channel::bounded::<()>(1); let Ok(max_bootstraps) = config.max_simultaneous_bootstraps.try_into() else { - return Err(BootstrapError::GeneralError("Fail to convert u32 to usize".to_string()).into()); + return Err(BootstrapError::GeneralError("Fail to convert u32 to usize".to_string())); }; // This is needed for now, as there is no ergonomic to "select!" on both a channel and a blocking std::net::TcpStream @@ -199,7 +199,7 @@ impl BootstrapServer<'_, D, C> mut list: SharedWhiteBlackList<'_>, interval: Duration, stopper: crossbeam::channel::Receiver<()>, - ) -> Result<(), Box> { + ) -> Result<(), BootstrapError> { let ticker = tick(interval); loop { @@ -207,7 +207,7 @@ impl BootstrapServer<'_, D, C> recv(stopper) -> res => { match res { Ok(()) => return Ok(()), - Err(e) => return Err(Box::new(BootstrapError::GeneralError(format!("update stopper error : {}", e)))), + Err(e) => return Err(BootstrapError::GeneralError(format!("update stopper error : {}", e))), } }, recv(ticker) -> _ => {list.update()?;}, @@ -225,7 +225,7 @@ impl BootstrapServer<'_, D, C> fn run_listener( mut listener: impl BSListener, listener_tx: crossbeam::channel::Sender>, - ) -> Result>, Box> { + ) -> Result>, BootstrapError> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; match listener_tx.send((msg, addr)) { @@ -241,7 +241,7 @@ impl BootstrapServer<'_, D, C> } } - fn run_loop(mut self, max_bootstraps: usize) -> Result<(), Box> { + fn run_loop(mut self, max_bootstraps: usize) -> Result<(), BootstrapError> { let Ok(bs_loop_rt) = runtime::Builder::new_multi_thread() .max_blocking_threads(max_bootstraps * 2) .enable_io() @@ -249,7 +249,7 @@ impl BootstrapServer<'_, D, C> .thread_name("bootstrap-main-loop-worker") .thread_keep_alive(Duration::from_millis(u64::MAX)) .build() else { - return Err(Box::new(BootstrapError::GeneralError("Failed to create bootstrap main-loop runtime".to_string()))); + return Err(BootstrapError::GeneralError("Failed to create bootstrap main-loop runtime".to_string())); }; // Use the strong-count of this variable to track the session count @@ -734,7 +734,7 @@ async fn manage_bootstrap( Ok(BootstrapClientMessage::BootstrapError { error }) => { return Err(BootstrapError::GeneralError(error)); } - Ok(msg) => return Err(BootstrapError::UnexpectedClientMessage(msg)), + Ok(msg) => return Err(BootstrapError::UnexpectedClientMessage(Box::new(msg))), }; let write_timeout: Duration = bootstrap_config.write_timeout.into(); diff --git a/massa-bootstrap/src/server/white_black_list.rs b/massa-bootstrap/src/server/white_black_list.rs index c653c429800..3fd89ddebf9 100644 --- a/massa-bootstrap/src/server/white_black_list.rs +++ b/massa-bootstrap/src/server/white_black_list.rs @@ -24,10 +24,7 @@ pub(crate) struct SharedWhiteBlackList<'a> { } impl SharedWhiteBlackList<'_> { - pub(crate) fn new( - white_path: PathBuf, - black_path: PathBuf, - ) -> Result> { + pub(crate) fn new(white_path: PathBuf, black_path: PathBuf) -> Result { let (white_list, black_list) = WhiteBlackListInner::init_list(&white_path, &black_path)?; Ok(Self { inner: Arc::new(RwLock::new(WhiteBlackListInner { @@ -41,7 +38,7 @@ impl SharedWhiteBlackList<'_> { /// Checks if the white/black list is up to date with a read-lock /// Creates a new list, and replaces the old one in a write-lock - pub(crate) fn update(&mut self) -> Result<(), Box> { + pub(crate) fn update(&mut self) -> Result<(), BootstrapError> { let read_lock = self.inner.read(); let (new_white, new_black) = WhiteBlackListInner::update_list(&self.white_path, &self.black_path)?; @@ -67,10 +64,7 @@ impl SharedWhiteBlackList<'_> { } #[cfg_attr(test, allow(unreachable_code, unused_variables))] - pub(crate) fn is_ip_allowed( - &self, - remote_addr: &SocketAddr, - ) -> Result<(), Box> { + pub(crate) fn is_ip_allowed(&self, remote_addr: &SocketAddr) -> Result<(), BootstrapError> { #[cfg(test)] return Ok(()); @@ -79,11 +73,11 @@ impl SharedWhiteBlackList<'_> { let read = self.inner.read(); if let Some(ip_list) = &read.black_list && ip_list.contains(&ip) { massa_trace!("bootstrap.lib.run.select.accept.refuse_blacklisted", {"remote_addr": remote_addr}); - Err(Box::new(BootstrapError::BlackListed(ip.to_string()))) + Err(BootstrapError::BlackListed(ip.to_string())) // whether the peer IP address is not present in the whitelist } else if let Some(ip_list) = &read.white_list && !ip_list.contains(&ip) { massa_trace!("bootstrap.lib.run.select.accept.refuse_not_whitelisted", {"remote_addr": remote_addr}); - Err(Box::new(BootstrapError::WhiteListed(ip.to_string()))) + Err(BootstrapError::WhiteListed(ip.to_string())) } else { Ok(()) } @@ -96,7 +90,7 @@ impl WhiteBlackListInner { fn update_list( whitelist_path: &Path, blacklist_path: &Path, - ) -> Result<(Option>, Option>), Box> { + ) -> Result<(Option>, Option>), BootstrapError> { Ok(( Self::load_list(whitelist_path, false)?, Self::load_list(blacklist_path, false)?, @@ -107,7 +101,7 @@ impl WhiteBlackListInner { fn init_list( whitelist_path: &Path, blacklist_path: &Path, - ) -> Result<(Option>, Option>), Box> { + ) -> Result<(Option>, Option>), BootstrapError> { Ok(( Self::load_list(whitelist_path, true)?, Self::load_list(blacklist_path, true)?, @@ -117,7 +111,7 @@ impl WhiteBlackListInner { fn load_list( list_path: &Path, is_init: bool, - ) -> Result>, Box> { + ) -> Result>, BootstrapError> { match std::fs::read_to_string(list_path) { Err(e) => { if is_init { From c6745338f918224be725b80da2600da88e945f62 Mon Sep 17 00:00:00 2001 From: modship Date: Mon, 3 Apr 2023 13:44:56 +0200 Subject: [PATCH 62/83] remove #[allow(clippy::result_large_err)] in white_black_list.rs --- massa-bootstrap/src/server/white_black_list.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/massa-bootstrap/src/server/white_black_list.rs b/massa-bootstrap/src/server/white_black_list.rs index 3fd89ddebf9..e82b5e68d71 100644 --- a/massa-bootstrap/src/server/white_black_list.rs +++ b/massa-bootstrap/src/server/white_black_list.rs @@ -85,7 +85,6 @@ impl SharedWhiteBlackList<'_> { } impl WhiteBlackListInner { - #[allow(clippy::result_large_err)] #[allow(clippy::type_complexity)] fn update_list( whitelist_path: &Path, From 45e37f66488052640e8d1cdfff8969c7ee0a2a6d Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Mon, 3 Apr 2023 18:47:53 +0200 Subject: [PATCH 63/83] why unwrap, when you have ?... ? --- massa-bootstrap/src/client_binder.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 2d0d3d3b988..f66470f3f2b 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -72,7 +72,7 @@ impl BootstrapClientBinder { &mut self, duration: Option, ) -> Result { - self.duplex.set_read_timeout(duration).unwrap(); + self.duplex.set_read_timeout(duration)?; // read signature let sig = { let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; From eb464465189595da5f9424dd979a4d9df3781028 Mon Sep 17 00:00:00 2001 From: Modship Date: Tue, 4 Apr 2023 12:49:26 +0200 Subject: [PATCH 64/83] improve error and readability (#3757) * refactor(bootstrap) : improve error and readability * send_msg timeout error (#3763) * refactor(bootstrap) : send_msg time out --------- Co-authored-by: modship Co-authored-by: Ben * refactor(bootstrap) : remove comment * add comment Co-authored-by: Ben --------- Co-authored-by: modship Co-authored-by: Ben --- massa-bootstrap/src/client.rs | 91 ++++++------------------ massa-bootstrap/src/client_binder.rs | 4 +- massa-bootstrap/src/error.rs | 2 + massa-bootstrap/src/messages.rs | 15 ++++ massa-bootstrap/src/server.rs | 100 ++++++++------------------- massa-bootstrap/src/server_binder.rs | 16 ++++- 6 files changed, 81 insertions(+), 147 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index 09a3057139f..b085cf8c5c7 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -32,31 +32,13 @@ fn stream_final_state_and_consensus( global_bootstrap_state: &mut GlobalBootstrapState, ) -> Result<(), BootstrapError> { if let BootstrapClientMessage::AskBootstrapPart { .. } = &next_bootstrap_message { - match client.send_timeout( + client.send_timeout( next_bootstrap_message, Some(cfg.write_timeout.to_duration()), - ) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + )?; + loop { - let msg = match client.next_timeout(Some(cfg.read_timeout.to_duration())) { - Err(BootstrapError::TimedOut(_)) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "final state bootstrap read timed out", - ) - .into()); - } - Err(e) => return Err(e), - Ok(msg) => msg, - }; - match msg { + match client.next_timeout(Some(cfg.read_timeout.to_duration()))? { BootstrapServerMessage::BootstrapPart { slot, ledger_part, @@ -169,15 +151,14 @@ fn stream_final_state_and_consensus( write_final_state.reset(); return Err(BootstrapError::GeneralError(String::from("Slot too old"))); } + // At this point, we have succesfully received the next message from the server, and it's an error-message String BootstrapServerMessage::BootstrapError { error } => { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, error).into()) + return Err(BootstrapError::GeneralError(error)) } _ => { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "unexpected message", - ) - .into()) + return Err(BootstrapError::GeneralError( + "unexpected message".to_string(), + )) } } } @@ -219,18 +200,7 @@ fn bootstrap_from_server( // handshake let send_time_uncompensated = MassaTime::now()?; // client.handshake() is not cancel-safe but we drop the whole client object if cancelled => it's OK - match // tokio::time::timeout(cfg.write_timeout.into(), - client.handshake(our_version) { - Err(BootstrapError::TimedOut(_)) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap handshake timed out", - ) - .into()); - } - Err(e) => return Err(e), - Ok(_) => {} - } + client.handshake(our_version)?; // compute ping let ping = MassaTime::now()?.saturating_sub(send_time_uncompensated); @@ -243,13 +213,6 @@ fn bootstrap_from_server( // First, clock and version. // client.next() is not cancel-safe but we drop the whole client object if cancelled => it's OK let server_time = match client.next_timeout(Some(cfg.read_timeout.into())) { - Err(BootstrapError::TimedOut(_)) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap clock sync read timed out", - ) - .into()) - } Err(e) => return Err(e), Ok(BootstrapServerMessage::BootstrapTime { server_time, @@ -326,15 +289,7 @@ fn bootstrap_from_server( *next_bootstrap_message = BootstrapClientMessage::BootstrapSuccess; } BootstrapClientMessage::BootstrapSuccess => { - match client.send_timeout(next_bootstrap_message, Some(write_timeout)) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "send bootstrap success timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + client.send_timeout(next_bootstrap_message, Some(write_timeout))?; break; } BootstrapClientMessage::BootstrapError { error: _ } => { @@ -353,20 +308,16 @@ fn send_client_message( read_timeout: Duration, error: &str, ) -> Result { - match client.send_timeout(message_to_send, Some(write_timeout)) { - Err(BootstrapError::TimedOut(_)) => { - Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) - } - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; - match client.next_timeout(Some(read_timeout)) { - Err(BootstrapError::TimedOut(_)) => { - Err(std::io::Error::new(std::io::ErrorKind::TimedOut, error).into()) - } - Err(e) => Err(e), - Ok(msg) => Ok(msg), - } + client.send_timeout(message_to_send, Some(write_timeout))?; + + client + .next_timeout(Some(read_timeout)) + .map_err(|e| match e { + BootstrapError::TimedOut(_) => { + BootstrapError::TimedOut(std::io::Error::new(std::io::ErrorKind::TimedOut, error)) + } + _ => e, + }) } fn connect_to_server( diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index f66470f3f2b..6e432bdab92 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -100,7 +100,7 @@ impl BootstrapClientBinder { self.remote_pubkey.verify_signature(&msg_hash, &sig)?; let (_, msg) = message_deserializer .deserialize::(&sig_msg_bytes[HASH_SIZE_BYTES..]) - .map_err(|err| BootstrapError::GeneralError(format!("{}", err)))?; + .map_err(|err| BootstrapError::DeserializeError(format!("{}", err)))?; msg } else { self.prev_message = Some(Hash::compute_from(&sig.to_bytes())); @@ -110,7 +110,7 @@ impl BootstrapClientBinder { self.remote_pubkey.verify_signature(&msg_hash, &sig)?; let (_, msg) = message_deserializer .deserialize::(&sig_msg_bytes[..]) - .map_err(|err| BootstrapError::GeneralError(format!("{}", err)))?; + .map_err(|err| BootstrapError::DeserializeError(format!("{}", err)))?; msg } }; diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index 8b326d53990..ae10c6e562b 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -22,6 +22,8 @@ pub enum BootstrapError { TimedOut(std::io::Error), /// general bootstrap error: {0} GeneralError(String), + /// deserialization error: {0} + DeserializeError(String), /// models error: {0} ModelsError(#[from] massa_models::error::ModelsError), /// serialize error: {0} diff --git a/massa-bootstrap/src/messages.rs b/massa-bootstrap/src/messages.rs index 14c1beb602b..a5bf51bf1a2 100644 --- a/massa-bootstrap/src/messages.rs +++ b/massa-bootstrap/src/messages.rs @@ -93,6 +93,21 @@ pub enum BootstrapServerMessage { }, } +impl ToString for BootstrapServerMessage { + fn to_string(&self) -> String { + match self { + BootstrapServerMessage::BootstrapTime { .. } => "BootstrapTime".to_string(), + BootstrapServerMessage::BootstrapPeers { .. } => "BootstrapPeers".to_string(), + BootstrapServerMessage::BootstrapPart { .. } => "BootstrapPart".to_string(), + BootstrapServerMessage::BootstrapFinished => "BootstrapFinished".to_string(), + BootstrapServerMessage::SlotTooOld => "SlotTooOld".to_string(), + BootstrapServerMessage::BootstrapError { error } => { + format!("BootstrapError {{ error: {} }}", error) + } + } + } +} + #[derive(IntoPrimitive, Debug, Eq, PartialEq, TryFromPrimitive)] #[repr(u32)] enum MessageServerTypeId { diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 921656249d7..3c66d01b2ff 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -228,15 +228,13 @@ impl BootstrapServer<'_, D, C> ) -> Result>, BootstrapError> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; - match listener_tx.send((msg, addr)) { - Ok(_) => continue, - Err(SendError((dplx, remote_addr))) => { - warn!( - "listener channel disconnected after accepting connection from {}", - remote_addr - ); - return Ok(Err((dplx, remote_addr))); - } + + if let Err(SendError((dplx, remote_addr))) = listener_tx.send((msg, addr)) { + warn!( + "listener channel disconnected after accepting connection from {}", + remote_addr + ); + return Ok(Err((dplx, remote_addr))); }; } } @@ -262,7 +260,7 @@ impl BootstrapServer<'_, D, C> loop { // block until we have a connection to work with, or break out of main-loop // if a stop-signal is received - let Some((dplx, remote_addr)) = self.receive_connection(&mut selector).map_err(BootstrapError::GeneralError)? else { break; }; + let Some((dplx, remote_addr)) = self.receive_connection(&mut selector)? else { break; }; // claim a slot in the max_bootstrap_sessions let server_binding = BootstrapServerBinder::new( dplx, @@ -375,7 +373,10 @@ impl BootstrapServer<'_, D, C> /// - 3.a. double check the stop-signal is absent /// - 3.b. If present, fall-back to the stop behaviour /// - 3.c. If absent, all's clear to rock-n-roll. - fn receive_connection(&self, selector: &mut Select) -> Result>, String> { + fn receive_connection( + &self, + selector: &mut Select, + ) -> Result>, BootstrapError> { // 1. Block until _something_ is ready let rdy = selector.ready(); @@ -396,7 +397,9 @@ impl BootstrapServer<'_, D, C> // 3.b. If present, fall-back to the stop behaviour return Ok(None); } else if unlikely(stop == Err(crossbeam::channel::TryRecvError::Disconnected)) { - return Err("Unexpected stop-channel disconnection".to_string()); + return Err(BootstrapError::GeneralError( + "Unexpected stop-channel disconnection".to_string(), + )); }; } // - 3.c. If absent, all's clear to rock-n-roll. @@ -405,7 +408,9 @@ impl BootstrapServer<'_, D, C> Err(try_rcv_err) => match try_rcv_err { crossbeam::channel::TryRecvError::Empty => return Ok(None), crossbeam::channel::TryRecvError::Disconnected => { - return Err("listener recv channel disconnected unexpectedly".to_string()); + return Err(BootstrapError::GeneralError( + "listener recv channel disconnected unexpectedly".to_string(), + )); } }, }; @@ -603,16 +608,7 @@ pub async fn stream_bootstrap_information( } if slot_too_old { - match server.send_msg(write_timeout, BootstrapServerMessage::SlotTooOld) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "SlotTooOld message send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; - return Ok(()); + return server.send_msg(write_timeout, BootstrapServerMessage::SlotTooOld); } // Setup final state global cursor @@ -665,20 +661,12 @@ pub async fn stream_bootstrap_information( && final_state_changes_step.finished() && last_consensus_step.finished() { - match server.send_msg(write_timeout, BootstrapServerMessage::BootstrapFinished) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + server.send_msg(write_timeout, BootstrapServerMessage::BootstrapFinished)?; break; } // At this point we know that consensus, final state or both are not finished - match server.send_msg( + server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPart { slot: current_slot, @@ -691,15 +679,7 @@ pub async fn stream_bootstrap_information( consensus_part, consensus_outdated_ids, }, - ) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap ask ledger part send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + )?; } Ok(()) } @@ -716,17 +696,7 @@ async fn manage_bootstrap( massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - match server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into())) { - Err(BootstrapError::TimedOut(_)) => { - return Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap handshake send timed out", - ) - .into()); - } - Err(e) => return Err(e), - Ok(_) => (), - }; + server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))?; match server.next_timeout(Some(read_error_timeout)) { Err(BootstrapError::TimedOut(_)) => {} @@ -742,21 +712,13 @@ async fn manage_bootstrap( // Sync clocks. let server_time = MassaTime::now()?; - match server.send_msg( + server.send_msg( write_timeout, BootstrapServerMessage::BootstrapTime { server_time, version, }, - ) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap clock send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + )?; loop { match server.next_timeout(Some(bootstrap_config.read_timeout.into())) { @@ -764,20 +726,12 @@ async fn manage_bootstrap( Err(e) => break Err(e), Ok(msg) => match msg { BootstrapClientMessage::AskBootstrapPeers => { - match server.send_msg( + server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { peers: network_command_sender.get_bootstrap_peers().await?, }, - ) { - Err(BootstrapError::TimedOut(_)) => Err(std::io::Error::new( - std::io::ErrorKind::TimedOut, - "bootstrap peers send timed out", - ) - .into()), - Err(e) => Err(e), - Ok(_) => Ok(()), - }?; + )?; } BootstrapClientMessage::AskBootstrapPart { last_slot, diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 606ba7741be..c5da9603d54 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -108,7 +108,19 @@ impl BootstrapServerBinder { timeout: Duration, msg: BootstrapServerMessage, ) -> Result<(), BootstrapError> { - self.send_timeout(msg, Some(timeout)) + let to_str = msg.to_string(); + self.send_timeout(msg, Some(timeout)).map_err(|e| match e { + BootstrapError::IoError(e) + // On some systems, a timed out send returns WouldBlock + if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock => + { + BootstrapError::TimedOut(std::io::Error::new( + std::io::ErrorKind::TimedOut, + format!("BootstrapServerMessage::{} send timed out", to_str), + )) + } + _ => e, + }) } /// 1. Spawns a thread @@ -148,7 +160,7 @@ impl BootstrapServerBinder { ) .map_err(|e| match e { BootstrapError::IoError(e) if e.kind() == ErrorKind::WouldBlock => { - BootstrapError::IoError(ErrorKind::TimedOut.into()) + BootstrapError::TimedOut(ErrorKind::TimedOut.into()) } e => e, }) From 3b299cfe927a2f393c462dcf0df6adcbf734da37 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 4 Apr 2023 13:20:29 +0200 Subject: [PATCH 65/83] Remove the Duplex trait. In the future, might need to be re-introduced if we wish to mock the data being sent/received --- massa-bootstrap/src/client.rs | 16 ++++----- massa-bootstrap/src/client_binder.rs | 11 +++--- massa-bootstrap/src/establisher.rs | 25 ------------- massa-bootstrap/src/server.rs | 50 ++++++++++++-------------- massa-bootstrap/src/server_binder.rs | 13 ++++--- massa-bootstrap/src/tests/binders.rs | 6 ++-- massa-bootstrap/src/tests/scenarios.rs | 4 +-- 7 files changed, 48 insertions(+), 77 deletions(-) diff --git a/massa-bootstrap/src/client.rs b/massa-bootstrap/src/client.rs index b085cf8c5c7..11fbfb5a088 100644 --- a/massa-bootstrap/src/client.rs +++ b/massa-bootstrap/src/client.rs @@ -16,7 +16,7 @@ use tracing::{debug, info, warn}; use crate::{ client_binder::BootstrapClientBinder, error::BootstrapError, - establisher::{BSConnector, Duplex}, + establisher::BSConnector, messages::{BootstrapClientMessage, BootstrapServerMessage}, settings::IpType, BootstrapConfig, GlobalBootstrapState, @@ -25,9 +25,9 @@ use crate::{ /// This function will send the starting point to receive a stream of the ledger and will receive and process each part until receive a `BootstrapServerMessage::FinalStateFinished` message from the server. /// `next_bootstrap_message` passed as parameter must be `BootstrapClientMessage::AskFinalStatePart` enum variant. /// `next_bootstrap_message` will be updated after receiving each part so that in case of connection lost we can restart from the last message we processed. -fn stream_final_state_and_consensus( +fn stream_final_state_and_consensus( cfg: &BootstrapConfig, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, global_bootstrap_state: &mut GlobalBootstrapState, ) -> Result<(), BootstrapError> { @@ -172,9 +172,9 @@ fn stream_final_state_and_consensus( /// Gets the state from a bootstrap server (internal private function) /// needs to be CANCELLABLE -fn bootstrap_from_server( +fn bootstrap_from_server( cfg: &BootstrapConfig, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, next_bootstrap_message: &mut BootstrapClientMessage, global_bootstrap_state: &mut GlobalBootstrapState, our_version: Version, @@ -301,9 +301,9 @@ fn bootstrap_from_server( Ok(()) } -fn send_client_message( +fn send_client_message( message_to_send: &BootstrapClientMessage, - client: &mut BootstrapClientBinder, + client: &mut BootstrapClientBinder, write_timeout: Duration, read_timeout: Duration, error: &str, @@ -325,7 +325,7 @@ fn connect_to_server( bootstrap_config: &BootstrapConfig, addr: &SocketAddr, pub_key: &PublicKey, -) -> Result, BootstrapError> { +) -> Result { let socket = connector.connect_timeout(*addr, Some(bootstrap_config.connect_timeout))?; socket.set_nonblocking(false)?; Ok(BootstrapClientBinder::new( diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 6e432bdab92..976ab527a7d 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -1,9 +1,10 @@ // Copyright (c) 2022 MASSA LABS +use std::io::{Read, Write}; +use std::net::TcpStream; use std::time::Duration; use crate::error::BootstrapError; -use crate::establisher::Duplex; use crate::messages::{ BootstrapClientMessage, BootstrapClientMessageSerializer, BootstrapServerMessage, BootstrapServerMessageDeserializer, @@ -17,24 +18,24 @@ use massa_signature::{PublicKey, Signature, SIGNATURE_SIZE_BYTES}; use rand::{rngs::StdRng, RngCore, SeedableRng}; /// Bootstrap client binder -pub struct BootstrapClientBinder { +pub struct BootstrapClientBinder { // max_bootstrap_message_size: u32, size_field_len: usize, remote_pubkey: PublicKey, - duplex: D, + duplex: TcpStream, prev_message: Option, version_serializer: VersionSerializer, cfg: BootstrapClientConfig, } -impl BootstrapClientBinder { +impl BootstrapClientBinder { /// Creates a new `WriteBinder`. /// /// # Argument /// * duplex: duplex stream. /// * limit: limit max bytes per second (up and down) #[allow(clippy::too_many_arguments)] - pub fn new(duplex: D, remote_pubkey: PublicKey, cfg: BootstrapClientConfig) -> Self { + pub fn new(duplex: TcpStream, remote_pubkey: PublicKey, cfg: BootstrapClientConfig) -> Self { let size_field_len = u32::be_bytes_min_length(cfg.max_bootstrap_message_size); BootstrapClientBinder { size_field_len, diff --git a/massa-bootstrap/src/establisher.rs b/massa-bootstrap/src/establisher.rs index ee3cb6b6616..b2ee316adcf 100644 --- a/massa-bootstrap/src/establisher.rs +++ b/massa-bootstrap/src/establisher.rs @@ -3,33 +3,8 @@ use massa_time::MassaTime; use std::{ io, net::{SocketAddr, TcpListener, TcpStream}, - time::Duration, }; -/// duplex connection -pub trait Duplex: -// static because need to send between threads :( - 'static + Send + io::Read + io::Write -{ - fn set_read_timeout(&mut self, duration: Option) -> io::Result<()>; - fn set_write_timeout(&mut self, duration: Option) -> io::Result<()>; - fn set_nonblocking(&mut self, setting: bool) -> io::Result<()>; -} - -impl Duplex for std::net::TcpStream { - fn set_read_timeout(&mut self, duration: Option) -> io::Result<()> { - Self::set_read_timeout(self, duration) - } - - fn set_write_timeout(&mut self, duration: Option) -> io::Result<()> { - Self::set_write_timeout(self, duration) - } - - fn set_nonblocking(&mut self, setting: bool) -> io::Result<()> { - Self::set_nonblocking(self, setting) - } -} - /// Specifies a common interface that can be used by standard, or mockers #[cfg_attr(test, mockall::automock)] pub trait BSListener { diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 3c66d01b2ff..ccf07945a33 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -55,7 +55,7 @@ use tracing::{debug, error, info, warn}; use crate::{ error::BootstrapError, - establisher::{BSListener, Duplex}, + establisher::BSListener, messages::{BootstrapClientMessage, BootstrapServerMessage}, server_binder::BootstrapServerBinder, BootstrapConfig, @@ -63,20 +63,20 @@ use crate::{ /// Abstraction layer over data produced by the listener, and transported /// over to the worker via a channel -type BsConn = (D, SocketAddr); +type BsConn = (TcpStream, SocketAddr); /// handle on the bootstrap server -pub struct BootstrapManager { +pub struct BootstrapManager { update_handle: thread::JoinHandle>, // need to preserve the listener handle up to here to prevent it being destroyed #[allow(clippy::type_complexity)] - _listen_handle: thread::JoinHandle>, BootstrapError>>, + _listen_handle: thread::JoinHandle, BootstrapError>>, main_handle: thread::JoinHandle>, listen_stopper_tx: crossbeam::channel::Sender<()>, update_stopper_tx: crossbeam::channel::Sender<()>, } -impl BootstrapManager { +impl BootstrapManager { /// stop the bootstrap server pub fn stop(self) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.stop", {}); @@ -101,7 +101,7 @@ impl BootstrapManager { } /// See module level documentation for details -pub fn start_bootstrap_server( +pub fn start_bootstrap_server( consensus_controller: Box, network_command_sender: C, final_state: Arc>, @@ -109,7 +109,7 @@ pub fn start_bootstrap_server( listener: impl BSListener + Send + 'static, keypair: KeyPair, version: Version, -) -> Result>, BootstrapError> { +) -> Result, BootstrapError> { massa_trace!("bootstrap.lib.start_bootstrap_server", {}); // TODO(low prio): See if a zero capacity channel model can work @@ -121,8 +121,7 @@ pub fn start_bootstrap_server( }; // This is needed for now, as there is no ergonomic to "select!" on both a channel and a blocking std::net::TcpStream - let (listener_tx, listener_rx) = - crossbeam::channel::bounded::>(max_bootstraps * 2); + let (listener_tx, listener_rx) = crossbeam::channel::bounded::(max_bootstraps * 2); let white_black_list = SharedWhiteBlackList::new( config.bootstrap_whitelist_path.clone(), @@ -133,7 +132,7 @@ pub fn start_bootstrap_server( let update_handle = thread::Builder::new() .name("wb_list_updater".to_string()) .spawn(move || { - let res = BootstrapServer::::run_updater( + let res = BootstrapServer::::run_updater( updater_lists, config.cache_duration.into(), update_stopper_rx, @@ -149,7 +148,7 @@ pub fn start_bootstrap_server( .name("bs_listener".to_string()) // FIXME: The interface being used shouldn't have `: Send + 'static` as a constraint on the listener assosciated type. // GAT lifetime is likely to remedy this, however. - .spawn(move || BootstrapServer::::run_listener(listener, listener_tx)) + .spawn(move || BootstrapServer::::run_listener(listener, listener_tx)) .expect("in `start_bootstrap_server`, OS failed to spawn listener thread"); let main_handle = thread::Builder::new() @@ -181,11 +180,11 @@ pub fn start_bootstrap_server( })) } -struct BootstrapServer<'a, D: Duplex, C: NetworkCommandSenderTrait> { +struct BootstrapServer<'a, C: NetworkCommandSenderTrait> { consensus_controller: Box, network_command_sender: C, final_state: Arc>, - listener_rx: crossbeam::channel::Receiver>, + listener_rx: crossbeam::channel::Receiver, listen_stopper_rx: crossbeam::channel::Receiver<()>, white_black_list: SharedWhiteBlackList<'a>, keypair: KeyPair, @@ -194,7 +193,7 @@ struct BootstrapServer<'a, D: Duplex, C: NetworkCommandSenderTrait> { ip_hist_map: HashMap, } -impl BootstrapServer<'_, D, C> { +impl BootstrapServer<'_, C> { fn run_updater( mut list: SharedWhiteBlackList<'_>, interval: Duration, @@ -224,8 +223,8 @@ impl BootstrapServer<'_, D, C> /// TODO: Integrate the listener into the bootstrap-main-loop fn run_listener( mut listener: impl BSListener, - listener_tx: crossbeam::channel::Sender>, - ) -> Result>, BootstrapError> { + listener_tx: crossbeam::channel::Sender, + ) -> Result, BootstrapError> { loop { let (msg, addr) = listener.accept().map_err(BootstrapError::IoError)?; @@ -298,7 +297,7 @@ impl BootstrapServer<'_, D, C> } // check IP's bootstrap attempt history - if let Err(msg) = BootstrapServer::::greedy_client_check( + if let Err(msg) = BootstrapServer::::greedy_client_check( &mut self.ip_hist_map, remote_addr, now, @@ -373,10 +372,7 @@ impl BootstrapServer<'_, D, C> /// - 3.a. double check the stop-signal is absent /// - 3.b. If present, fall-back to the stop behaviour /// - 3.c. If absent, all's clear to rock-n-roll. - fn receive_connection( - &self, - selector: &mut Select, - ) -> Result>, BootstrapError> { + fn receive_connection(&self, selector: &mut Select) -> Result, BootstrapError> { // 1. Block until _something_ is ready let rdy = selector.ready(); @@ -454,8 +450,8 @@ impl BootstrapServer<'_, D, C> /// The arc_counter variable is used as a proxy to keep track the number of active bootstrap /// sessions. #[allow(clippy::too_many_arguments)] -fn run_bootstrap_session( - mut server: BootstrapServerBinder, +fn run_bootstrap_session( + mut server: BootstrapServerBinder, arc_counter: Arc<()>, config: BootstrapConfig, remote_addr: SocketAddr, @@ -515,8 +511,8 @@ fn run_bootstrap_session( } #[allow(clippy::too_many_arguments)] -pub async fn stream_bootstrap_information( - server: &mut BootstrapServerBinder, +pub async fn stream_bootstrap_information( + server: &mut BootstrapServerBinder, final_state: Arc>, consensus_controller: Box, mut last_slot: Option, @@ -685,9 +681,9 @@ pub async fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -async fn manage_bootstrap( +async fn manage_bootstrap( bootstrap_config: &BootstrapConfig, - server: &mut BootstrapServerBinder, + server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index c5da9603d54..524766a987f 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -1,7 +1,6 @@ // Copyright (c) 2022 MASSA LABS use crate::error::BootstrapError; -use crate::establisher::Duplex; use crate::messages::{ BootstrapClientMessage, BootstrapClientMessageDeserializer, BootstrapServerMessage, BootstrapServerMessageSerializer, @@ -15,14 +14,14 @@ use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::KeyPair; use massa_time::MassaTime; use std::convert::TryInto; -use std::io::ErrorKind; -use std::net::SocketAddr; +use std::io::{ErrorKind, Read, Write}; +use std::net::{SocketAddr, TcpStream}; use std::thread; use std::time::Duration; use tracing::error; /// Bootstrap server binder -pub struct BootstrapServerBinder { +pub struct BootstrapServerBinder { max_bootstrap_message_size: u32, max_consensus_block_ids: u64, thread_count: u8, @@ -30,14 +29,14 @@ pub struct BootstrapServerBinder { randomness_size_bytes: usize, size_field_len: usize, local_keypair: KeyPair, - duplex: D, + duplex: TcpStream, prev_message: Option, version_serializer: VersionSerializer, version_deserializer: VersionDeserializer, write_error_timeout: MassaTime, } -impl BootstrapServerBinder { +impl BootstrapServerBinder { /// Creates a new `WriteBinder`. /// /// # Argument @@ -45,7 +44,7 @@ impl BootstrapServerBinder { /// * `local_keypair`: local node user keypair /// * `limit`: limit max bytes per second (up and down) #[allow(clippy::too_many_arguments)] - pub fn new(duplex: D, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { + pub fn new(duplex: TcpStream, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { let BootstrapSrvBindCfg { max_bytes_read_write: _limit, max_bootstrap_message_size, diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 4cf4384546d..a4b34b43315 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -1,4 +1,3 @@ -use crate::establisher::Duplex; use crate::messages::{BootstrapClientMessage, BootstrapServerMessage}; use crate::settings::{BootstrapClientConfig, BootstrapSrvBindCfg}; use crate::BootstrapConfig; @@ -19,6 +18,7 @@ use massa_models::node::NodeId; use massa_models::version::Version; use massa_signature::{KeyPair, PublicKey}; use massa_time::MassaTime; +use std::net::TcpStream; use std::str::FromStr; lazy_static::lazy_static! { @@ -28,8 +28,8 @@ lazy_static::lazy_static! { }; } -impl BootstrapClientBinder { - pub fn test_default(client_duplex: D, remote_pubkey: PublicKey) -> Self { +impl BootstrapClientBinder { + pub fn test_default(client_duplex: TcpStream, remote_pubkey: PublicKey) -> Self { let cfg = BootstrapClientConfig { max_bytes_read_write: f64::INFINITY, max_bootstrap_message_size: MAX_BOOTSTRAP_MESSAGE_SIZE, diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 0fd597c65b4..f66d05297ca 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -41,7 +41,7 @@ use massa_pos_worker::start_selector_worker; use massa_signature::KeyPair; use massa_time::MassaTime; use parking_lot::RwLock; -use std::{net::TcpStream, path::PathBuf, str::FromStr, sync::Arc, time::Duration}; +use std::{path::PathBuf, str::FromStr, sync::Arc, time::Duration}; use tempfile::TempDir; lazy_static::lazy_static! { @@ -185,7 +185,7 @@ fn test_bootstrap_server() { let bootstrap_manager_thread = std::thread::Builder::new() .name("bootstrap_thread".to_string()) .spawn(move || { - start_bootstrap_server::( + start_bootstrap_server::( stream_mock1, mocked1, final_state_server_clone1, From 805ce3660889c949afd2bbc04b7e29b986ea4116 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 4 Apr 2023 16:40:54 +0200 Subject: [PATCH 66/83] Fix Duplex deletion commit --- massa-node/src/main.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/massa-node/src/main.rs b/massa-node/src/main.rs index b397c8d5ea9..433cd5992c9 100644 --- a/massa-node/src/main.rs +++ b/massa-node/src/main.rs @@ -54,7 +54,7 @@ use massa_models::config::{ CONSENSUS_BOOTSTRAP_PART_SIZE, DENUNCIATION_EXPIRE_PERIODS, DENUNCIATION_ITEMS_MAX_CYCLE_DELTA, }; use massa_models::denunciation::DenunciationPrecursor; -use massa_network_exports::{Establisher, NetworkConfig, NetworkManager}; +use massa_network_exports::{Establisher, NetworkCommandSender, NetworkConfig, NetworkManager}; use massa_network_worker::start_network_controller; use massa_pool_exports::{PoolChannels, PoolConfig, PoolManager}; use massa_pool_worker::start_pool_controller; @@ -70,7 +70,6 @@ use massa_time::MassaTime; use massa_versioning_worker::versioning::{MipStatsConfig, MipStore}; use massa_wallet::Wallet; use parking_lot::RwLock; -use std::net::TcpStream; use std::path::PathBuf; use std::sync::atomic::{AtomicUsize, Ordering}; use std::thread::sleep; @@ -89,7 +88,7 @@ async fn launch( node_wallet: Arc>, ) -> ( Receiver, - Option>, + Option, Box, Box, Box, @@ -547,7 +546,7 @@ async fn launch( // launch bootstrap server // TODO: use std::net::TcpStream let bootstrap_manager = match bootstrap_config.listen_addr { - Some(addr) => start_bootstrap_server::( + Some(addr) => start_bootstrap_server::( consensus_controller.clone(), network_command_sender.clone(), final_state.clone(), @@ -687,7 +686,7 @@ async fn launch( } struct Managers { - bootstrap_manager: Option>, + bootstrap_manager: Option, consensus_manager: Box, execution_manager: Box, selector_manager: Box, From 62e1781df7634ebd238b14690348760de83b3075 Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 7 Apr 2023 19:11:14 +0200 Subject: [PATCH 67/83] Removes the `tokio` dependency from the bootstrap module (#3782) --- Cargo.lock | 1 - massa-bootstrap/Cargo.toml | 1 - massa-bootstrap/src/error.rs | 2 - massa-bootstrap/src/server.rs | 117 ++++++++---------- massa-bootstrap/src/tests/scenarios.rs | 3 +- massa-network-exports/src/lib.rs | 2 +- .../src/network_controller.rs | 10 ++ 7 files changed, 62 insertions(+), 74 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9b2d7f8f78..a75450a5664 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2501,7 +2501,6 @@ dependencies = [ "substruct", "tempfile", "thiserror", - "tokio", "tracing", ] diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index f73cba4def6..a5821e1f1c3 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -16,7 +16,6 @@ serde_json = "1.0" humantime = "2.1.0" thiserror = "1.0" parking_lot = { version = "0.12", features = ["deadlock_detection"] } -tokio = { version = "1.23", features = ["full"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index ae10c6e562b..0a896d83712 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -48,8 +48,6 @@ pub enum BootstrapError { FinalStateError(#[from] FinalStateError), /// Proof-of-Stake error: {0} PoSError(#[from] PosError), - /// join error: {0} - JoinError(#[from] tokio::task::JoinError), /// missing keypair file MissingKeyError, /// incompatible version: {0} diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 3efcc7be945..ad1c229ac87 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -50,7 +50,6 @@ use std::{ thread, time::{Duration, Instant}, }; -use tokio::runtime::{self, Handle}; use tracing::{debug, error, info, warn}; use white_black_list::*; @@ -244,16 +243,6 @@ impl BootstrapServer<'_, C> { } fn run_loop(mut self, max_bootstraps: usize) -> Result<(), BootstrapError> { - let Ok(bs_loop_rt) = runtime::Builder::new_multi_thread() - .max_blocking_threads(max_bootstraps * 2) - .enable_io() - .enable_time() - .thread_name("bootstrap-main-loop-worker") - .thread_keep_alive(Duration::from_millis(u64::MAX)) - .build() else { - return Err(BootstrapError::GeneralError("Failed to create bootstrap main-loop runtime".to_string())); - }; - // Use the strong-count of this variable to track the session count let bootstrap_sessions_counter: Arc<()> = Arc::new(()); let per_ip_min_interval = self.bootstrap_config.per_ip_min_interval.to_duration(); @@ -334,7 +323,6 @@ impl BootstrapServer<'_, C> { let config = self.bootstrap_config.clone(); let bootstrap_count_token = bootstrap_sessions_counter.clone(); - let session_handle = bs_loop_rt.handle().clone(); let mip_store = self.mip_store.clone(); let _ = thread::Builder::new() @@ -349,7 +337,6 @@ impl BootstrapServer<'_, C> { version, consensus_command_sender, network_command_sender, - session_handle, mip_store, ) }); @@ -367,7 +354,6 @@ impl BootstrapServer<'_, C> { } // Give any remaining processes 20 seconds to clean up, otherwise force them to shutdown - bs_loop_rt.shutdown_timeout(Duration::from_secs(20)); Ok(()) } @@ -466,61 +452,56 @@ fn run_bootstrap_session( version: Version, consensus_command_sender: Box, network_command_sender: C, - bs_loop_rt_handle: Handle, mip_store: MipStore, ) { debug!("running bootstrap for peer {}", remote_addr); - bs_loop_rt_handle.block_on(async move { - let res = tokio::time::timeout( - config.bootstrap_timeout.into(), - manage_bootstrap( - &config, - &mut server, - data_execution, - version, - consensus_command_sender, - network_command_sender, - mip_store, - ), - ) - .await; - // This drop allows the server to accept new connections before having to complete the error notifications - // account for this session being finished, as well as the root-instance - massa_trace!("bootstrap.session.finished", { - "sessions_remaining": Arc::strong_count(&arc_counter) - 2 - }); - drop(arc_counter); - match res { - Ok(mgmt) => match mgmt { - Ok(_) => { - info!("bootstrapped peer {}", remote_addr); - } - Err(BootstrapError::ReceivedError(error)) => debug!( - "bootstrap serving error received from peer {}: {}", - remote_addr, error - ), - Err(err) => { - debug!("bootstrap serving error for peer {}: {}", remote_addr, err); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(err.to_string()); - } - }, - Err(_timeout) => { - debug!("bootstrap timeout for peer {}", remote_addr); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(format!( - "Bootstrap process timedout ({})", - format_duration(config.bootstrap_timeout.to_duration()) - )); - } - } + let deadline = Instant::now() + config.bootstrap_timeout.to_duration(); + // TODO: reinstate prevention of bootstrap slot camping. Deadline cancellation is one option + let res = manage_bootstrap( + &config, + &mut server, + data_execution, + version, + consensus_command_sender, + network_command_sender, + deadline, + mip_store, + ); + // TODO: handle the deadline management + // This drop allows the server to accept new connections before having to complete the error notifications + // account for this session being finished, as well as the root-instance + massa_trace!("bootstrap.session.finished", { + "sessions_remaining": Arc::strong_count(&arc_counter) - 2 }); + drop(arc_counter); + match res { + Err(BootstrapError::TimedOut(_)) => { + debug!("bootstrap timeout for peer {}", remote_addr); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(format!( + "Bootstrap process timedout ({})", + format_duration(config.bootstrap_timeout.to_duration()) + )); + } + Err(BootstrapError::ReceivedError(error)) => debug!( + "bootstrap serving error received from peer {}: {}", + remote_addr, error + ), + Err(err) => { + debug!("bootstrap serving error for peer {}: {}", remote_addr, err); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(err.to_string()); + } + Ok(_) => { + info!("bootstrapped peer {}", remote_addr); + } + } } #[allow(clippy::too_many_arguments)] -pub async fn stream_bootstrap_information( +pub fn stream_bootstrap_information( server: &mut BootstrapServerBinder, final_state: Arc>, consensus_controller: Box, @@ -538,7 +519,7 @@ pub async fn stream_bootstrap_information( #[cfg(test)] { // Necessary for test_bootstrap_server in tests/scenarios.rs - tokio::time::sleep(Duration::from_millis(500)).await; + std::thread::sleep(Duration::from_millis(500)); } let current_slot; @@ -701,17 +682,19 @@ pub async fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -async fn manage_bootstrap( +fn manage_bootstrap( bootstrap_config: &BootstrapConfig, server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, network_command_sender: C, + _deadline: Instant, mip_store: MipStore, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); + let rt_hack = massa_network_exports::make_runtime(); server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))?; @@ -746,7 +729,8 @@ async fn manage_bootstrap( server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { - peers: network_command_sender.get_bootstrap_peers().await?, + peers: rt_hack + .block_on(network_command_sender.get_bootstrap_peers())?, }, )?; } @@ -773,8 +757,7 @@ async fn manage_bootstrap( last_consensus_step, send_last_start_period, write_timeout, - ) - .await?; + )?; } BootstrapClientMessage::AskBootstrapMipStore => { let vs = mip_store.0.read().to_owned(); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index a4d08717130..37cd5b7d0a1 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -273,8 +273,7 @@ fn test_bootstrap_server() { // .times(1) // .returning(|_, _| todo!()); // launch the get_state process - let bootstrap_res = tokio::runtime::Runtime::new() - .unwrap() + let bootstrap_res = massa_network_exports::make_runtime() .block_on(get_state( bootstrap_config, final_state_client_clone, diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index 452d1f9d26c..9af80b1b1a2 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -13,7 +13,7 @@ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; pub use network_controller::{ - MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, + make_runtime, MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, }; pub use peers::{ diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index db8bb7a26f4..a9cd2326cf5 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -450,3 +450,13 @@ impl NetworkManager { Ok(()) } } + +/// Used by the bootstrap server to run async tasks, allowing the bootstrap module to +/// remove the tokio dependency. +pub fn make_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("network-provided-runtime") + .build() + .expect("failed to create runtime") +} From d92ce878e331487488b8aee4cd27de2a6be6709f Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 7 Apr 2023 19:13:58 +0200 Subject: [PATCH 68/83] Revert "Removes the `tokio` dependency from the bootstrap module (#3782)" (#3795) This reverts commit 62e1781df7634ebd238b14690348760de83b3075. --- Cargo.lock | 1 + massa-bootstrap/Cargo.toml | 1 + massa-bootstrap/src/error.rs | 2 + massa-bootstrap/src/server.rs | 117 ++++++++++-------- massa-bootstrap/src/tests/scenarios.rs | 3 +- massa-network-exports/src/lib.rs | 2 +- .../src/network_controller.rs | 10 -- 7 files changed, 74 insertions(+), 62 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a75450a5664..d9b2d7f8f78 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2501,6 +2501,7 @@ dependencies = [ "substruct", "tempfile", "thiserror", + "tokio", "tracing", ] diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index a5821e1f1c3..f73cba4def6 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -16,6 +16,7 @@ serde_json = "1.0" humantime = "2.1.0" thiserror = "1.0" parking_lot = { version = "0.12", features = ["deadlock_detection"] } +tokio = { version = "1.23", features = ["full"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index 0a896d83712..ae10c6e562b 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -48,6 +48,8 @@ pub enum BootstrapError { FinalStateError(#[from] FinalStateError), /// Proof-of-Stake error: {0} PoSError(#[from] PosError), + /// join error: {0} + JoinError(#[from] tokio::task::JoinError), /// missing keypair file MissingKeyError, /// incompatible version: {0} diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index ad1c229ac87..3efcc7be945 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -50,6 +50,7 @@ use std::{ thread, time::{Duration, Instant}, }; +use tokio::runtime::{self, Handle}; use tracing::{debug, error, info, warn}; use white_black_list::*; @@ -243,6 +244,16 @@ impl BootstrapServer<'_, C> { } fn run_loop(mut self, max_bootstraps: usize) -> Result<(), BootstrapError> { + let Ok(bs_loop_rt) = runtime::Builder::new_multi_thread() + .max_blocking_threads(max_bootstraps * 2) + .enable_io() + .enable_time() + .thread_name("bootstrap-main-loop-worker") + .thread_keep_alive(Duration::from_millis(u64::MAX)) + .build() else { + return Err(BootstrapError::GeneralError("Failed to create bootstrap main-loop runtime".to_string())); + }; + // Use the strong-count of this variable to track the session count let bootstrap_sessions_counter: Arc<()> = Arc::new(()); let per_ip_min_interval = self.bootstrap_config.per_ip_min_interval.to_duration(); @@ -323,6 +334,7 @@ impl BootstrapServer<'_, C> { let config = self.bootstrap_config.clone(); let bootstrap_count_token = bootstrap_sessions_counter.clone(); + let session_handle = bs_loop_rt.handle().clone(); let mip_store = self.mip_store.clone(); let _ = thread::Builder::new() @@ -337,6 +349,7 @@ impl BootstrapServer<'_, C> { version, consensus_command_sender, network_command_sender, + session_handle, mip_store, ) }); @@ -354,6 +367,7 @@ impl BootstrapServer<'_, C> { } // Give any remaining processes 20 seconds to clean up, otherwise force them to shutdown + bs_loop_rt.shutdown_timeout(Duration::from_secs(20)); Ok(()) } @@ -452,56 +466,61 @@ fn run_bootstrap_session( version: Version, consensus_command_sender: Box, network_command_sender: C, + bs_loop_rt_handle: Handle, mip_store: MipStore, ) { debug!("running bootstrap for peer {}", remote_addr); - let deadline = Instant::now() + config.bootstrap_timeout.to_duration(); - // TODO: reinstate prevention of bootstrap slot camping. Deadline cancellation is one option - let res = manage_bootstrap( - &config, - &mut server, - data_execution, - version, - consensus_command_sender, - network_command_sender, - deadline, - mip_store, - ); - // TODO: handle the deadline management - // This drop allows the server to accept new connections before having to complete the error notifications - // account for this session being finished, as well as the root-instance - massa_trace!("bootstrap.session.finished", { - "sessions_remaining": Arc::strong_count(&arc_counter) - 2 - }); - drop(arc_counter); - match res { - Err(BootstrapError::TimedOut(_)) => { - debug!("bootstrap timeout for peer {}", remote_addr); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(format!( - "Bootstrap process timedout ({})", - format_duration(config.bootstrap_timeout.to_duration()) - )); - } - Err(BootstrapError::ReceivedError(error)) => debug!( - "bootstrap serving error received from peer {}: {}", - remote_addr, error - ), - Err(err) => { - debug!("bootstrap serving error for peer {}: {}", remote_addr, err); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(err.to_string()); - } - Ok(_) => { - info!("bootstrapped peer {}", remote_addr); + bs_loop_rt_handle.block_on(async move { + let res = tokio::time::timeout( + config.bootstrap_timeout.into(), + manage_bootstrap( + &config, + &mut server, + data_execution, + version, + consensus_command_sender, + network_command_sender, + mip_store, + ), + ) + .await; + // This drop allows the server to accept new connections before having to complete the error notifications + // account for this session being finished, as well as the root-instance + massa_trace!("bootstrap.session.finished", { + "sessions_remaining": Arc::strong_count(&arc_counter) - 2 + }); + drop(arc_counter); + match res { + Ok(mgmt) => match mgmt { + Ok(_) => { + info!("bootstrapped peer {}", remote_addr); + } + Err(BootstrapError::ReceivedError(error)) => debug!( + "bootstrap serving error received from peer {}: {}", + remote_addr, error + ), + Err(err) => { + debug!("bootstrap serving error for peer {}: {}", remote_addr, err); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(err.to_string()); + } + }, + Err(_timeout) => { + debug!("bootstrap timeout for peer {}", remote_addr); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(format!( + "Bootstrap process timedout ({})", + format_duration(config.bootstrap_timeout.to_duration()) + )); + } } - } + }); } #[allow(clippy::too_many_arguments)] -pub fn stream_bootstrap_information( +pub async fn stream_bootstrap_information( server: &mut BootstrapServerBinder, final_state: Arc>, consensus_controller: Box, @@ -519,7 +538,7 @@ pub fn stream_bootstrap_information( #[cfg(test)] { // Necessary for test_bootstrap_server in tests/scenarios.rs - std::thread::sleep(Duration::from_millis(500)); + tokio::time::sleep(Duration::from_millis(500)).await; } let current_slot; @@ -682,19 +701,17 @@ pub fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -fn manage_bootstrap( +async fn manage_bootstrap( bootstrap_config: &BootstrapConfig, server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, network_command_sender: C, - _deadline: Instant, mip_store: MipStore, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - let rt_hack = massa_network_exports::make_runtime(); server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))?; @@ -729,8 +746,7 @@ fn manage_bootstrap( server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { - peers: rt_hack - .block_on(network_command_sender.get_bootstrap_peers())?, + peers: network_command_sender.get_bootstrap_peers().await?, }, )?; } @@ -757,7 +773,8 @@ fn manage_bootstrap( last_consensus_step, send_last_start_period, write_timeout, - )?; + ) + .await?; } BootstrapClientMessage::AskBootstrapMipStore => { let vs = mip_store.0.read().to_owned(); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 37cd5b7d0a1..a4d08717130 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -273,7 +273,8 @@ fn test_bootstrap_server() { // .times(1) // .returning(|_, _| todo!()); // launch the get_state process - let bootstrap_res = massa_network_exports::make_runtime() + let bootstrap_res = tokio::runtime::Runtime::new() + .unwrap() .block_on(get_state( bootstrap_config, final_state_client_clone, diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index 9af80b1b1a2..452d1f9d26c 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -13,7 +13,7 @@ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; pub use network_controller::{ - make_runtime, MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, + MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, }; pub use peers::{ diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index a9cd2326cf5..db8bb7a26f4 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -450,13 +450,3 @@ impl NetworkManager { Ok(()) } } - -/// Used by the bootstrap server to run async tasks, allowing the bootstrap module to -/// remove the tokio dependency. -pub fn make_runtime() -> tokio::runtime::Runtime { - tokio::runtime::Builder::new_multi_thread() - .enable_all() - .thread_name("network-provided-runtime") - .build() - .expect("failed to create runtime") -} From 81c1f4129f10ffef1a1fd2a506e410da96938d37 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 11 Apr 2023 18:16:29 +0200 Subject: [PATCH 69/83] Apply self-review changes --- Cargo.lock | 2 +- massa-bootstrap/src/client_binder.rs | 9 ++++---- massa-bootstrap/src/server_binder.rs | 12 ++++++---- massa-bootstrap/src/tests/binders.rs | 2 +- massa-bootstrap/src/tests/scenarios.rs | 23 +++---------------- massa-consensus-exports/Cargo.toml | 3 ++- massa-network-exports/Cargo.toml | 2 +- massa-network-exports/src/lib.rs | 7 ++++-- .../src/network_controller.rs | 4 +++- 9 files changed, 28 insertions(+), 36 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d9b2d7f8f78..1f619982ebb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3250,7 +3250,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22ce75669015c4f47b289fd4d4f56e894e4c96003ffdf3ac51313126f94c6cbb" dependencies = [ "cfg-if", - "proc-macro2 1.0.54", + "proc-macro2 1.0.56", "quote 1.0.26", "syn 1.0.109", ] diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index 976ab527a7d..cd72a6434cf 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -1,9 +1,5 @@ // Copyright (c) 2022 MASSA LABS -use std::io::{Read, Write}; -use std::net::TcpStream; -use std::time::Duration; - use crate::error::BootstrapError; use crate::messages::{ BootstrapClientMessage, BootstrapClientMessageSerializer, BootstrapServerMessage, @@ -16,6 +12,11 @@ use massa_models::version::{Version, VersionSerializer}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::{PublicKey, Signature, SIGNATURE_SIZE_BYTES}; use rand::{rngs::StdRng, RngCore, SeedableRng}; +use std::{ + io::{Read, Write}, + net::TcpStream, + time::Duration, +}; /// Bootstrap client binder pub struct BootstrapClientBinder { diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 524766a987f..e49bf48c262 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -13,11 +13,13 @@ use massa_models::version::{Version, VersionDeserializer, VersionSerializer}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; use massa_signature::KeyPair; use massa_time::MassaTime; -use std::convert::TryInto; -use std::io::{ErrorKind, Read, Write}; -use std::net::{SocketAddr, TcpStream}; -use std::thread; -use std::time::Duration; +use std::{ + convert::TryInto, + io::{ErrorKind, Read, Write}, + net::{SocketAddr, TcpStream}, + thread, + time::Duration, +}; use tracing::error; /// Bootstrap server binder diff --git a/massa-bootstrap/src/tests/binders.rs b/massa-bootstrap/src/tests/binders.rs index 9326f3d30e5..4ab55d61f56 100644 --- a/massa-bootstrap/src/tests/binders.rs +++ b/massa-bootstrap/src/tests/binders.rs @@ -70,7 +70,7 @@ fn test_binders() { let addr = server.local_addr().unwrap(); let client = std::net::TcpStream::connect(addr).unwrap(); let server = server.accept().unwrap(); - // let (client, server) = duplex(1000000); + let mut server = BootstrapServerBinder::new( server.0, server_keypair.clone(), diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index a4d08717130..33f1db4062d 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -157,27 +157,14 @@ fn test_bootstrap_server() { .unwrap(), final_state_local_config, ))); + + // setup final state mocks. + // TODO: work out a way to handle the clone shenanigans in a cleaner manner let final_state_client_clone = final_state_client.clone(); let final_state_server_clone1 = final_state_server.clone(); let final_state_server_clone2 = final_state_server.clone(); let (mock_bs_listener, mock_remote_connector) = conn_establishment_mocks(); - // // start bootstrap server - // let (mut mock_bs_listener, bootstrap_interface) = mock_establisher::new(); - // let bootstrap_manager = start_bootstrap_server::( - // consensus_controller, - // NetworkCommandSender(network_cmd_tx), - // final_state_server.clone(), - // bootstrap_config.clone(), - // mock_bs_listener - // .get_listener(&bootstrap_config.listen_addr.unwrap()) - // .unwrap(), - // keypair.clone(), - // Version::from_str("TEST.1.10").unwrap(), - // mip_store.clone(), - // ) - // .unwrap() - // .unwrap(); // Setup network command mock-story: hard-code the result of getting bootstrap peers let mut mocked1 = MockNetworkCommandSender::new(); @@ -268,10 +255,6 @@ fn test_bootstrap_server() { }) .unwrap(); - // mock_remote_connector - // .expect_connect_timeout() - // .times(1) - // .returning(|_, _| todo!()); // launch the get_state process let bootstrap_res = tokio::runtime::Runtime::new() .unwrap() diff --git a/massa-consensus-exports/Cargo.toml b/massa-consensus-exports/Cargo.toml index 9141d994edf..3e371d6f2e2 100644 --- a/massa-consensus-exports/Cargo.toml +++ b/massa-consensus-exports/Cargo.toml @@ -15,7 +15,6 @@ serde_json = "1.0" thiserror = "1.0" jsonrpsee = { git = "https://github.com/paritytech/jsonrpsee.git", rev = "118acc3", features = ["server"] } tokio = { version = "1.23", features = ["sync"] } -mockall = {version = "0.11.4", features = ["nightly"]} #custom modules massa_hash = { path = "../massa-hash"} massa_execution_exports = { path = "../massa-execution-exports" } @@ -28,5 +27,7 @@ massa_serialization = { path = "../massa-serialization" } massa_time = { path = "../massa-time" } massa_signature = { path = "../massa-signature" } +[dev-dependencies] +mockall = {version = "0.11.4", features = ["nightly"]} [features] testing = ["massa_models/testing", "massa_execution_exports/testing", "massa_pool_exports/testing", "massa_pos_exports/testing", "massa_protocol_exports/testing", "massa_storage/testing"] diff --git a/massa-network-exports/Cargo.toml b/massa-network-exports/Cargo.toml index 010884b4953..149aff78bf8 100644 --- a/massa-network-exports/Cargo.toml +++ b/massa-network-exports/Cargo.toml @@ -26,10 +26,10 @@ tracing = { version = "0.1", features = [ "release_max_level_debug", ] } async-trait = "0.1.68" -mockall = {version = "0.11.4", features = ["nightly"] } [dev-dependencies] massa_models = { path = "../massa-models", features = ["testing"] } +mockall = {version = "0.11.4", features = ["nightly"] } # for more information on what are the following features used for, see the cargo.toml at workspace level diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index 452d1f9d26c..d88975d2371 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -12,9 +12,12 @@ pub use commands::{ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; + +#[cfg(test)] +pub use network_controller::MockNetworkCommandSender; + pub use network_controller::{ - MockNetworkCommandSender, NetworkCommandSender, NetworkCommandSenderTrait, - NetworkEventReceiver, NetworkManager, + NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, }; pub use peers::{ BootstrapPeers, BootstrapPeersDeserializer, BootstrapPeersSerializer, ConnectionCount, Peer, diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index db8bb7a26f4..79a78a27d8a 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -15,6 +15,7 @@ use massa_models::{ operation::{OperationPrefixIds, SecureShareOperation}, stats::NetworkStats, }; +#[cfg(test)] use mockall::{automock, mock}; use std::{ collections::{HashMap, VecDeque}, @@ -32,6 +33,7 @@ use tracing::{info, warn}; /// Network command sender #[derive(Debug, Clone)] pub struct NetworkCommandSender(pub mpsc::Sender); +#[cfg(test)] mock! { pub NetworkCommandSender{} impl Clone for NetworkCommandSender { @@ -102,7 +104,7 @@ mock! { } } -#[automock] +#[cfg_attr(test, automock)] #[async_trait] /// Network command sender interface. Can be mocked for testing pub trait NetworkCommandSenderTrait: Send + 'static { From 5bb53afff80af6bca3a7d4d32ff98cb555d72d06 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 11 Apr 2023 18:33:32 +0200 Subject: [PATCH 70/83] Hide mocks behind testing flags/builds --- massa-bootstrap/src/tests/scenarios.rs | 1 + massa-consensus-exports/Cargo.toml | 5 ++--- massa-consensus-exports/src/test_exports/mock.rs | 1 + massa-network-exports/Cargo.toml | 4 ++-- massa-network-exports/src/lib.rs | 2 +- massa-network-exports/src/network_controller.rs | 5 +++-- 6 files changed, 10 insertions(+), 8 deletions(-) diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 33f1db4062d..53bd1def237 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -34,6 +34,7 @@ use massa_models::{ }, prehash::PreHashSet, }; +#[cfg(any(test, feature = "testing"))] use massa_network_exports::MockNetworkCommandSender; use massa_pos_exports::{ test_exports::assert_eq_pos_selection, PoSConfig, PoSFinalState, SelectorConfig, diff --git a/massa-consensus-exports/Cargo.toml b/massa-consensus-exports/Cargo.toml index 3e371d6f2e2..4b3f7c80cf4 100644 --- a/massa-consensus-exports/Cargo.toml +++ b/massa-consensus-exports/Cargo.toml @@ -26,8 +26,7 @@ massa_storage = { path = "../massa-storage" } massa_serialization = { path = "../massa-serialization" } massa_time = { path = "../massa-time" } massa_signature = { path = "../massa-signature" } +mockall = {version = "0.11.4", features = ["nightly"], optional = true} -[dev-dependencies] -mockall = {version = "0.11.4", features = ["nightly"]} [features] -testing = ["massa_models/testing", "massa_execution_exports/testing", "massa_pool_exports/testing", "massa_pos_exports/testing", "massa_protocol_exports/testing", "massa_storage/testing"] +testing = ["massa_models/testing", "massa_execution_exports/testing", "massa_pool_exports/testing", "massa_pos_exports/testing", "massa_protocol_exports/testing", "massa_storage/testing", "dep:mockall"] diff --git a/massa-consensus-exports/src/test_exports/mock.rs b/massa-consensus-exports/src/test_exports/mock.rs index 2a10908d635..7768109c70f 100644 --- a/massa-consensus-exports/src/test_exports/mock.rs +++ b/massa-consensus-exports/src/test_exports/mock.rs @@ -90,6 +90,7 @@ pub enum MockConsensusControllerMessage { #[derive(Clone)] pub struct ConsensusControllerImpl(Arc>>); +#[cfg(feature = "testing")] mockall::mock! { pub ConsensusControllerImpl {} impl Clone for ConsensusControllerImpl { diff --git a/massa-network-exports/Cargo.toml b/massa-network-exports/Cargo.toml index 149aff78bf8..38e9dc9e896 100644 --- a/massa-network-exports/Cargo.toml +++ b/massa-network-exports/Cargo.toml @@ -26,12 +26,12 @@ tracing = { version = "0.1", features = [ "release_max_level_debug", ] } async-trait = "0.1.68" +mockall = {version = "0.11.4", features = ["nightly"], optional = true } [dev-dependencies] massa_models = { path = "../massa-models", features = ["testing"] } -mockall = {version = "0.11.4", features = ["nightly"] } # for more information on what are the following features used for, see the cargo.toml at workspace level [features] -testing = ["massa_models/testing", "tempfile"] +testing = ["massa_models/testing", "tempfile", "dep:mockall"] diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index d88975d2371..488023e8963 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -13,7 +13,7 @@ pub use common::{ConnectionClosureReason, ConnectionId}; pub use error::{HandshakeErrorType, NetworkConnectionErrorType, NetworkError}; pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; -#[cfg(test)] +#[cfg(any(test, feature = "testing"))] pub use network_controller::MockNetworkCommandSender; pub use network_controller::{ diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index 79a78a27d8a..f35b15ecc3a 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -33,8 +33,9 @@ use tracing::{info, warn}; /// Network command sender #[derive(Debug, Clone)] pub struct NetworkCommandSender(pub mpsc::Sender); -#[cfg(test)] -mock! { + +#[cfg(any(test, feature = "testing"))] +mockall::mock! { pub NetworkCommandSender{} impl Clone for NetworkCommandSender { fn clone(&self) -> Self; From 3ec51db54f6e78da6b91b3273c4f32a9dc5b2819 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Tue, 11 Apr 2023 18:35:22 +0200 Subject: [PATCH 71/83] Clear cargo-doc warnings --- massa-bootstrap/src/error.rs | 2 +- massa-proto/src/api.bin | Bin 105233 -> 104489 bytes 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index ae10c6e562b..a950ca65a0e 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -70,7 +70,7 @@ pub enum BootstrapError { /// /// Platforms may return a different error code whenever a read times out as /// a result of setting this option. For example Unix typically returns an -/// error of the kind [`WouldBlock`], but Windows may return [`TimedOut`].) +/// error of the kind [`ErrorKind::WouldBlock`], but Windows may return [`ErrorKind::TimedOut`].) impl From for BootstrapError { fn from(e: std::io::Error) -> Self { if e.kind() == ErrorKind::TimedOut || e.kind() == ErrorKind::WouldBlock { diff --git a/massa-proto/src/api.bin b/massa-proto/src/api.bin index 293801b35f312803312697ff39415311ca9d816b..ebde0306b7320dd19be57488f2af4ff4423ee229 100644 GIT binary patch delta 10249 zcmaJ{d301&n$NxOzE_o3m89|#wg}`A_I+^|(u{)L;_kHAN3gXED3k~n?o}!EflE7<>s zL-qUSE{yV*S@2*@zmY$iGOc>*tO zPgPD#PfCq+-8|Q(Swc>0g*L%hq3LA=y;1`$f@R&^=b024@x&pYEdv8X5I~&Aqs}WtgG+FrDnOFI09$r&RiG?XmrVitB$Z(2h$6tBdnt_G8aro z$jJI!H%g9mJ#oIi%b&K3^7A2w3vJG`Sb^qn!Ev4?42KJj^Q^3gTjp|Md(K6c^IcHT z&@FS3MLSCsZJy(aZy8tm13^VVr6M5czct1Jg8p0PwU^QWLH{l5RHB*CdC3@Sa|R|r z!^B|hC1Y$)bj#7&T0l@;B5Qj?Q^#|DU^z>I!wtPMKd|TxlHVase>9o`fge!u147du zji!L0|Iug)2>Kt5rVdBv6~+fiGl0MZXqZUTD=gMs_GZv6NB8J!MuX}K8!%8S#Xa#8 z<7%+OIj8^)6>Wx}SWNi~y~Ie`0U+)cxDnL}H_YRCUf2Mc^LSCmumMu% z@f$j~@?pRWK}}=VQ7^P5Vjy~ZNUh+>s7Af8wf%G)dB&6{#8Q{2>MsKj&DHFze*O; zW{%M6)pD+E0Rj`CVIqIka-CQ*=$4}k)#e5SRV^=3p&a3enq{2#k$wOI6`-M_s9DBi zYD=OQOv`v@6=Hy3TE=_zc6-3)sOPNUIj?b6(%v5}Q}_hry=3s#uAut}0MJm857zKlC({e2HN2#ofnZw0d-QaNzz4aW^GBZZC-|W3 z|M+0{+oj|F$fI4PxLoqVI^zQ%hyzsO0O5mm#s`29x6b$g5aQO653&^Lx#Wb6!EL+i zG8;n%TK|n91DU>&dn#Bk0}LBE4i<`}Tw4B3oR>*!fS?PY(S?I)Q^-WpHgT_u84RXP z-0wzAkAMpB4bFS&UJF!!hKjb|8$8z0^n&UQ-bt+wrpme9d$@N?r7(4K@YFC5Jz;A~|y=w&CQe^mN%UNETiGWbO^#Hs1@%v9(TX-D5dDmvjp;XO=FY{giBFTw?&E%XKVRD!^0EWj7MZ$Edf zB^LmD=p5(6OO{%C0if)xRM6qzJ`0`nPc(cT2ULO1DZv%k;dj&x=_Yg2%ckgRA~ngHqWton?<7(F~@3MG)fV3 zEG*h7S9uj&^XtccG>MhZ4<$H|Ki|@2hRYpDm~ZLq4G0PItX%7rRb=J*yxR+FBk9lHs@cAd-k6RrIh(W;WP(TaZJ}z^n!k^HS}ip6{UWjCpHA`(WN6cg#3Bn-w+089{4v{p2=Bx;NL*&Fh(kv z?}^ugqet&6e?26~2f^zhK|Tmxw>(vTxI7;OuUkEPyGzstDZrUpKjByr8?iN{C;-LQ zkfHz-Tdmf^u>cfXt!~}jf0a5UJh464dVEg#_K+X}g6$ze0tDMb9TFhePCC4;+)(I= z#-Q@EJ>`udLm?O%Lxw^yG+O#9;POH+G+Mao{zWl(p4c7y_Qb)vc83fe7IY!RsThg3zN+8_F(2vqyYAHSFW zDE7nwi{DXFnZ53^n}fGbjvfJQsEU!14p=c$XcU9$fK_^-JQ=|OVCz% zKN$2sFJ<1h@>E&p+El3ks*fdr$gJ;M?K-+Knt(0-SBvMX4WEm;=dV^QPjTnc#I_I< zC5+?bhP}y{T{MnG7YV5NiKNSdPt;AqWNCm=K zYnXBz2*7{Vy6JAy3;wg#$kA@I;)ks*!H%z5#yR$rCYI9JE+n)H(rz zzr`vYqWS4iXtDYYcGoyke<(A*{*}{Z*6{iGbfX7Rc`#q-Ga@Fh!}CQ#U2*_vj_BM4 zH# zs3H>(f|m$HCIwg=*L9I|K?fKm&7~q*?5?FG6sH2cu71z?o-9*mv_d1asuTK-4Tw2) zqELNZ0fbg{0=G1ZFF>qMowz~OUVvDiI?-zYiY@R{?Xz6)*My};3*uTXB5}7A#SOYi zffYg&lmwG5bjjc^S5_{v2Ue^QdHJ$zqXtY}C3FQOv1*96&Y+TL6)gu9dNcxfjS%fx z2M`4sRCOc=$PuWHj*>f^I1N}|@3cI@%GZbfp#8AkgcHWlAdxWsp#89({IQ(&Y@CE{ zt6zO_1hDI3zu#aeTHh6GSAsA z_}i^6C3UuoNFF}vk=2Y`+8~5i5;T6-vkZT!no;&Zw+4Zlv9BJAsf_{qzJFO`+hOF# z#WE3tZPu#_QL7pL9~nAi6H2S#3`YfuGx)7afdmEI8x1=+w{j!WqW}z7n3M{1T@-{i)#b>t8(yi^Zu>2G#GU zLK#%QpAxzj!7Ru)C3KDfgp5-{=NLf9I3@q7 z5P%SJ#$*VpkG@eEVrZN)#5X2G0AU`W$`F89wr?mybVg>0BLy~R?`N#*JewU;z_!G8VGPVH1mP^JKK+L;jYypHVmy9h` zi(S^)E{|-vY-|C-JV0d&AT+ydvR%IC{9WYK*y=c?3heI!pCf()&GJdJYa;svXUXzw zMk};J)HSn=fS7YlB-B?yKxlPMbS-t8DBkjEI@kaa33F`(8=WlqSc-Ysy@GWcIM3E+QNCP?wh#tONdC^J9r$9l zz-EsOe=*dEBI-q3A1oLHjb60%!2$@4UbOYWLRIU+Y-yDJ>cM>S^g>&oD?peAs5}h_ zQ44K-t`vCAB0Fcf_B0i$i|lAe_Xl!Q0l7NJ{*%N0K0Yu?p%Ib-<0e4N35=Tnp;Tbp z1PC_;#!Y~5Q$TJaqYKEMT02<$&Ar(Z5#|i94TaFh(OP3U2Fj7zazF^FHI`GcT}PHb zF{aKKT_BAHR7L~B=sII`!gE&GId5sBskmNYqgwlvK9wZM==BZzqwE+PzuqW>R!CWI ztOmrS^~P#ID74Co21ou;&1G&O`#Mrdy_F6y>h5F z8xT@98MEn%@TOYu1TFZR#&#e~15~yH!uB_f?S-DR#m;$O+fLVlEq1h>dxUH+B-?j3 z^oz5r@^RBJm z82};tUAwa?m;m8~cWwQ#3J^|s&vw)`77$K&&o1l{ml04%f)A(&z~2yaAQVGe{(y-9 z^g_%5v*iIH=75O+y5b#_5s;lz$n3!fO#~Fm2mn+O00@BxDFPN+$orn>9IoxvYv`a2}XK%?WfOJ2hvr(W1T zvqhP7tHdw}4eFImHbBqM>^!wVvH=vI*~MMLDKP6Z8=vV7CkC9bMRzqI10l2+XbDLv zS)U*a=`D{)X+cidxMa6Bfb@ioZ_rfJcobeIZLS^$fWQZ+78ej+KWWE`WO)Jzu9J4L zDqg&HGQ>~XsO(ly5PP&JUuIXfW2-V>h8(2Pm$ohrF@^@&`nv%jwE5ER*-LAK1!=Om z`b^`=7yvZd(9NnTWFm1*wx>SvU@(}PY<$6kT^T3?_?4|+$ADlMprN87?JHYf0?-Sp zuk1qgH3U%3#p{@I5vG1^>(^P2Qz-^$n5Y!{I#h~Ev9Iln{-}Y`p!z!WHU^)zx%z10 zaVoh04HcDKr$eQvof&}QVJ_`#?}vCo~o-trW@%~y@^zFU?u$_G*g4;3&vjs*CN5IO|3e{`vxq`tWy9f)?D^=#m5$aDt&qIZ(J=uC zEmu0ls=5aR^-2eCRCGfsqMftK;TIisLjuO=RStH^+z7S_FbUb=hyf+vW_NaDM*U!`RO$aXkIDig{E2;bo#6H=_p2IQL^PUrq^0>6Gx z4T!KEPM3b}M_elal3k?0Uhdzi-YcdiXW{`OIkCE7Pfu3PGUyJqpm5&RRtpN~U2V0X zklxi+3(9)C+G@d~&|(*9u`aH)Ktr=5`j#w>(?dixz0v$KJ*#SBYE*jt?|U;LvreWz z>wN>V&LG$K^$Se0!k?6$H34rzl$Q4PXX9yPL+JoEon^onDo(qn+0p5Sd_nuC*(vVP z>Mhzb%?@@Dr5hRne8Ca@S_jZ3LPN+&+vq}DPTEKpoZ_BxN=9->l0JP~Nx{+Lh+(ay zw4cxrlG2W9X-i6bs>LZD7)}9ai!)>>eTLzp7(euIa%v*15$OeeIZURR(AaJgm zz)=l|lj{^X4{2%OcrGG(l#KMpOv0nxw8`>S56^aZrprDtl-)iW-O9H_K+*eLLK8oy0zKP-A@ zud?OdCOBaTs(Gc&WW^K?Et zm1%hPm>pZiZ6O4=9I=P-aU~CD?%QgxFWY&xNv%RQ+Zvh#BZV3=f=-!+2Ep>N_M=Qv zh8(fa+x^0_>H8Sh3?VS=W09CYB?N|jtZGM82s`4Sx1#rQ>R`YS2E)NX zhcFlpGH0aE5C+3RHrlnLQiWVc9D-GTXD+M()Vk!t=0hwpO!tE65F4(o$pzCPvL=&j zhsi^>BTi=We!fjiKN)b?(B>qI6!;uAI8L&d=CHwWl9i9M%WW=9$2!Y$&U;t4k4c?n z;ZagWljk_1pK+x>5L5(IDguJOUmFVu`hMmNmC^t~-_J&r_)O?Lr;Rl^0~4TTVleic zHg-I^=-~d2x3~a72lsb8AP71*qNWZ} z6LKt%=dAG-{GvAH@vz#0A#ztI_v0G~LIBkQ0fN3$(*uIOll$=v2>MR4kT!FORc7ffkB zN`)98n9_X0M0*@;4m;K}Jm<%pl?-)f$|HDAqRowa>;I!D^$ZV>mYRpj2WzztfFKS~ zi35b@YqbvmA#Sbq0U*S!)jkN5F6%g-EENX=6QE|IowJTdT-lpKw;VlYq8<&Zb$n8l zuM~XH!}$cilY9WEsmKRCJTgM}f~kj>jL{HGJ$&4FyB0pkb*yK3&d=b3^1t%I3p>kZ zKg+|VQd}Zg2!S|3c6bH|ReZQ-;nuf8;UYd}ETZ!7n& zmt491w(^)JpzXJngMcn;^y0eQ%Ew=?A@1L;y!JYu9!$NUFZRCp>NWA+K!iyhfd~^i z_68zMi0BPOm=Mt$h%h0dmqfgfk3g^;>usL%hQX%Z<$52yI?X%#YA92>>mw`nHV+rs z7ijTpii5*m(VkCIhqVr9gARwa%K@RoVeY7N8W1`h<~XPO>72G{L5^@dpHZ--x=QJl8CKF`~#57CpBv7l8k8yKb zlk=(B9;~D*^_l}y6K!NjQ%hS@Lqg7OYf7~sMRc{{$%XBjty+a69ET8t=duu9KN5JI zw#*Ukr~?6m;q@cDa+3WMLwX&1<^<NsX70}alcZ$cR_yn}yPVw5Q8iM{5zh@bF5 zmwBu(I}G@W_jd>GNL}F~mK{b%;1ym}EDuDRmgOoI*H_ep>419COq^m@xt}LvECgTW z!_}z+2%E3+iF(oKgt!_k8l4bVY0-|@%B$d7mWjR9z@{%VvJx!FUuO7OhRrQVSZ4T# zHy|V|Gs?!;Z^J!UDNuH0es=I1Jk=FYP+`$!_$MXCfTGLDMpg(ET}C#tLQt~H$VOI( z7!4>%w*#sqN31ruUbQfs0jQlp<;QAYPKpLY&}t*5E&xCXT5VKJ@;w=rTvgLFgrL>N z^*8&1P@SZWtkd=P9g z#!s|mA#U>mT&bDT_lnrG-hiS26ukjO0VsNn>;+i>ie6*PSog&&(mKF z2x1_3F(8P6;Ke|P7zkb@9d;@=6gpyuSNZ<0zOW-;C3}-o98m2Hs2ou3G#q_`IH1~TR86sWDXJnzyyn&X z@z>K{3mA&P@LIr71cuiFe-we?HS))U(jUc+c-^b{?}u)CJs>E?IrO>_(J4kT2wpeJ zuJNO;7zD2y*WYMA#%x2-2H9&M|NKGv)iArwtNCC?YOj%}vO3$OYz0s~iU8td-D?aP zZbxV;w)lR7=c^5$i`-|w5y?|ha%tjVh=~$LadSi*>wuBt2_|W1%N#JmL+y7=SQL>N z(b3F>4-WHtk7}FH3O$eNjR1&AM~#@u{{1-yvVTez0kIK|8vfk`h>dX6z}@sQIXyx- z)^V@$Pp9rZ9`I9&eLUc&eBrnetMLgbUpQ_|RW1Yp_>UXc-mZJWf84lbrhQuR!`3t2 z_dYtCIur2IEp#T}Cmqh{b0Q*>gEK~{S}{QIpE1g6eSW$W&KQ$wl)K>deuEcDuSdx1 z{YE6JMoO;-FqKQ|B{3cT5jtl3jePZ<1N1jgjzzB;D^a226O+FEMgVp1k1gp@xaV=f zN0+R!+;27}WUhyQGW-7=Gj2wQ+8Ys0Da_--zsCSU`nZUxIt);tS*sr$=?igz;}sovHdJ%6j3TDx-%zEj%TTE?M5_0t8U>x2LNJDwJ}qao(PCo>J}3wBQXR&>i$L?~*PArV72DX>n6f)ejv zPM4)lUOx`vok1t>!D?#q?ASg|Emx~vIB?$4Jwn9 z1LO!~a)-$sR+I*8%G92@k4@hc_(PV*I;=2;28o#Vhb)iDAKi4mL`mqj%$~Drd1_lg zK~DnPgrB5g3@EkxAjGI$vI@s+{DfOmS| zzfd)C=ip&v=}sEAl`A0lk46tYc&^qO%3t(CXC1b(inV(Xl|f+YSHNR z0_JC-nIi0;MNxSceJEnZdbD)bFdYB}R{TRTVv_E~ihn5ZYWo5m_)&UeIVSiGB|m0v z@R37)0{K5&n)+XSTCH5z)|gz_++bq>e1VF1VH=zQ=d~u3X^WfM8(sCOrCu2gzoXo5 zY?{;5-n1wo9|7<}fVY78@s@hj*;uf)WJ}-;#NU0SvvvoSe#ZiD&_Q=h_)q`3Zy&qlrsGdr#(}4_nj5!_1pi=9!@KfceEVWJx z{|W?zjMKuu0s$f8w7?a3i*z`4Yro)=OX|Fb)=W((t>|+O8ns1KcJ&9+sO;(w)|bkz zelbj4JyC8!TE8e!`2!%N^^5V7Y`P($w7&=Z8v+1g1_Ci5s4}2$2=qeCfW9FBA!b0| z5R}V*rf!H-lx~R6^bG-od4TGM0L0e)jBbcgxLKmO3NH$tS6+Z+oS$5TSokh|7_;R~ z9ilWWw{wsFWZ37X}LMN%>ENaWj>wg z9bGT6KhN$680g~Y2pH(3?lAp`#0)TWn0`b8B1Sq)KOzAU6&)rbk}{8cEYT{{YyNoT z)O=cxRi>Y?0D*Kcgv}C?&-3Ys?o8K)*`2rHZ@9cOP>!yoPSejl&nM7av)3tR4xaEz_fOGfn#->IY04TPIXte86IwbN)9U^pZBB}8|?Gh zJz6O=LQ;?R7$D~KXpaFxsUGbyKzOW2dkhdB>miSk=>=rZ24y<_hUpsuDfHIAL7R?V z8mdhPgp>{1bSfU7SEd(`>CbD^fiMkFnGOikpVy|x9BY%A^RjO`Rg9ZVlpycYb6|{2 z&-7g~*?VkuMr(yuh{|Z&0Wm3~Z3l!_8PhNI0bzT_^hXYp;vVA+*{?(he z2mEC9c5OAr&>(F!Ao#aytEuAMp{$OP)jPD+K$r%otOkVDJG9k>j`a&O=Uv}ws&ao} zhKJa{C#ws|>fL?uT=ow8r`=jBv_jNwZ8ab!?bcQULaW`{YCu@MTU!kXt9O&tXDOr# zDcjs@@;B4z2s<-luNfL9GeNYI?0u#fB{M;ZAqEY@f(F>J&%`&baY&751P1I+?}@Sr z<@-%hlpRO8(SCF2F#8679Pk_I!aR0k>J3xmXUEabd&5LQL%C5Q*sKF4f6G*loxli4 zG^oo25KD8w^uL+`Liho5l**|A;e-RG|Mm(9C%k1^>Qx31PI${K92b#mUr2%v$+%CK zz>N9HWV*YMO-$jBazA~-7$|eh zw8>w%Xw{eEkScd}Zq{-C>E+Fs$s$~U)!%v!# zBAM3!g6pJNtdc%whz#zNCer@36uu6v$JzAT4qKl(8*q?DXHCCyz!(~2`VU}$(B`Z; zeuA$J7Npb=pC2R@+IhMoxe111vJZ#pXJ!eB7&b7rA>)&rDt@s(G(2vg6S{;v!iSpfrTCQ1y?2TD<5cpj;j{K<(pk}&;lDmOG zD@yJLOhl8RQU{y(OWF+lI0<66JQdu%Mz1I zKC|3A8WJt-^y{cL528i2b6{%BFHbA{y7EVdp4YC1e(JpILl^w(Hd{hHsaFPCERG_3&tP#B+j|C!qv+l5|t&LQTo=_ zP9@_%Bp0^UCuSr!{`)v4bg8%wh_=Wuf#>CbK2?FAMpxjaJ0=vQT7* zJnsM@ZCPlj`r-}S`qT&R5l=RR)qXGRSk%f6(I!9-M%z%JQ)(>N^Tjv zk?7+rzCEX(tY$aPM7Q!S5m59#8S;ZbHb{cvGU;2{041Ib;Rim8eF>1bDs9)WCAY5{ z%tYa|YOpyJPOC!x&z^{h!e>kQR1`(42UAfLtq%D=d?G43 zn^*U}Tg&bh#=Dd4bkCivB>mOv*etdvz4kg*K4tR%`*X<$T9S)fYVhHH!JPEb>sUqZ zxk>bE${^(X#o=)G&>S7pC{q>glcnWHaH{X-2== Date: Tue, 11 Apr 2023 18:46:32 +0200 Subject: [PATCH 72/83] More whack-a-mole with CI ~warns~ errors --- massa-network-exports/src/network_controller.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index f35b15ecc3a..f2b44475179 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -15,8 +15,7 @@ use massa_models::{ operation::{OperationPrefixIds, SecureShareOperation}, stats::NetworkStats, }; -#[cfg(test)] -use mockall::{automock, mock}; + use std::{ collections::{HashMap, VecDeque}, net::IpAddr, @@ -105,7 +104,7 @@ mockall::mock! { } } -#[cfg_attr(test, automock)] +#[cfg_attr(any(test, feature = "testing"), mockall::automock)] #[async_trait] /// Network command sender interface. Can be mocked for testing pub trait NetworkCommandSenderTrait: Send + 'static { From f35e9025c7c5be8c6643a486a78fb5c9be79c0e1 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Apr 2023 11:45:59 +0200 Subject: [PATCH 73/83] Remove the dependency on tokio in the bootstrap module (#3796) --- Cargo.lock | 1 - massa-bootstrap/Cargo.toml | 1 - massa-bootstrap/src/error.rs | 2 - massa-bootstrap/src/server.rs | 117 ++++++++---------- massa-bootstrap/src/tests/scenarios.rs | 3 +- .../src/network_controller.rs | 10 ++ 6 files changed, 61 insertions(+), 73 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1f619982ebb..0c319cd8e1d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2501,7 +2501,6 @@ dependencies = [ "substruct", "tempfile", "thiserror", - "tokio", "tracing", ] diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index f73cba4def6..a5821e1f1c3 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -16,7 +16,6 @@ serde_json = "1.0" humantime = "2.1.0" thiserror = "1.0" parking_lot = { version = "0.12", features = ["deadlock_detection"] } -tokio = { version = "1.23", features = ["full"] } tracing = "0.1" substruct = { git = "https://github.com/sydhds/substruct" } socket2 = "0.4.7" diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index a950ca65a0e..3415b9c07d9 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -48,8 +48,6 @@ pub enum BootstrapError { FinalStateError(#[from] FinalStateError), /// Proof-of-Stake error: {0} PoSError(#[from] PosError), - /// join error: {0} - JoinError(#[from] tokio::task::JoinError), /// missing keypair file MissingKeyError, /// incompatible version: {0} diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 3efcc7be945..ad1c229ac87 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -50,7 +50,6 @@ use std::{ thread, time::{Duration, Instant}, }; -use tokio::runtime::{self, Handle}; use tracing::{debug, error, info, warn}; use white_black_list::*; @@ -244,16 +243,6 @@ impl BootstrapServer<'_, C> { } fn run_loop(mut self, max_bootstraps: usize) -> Result<(), BootstrapError> { - let Ok(bs_loop_rt) = runtime::Builder::new_multi_thread() - .max_blocking_threads(max_bootstraps * 2) - .enable_io() - .enable_time() - .thread_name("bootstrap-main-loop-worker") - .thread_keep_alive(Duration::from_millis(u64::MAX)) - .build() else { - return Err(BootstrapError::GeneralError("Failed to create bootstrap main-loop runtime".to_string())); - }; - // Use the strong-count of this variable to track the session count let bootstrap_sessions_counter: Arc<()> = Arc::new(()); let per_ip_min_interval = self.bootstrap_config.per_ip_min_interval.to_duration(); @@ -334,7 +323,6 @@ impl BootstrapServer<'_, C> { let config = self.bootstrap_config.clone(); let bootstrap_count_token = bootstrap_sessions_counter.clone(); - let session_handle = bs_loop_rt.handle().clone(); let mip_store = self.mip_store.clone(); let _ = thread::Builder::new() @@ -349,7 +337,6 @@ impl BootstrapServer<'_, C> { version, consensus_command_sender, network_command_sender, - session_handle, mip_store, ) }); @@ -367,7 +354,6 @@ impl BootstrapServer<'_, C> { } // Give any remaining processes 20 seconds to clean up, otherwise force them to shutdown - bs_loop_rt.shutdown_timeout(Duration::from_secs(20)); Ok(()) } @@ -466,61 +452,56 @@ fn run_bootstrap_session( version: Version, consensus_command_sender: Box, network_command_sender: C, - bs_loop_rt_handle: Handle, mip_store: MipStore, ) { debug!("running bootstrap for peer {}", remote_addr); - bs_loop_rt_handle.block_on(async move { - let res = tokio::time::timeout( - config.bootstrap_timeout.into(), - manage_bootstrap( - &config, - &mut server, - data_execution, - version, - consensus_command_sender, - network_command_sender, - mip_store, - ), - ) - .await; - // This drop allows the server to accept new connections before having to complete the error notifications - // account for this session being finished, as well as the root-instance - massa_trace!("bootstrap.session.finished", { - "sessions_remaining": Arc::strong_count(&arc_counter) - 2 - }); - drop(arc_counter); - match res { - Ok(mgmt) => match mgmt { - Ok(_) => { - info!("bootstrapped peer {}", remote_addr); - } - Err(BootstrapError::ReceivedError(error)) => debug!( - "bootstrap serving error received from peer {}: {}", - remote_addr, error - ), - Err(err) => { - debug!("bootstrap serving error for peer {}: {}", remote_addr, err); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(err.to_string()); - } - }, - Err(_timeout) => { - debug!("bootstrap timeout for peer {}", remote_addr); - // We allow unused result because we don't care if an error is thrown when - // sending the error message to the server we will close the socket anyway. - let _ = server.send_error_timeout(format!( - "Bootstrap process timedout ({})", - format_duration(config.bootstrap_timeout.to_duration()) - )); - } - } + let deadline = Instant::now() + config.bootstrap_timeout.to_duration(); + // TODO: reinstate prevention of bootstrap slot camping. Deadline cancellation is one option + let res = manage_bootstrap( + &config, + &mut server, + data_execution, + version, + consensus_command_sender, + network_command_sender, + deadline, + mip_store, + ); + // TODO: handle the deadline management + // This drop allows the server to accept new connections before having to complete the error notifications + // account for this session being finished, as well as the root-instance + massa_trace!("bootstrap.session.finished", { + "sessions_remaining": Arc::strong_count(&arc_counter) - 2 }); + drop(arc_counter); + match res { + Err(BootstrapError::TimedOut(_)) => { + debug!("bootstrap timeout for peer {}", remote_addr); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(format!( + "Bootstrap process timedout ({})", + format_duration(config.bootstrap_timeout.to_duration()) + )); + } + Err(BootstrapError::ReceivedError(error)) => debug!( + "bootstrap serving error received from peer {}: {}", + remote_addr, error + ), + Err(err) => { + debug!("bootstrap serving error for peer {}: {}", remote_addr, err); + // We allow unused result because we don't care if an error is thrown when + // sending the error message to the server we will close the socket anyway. + let _ = server.send_error_timeout(err.to_string()); + } + Ok(_) => { + info!("bootstrapped peer {}", remote_addr); + } + } } #[allow(clippy::too_many_arguments)] -pub async fn stream_bootstrap_information( +pub fn stream_bootstrap_information( server: &mut BootstrapServerBinder, final_state: Arc>, consensus_controller: Box, @@ -538,7 +519,7 @@ pub async fn stream_bootstrap_information( #[cfg(test)] { // Necessary for test_bootstrap_server in tests/scenarios.rs - tokio::time::sleep(Duration::from_millis(500)).await; + std::thread::sleep(Duration::from_millis(500)); } let current_slot; @@ -701,17 +682,19 @@ pub async fn stream_bootstrap_information( } #[allow(clippy::too_many_arguments)] -async fn manage_bootstrap( +fn manage_bootstrap( bootstrap_config: &BootstrapConfig, server: &mut BootstrapServerBinder, final_state: Arc>, version: Version, consensus_controller: Box, network_command_sender: C, + _deadline: Instant, mip_store: MipStore, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.manage_bootstrap", {}); let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); + let rt_hack = massa_network_exports::make_runtime(); server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))?; @@ -746,7 +729,8 @@ async fn manage_bootstrap( server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { - peers: network_command_sender.get_bootstrap_peers().await?, + peers: rt_hack + .block_on(network_command_sender.get_bootstrap_peers())?, }, )?; } @@ -773,8 +757,7 @@ async fn manage_bootstrap( last_consensus_step, send_last_start_period, write_timeout, - ) - .await?; + )?; } BootstrapClientMessage::AskBootstrapMipStore => { let vs = mip_store.0.read().to_owned(); diff --git a/massa-bootstrap/src/tests/scenarios.rs b/massa-bootstrap/src/tests/scenarios.rs index 53bd1def237..98b7677eb90 100644 --- a/massa-bootstrap/src/tests/scenarios.rs +++ b/massa-bootstrap/src/tests/scenarios.rs @@ -257,8 +257,7 @@ fn test_bootstrap_server() { .unwrap(); // launch the get_state process - let bootstrap_res = tokio::runtime::Runtime::new() - .unwrap() + let bootstrap_res = massa_network_exports::make_runtime() .block_on(get_state( bootstrap_config, final_state_client_clone, diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index f2b44475179..735e770a2e5 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -452,3 +452,13 @@ impl NetworkManager { Ok(()) } } + +/// Used by the bootstrap server to run async tasks, allowing the bootstrap module to +/// remove the tokio dependency. +pub fn make_runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_multi_thread() + .enable_all() + .thread_name("network-provided-runtime") + .build() + .expect("failed to create runtime") +} From f7583f241a6156f2063f095c85758d4fed863e17 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 13:30:34 +0200 Subject: [PATCH 74/83] Fix missing network-controller export of runtime constructor --- massa-network-exports/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/massa-network-exports/src/lib.rs b/massa-network-exports/src/lib.rs index 488023e8963..cc48f4208e0 100644 --- a/massa-network-exports/src/lib.rs +++ b/massa-network-exports/src/lib.rs @@ -17,7 +17,8 @@ pub use establisher::{Establisher, Listener, ReadHalf, WriteHalf}; pub use network_controller::MockNetworkCommandSender; pub use network_controller::{ - NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, NetworkManager, + make_runtime, NetworkCommandSender, NetworkCommandSenderTrait, NetworkEventReceiver, + NetworkManager, }; pub use peers::{ BootstrapPeers, BootstrapPeersDeserializer, BootstrapPeersSerializer, ConnectionCount, Peer, From de44109b9a9ada27977e5c9c641e59844f937e1b Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 13:32:24 +0200 Subject: [PATCH 75/83] Comment providing context of why an unused variable is retained --- massa-bootstrap/src/server_binder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index e49bf48c262..7fd180566ea 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -48,6 +48,7 @@ impl BootstrapServerBinder { #[allow(clippy::too_many_arguments)] pub fn new(duplex: TcpStream, local_keypair: KeyPair, cfg: BootstrapSrvBindCfg) -> Self { let BootstrapSrvBindCfg { + // TODO: Reintroduce bandwidth limits max_bytes_read_write: _limit, max_bootstrap_message_size, thread_count, From caf4d8769e25bb08934068809cebbbce5538f766 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 14:15:35 +0200 Subject: [PATCH 76/83] Minor cleanup of mocking code --- massa-network-exports/src/network_controller.rs | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/massa-network-exports/src/network_controller.rs b/massa-network-exports/src/network_controller.rs index 735e770a2e5..351d3d70465 100644 --- a/massa-network-exports/src/network_controller.rs +++ b/massa-network-exports/src/network_controller.rs @@ -39,67 +39,52 @@ mockall::mock! { impl Clone for NetworkCommandSender { fn clone(&self) -> Self; } + #[async_trait] impl NetworkCommandSenderTrait for NetworkCommandSender { async fn node_ban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; - async fn node_ban_by_ips(&self, ips: Vec) -> Result<(), NetworkError>; - async fn add_to_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; - async fn remove_from_whitelist(&self, ips: Vec) -> Result<(), NetworkError>; - async fn node_unban_by_ids(&self, ids: Vec) -> Result<(), NetworkError>; - async fn node_unban_ips(&self, ips: Vec) -> Result<(), NetworkError>; - async fn send_block_info( &self, node: NodeId, info: Vec<(BlockId, BlockInfoReply)>, ) -> Result<(), NetworkError>; - async fn ask_for_block_list( &self, list: HashMap>, ) -> Result<(), NetworkError>; - async fn send_block_header( &self, node: NodeId, header: SecuredHeader, ) -> Result<(), NetworkError>; - async fn get_peers(&self) -> Result; - async fn get_network_stats(&self) -> Result; - async fn get_bootstrap_peers(&self) -> Result; - async fn send_operations( &self, node: NodeId, operations: Vec, ) -> Result<(), NetworkError>; - async fn announce_operations( &self, to_node: NodeId, batch: OperationPrefixIds, ) -> Result<(), NetworkError>; - async fn send_ask_for_operations( &self, to_node: NodeId, wishlist: OperationPrefixIds, ) -> Result<(), NetworkError>; - async fn send_endorsements( &self, node: NodeId, endorsements: Vec, ) -> Result<(), NetworkError>; - async fn node_sign_message(&self, msg: Vec) -> Result; } } From 56122d1de25ba6c9d88eb2618a211fe99a80290e Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Apr 2023 14:30:52 +0200 Subject: [PATCH 77/83] Bootstrap server binding read-message in a single `read_exact` (#3774) --- massa-bootstrap/src/server_binder.rs | 37 ++++++++++++++++++---------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 7fd180566ea..4d888336d81 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -222,27 +222,38 @@ impl BootstrapServerBinder { duration: Option, ) -> Result { self.duplex.set_read_timeout(duration)?; - // read prev hash + + let peek_len = HASH_SIZE_BYTES + self.size_field_len; + let mut peek_buf = vec![0; peek_len]; + while self.duplex.peek(&mut peek_buf)? < peek_len { + // TODO: backoff spin of some sort + } + // construct prev-hash from peek let received_prev_hash = { if self.prev_message.is_some() { - let mut hash_bytes = [0u8; HASH_SIZE_BYTES]; - self.duplex.read_exact(&mut hash_bytes)?; - Some(Hash::from_bytes(&hash_bytes)) + Some(Hash::from_bytes( + peek_buf[..HASH_SIZE_BYTES] + .try_into() + .expect("bad slice logic"), + )) } else { None } }; - // read message length + // construct msg-len from peek let msg_len = { - let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..])?; - u32::from_be_bytes_min(&msg_len_bytes, self.max_bootstrap_message_size)?.0 + u32::from_be_bytes_min( + &peek_buf[HASH_SIZE_BYTES..], + self.max_bootstrap_message_size, + )? + .0 }; - // read message - let mut msg_bytes = vec![0u8; msg_len as usize]; + // read message, and discard the peek + let mut msg_bytes = vec![0u8; peek_len + (msg_len as usize)]; self.duplex.read_exact(&mut msg_bytes)?; + let msg_bytes = &msg_bytes[peek_len..]; // check previous hash if received_prev_hash != self.prev_message { @@ -257,11 +268,11 @@ impl BootstrapServerBinder { let mut hashed_bytes = Vec::with_capacity(HASH_SIZE_BYTES.saturating_add(msg_bytes.len())); hashed_bytes.extend(prev_hash.to_bytes()); - hashed_bytes.extend(&msg_bytes); + hashed_bytes.extend(msg_bytes); self.prev_message = Some(Hash::compute_from(&hashed_bytes)); } else { // no previous message: hash message only - self.prev_message = Some(Hash::compute_from(&msg_bytes)); + self.prev_message = Some(Hash::compute_from(msg_bytes)); } // deserialize message @@ -270,7 +281,7 @@ impl BootstrapServerBinder { self.max_datastore_key_length, self.max_consensus_block_ids, ) - .deserialize::(&msg_bytes) + .deserialize::(msg_bytes) .map_err(|err| BootstrapError::GeneralError(format!("{}", err)))?; Ok(msg) From 4bea0ab753c4cafa248819197d1d4b717ce88ebe Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 14:18:36 +0200 Subject: [PATCH 78/83] Expand the bandwith limit todo-comment --- massa-bootstrap/src/server_binder.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/massa-bootstrap/src/server_binder.rs b/massa-bootstrap/src/server_binder.rs index 4d888336d81..f45e823a0f0 100644 --- a/massa-bootstrap/src/server_binder.rs +++ b/massa-bootstrap/src/server_binder.rs @@ -31,6 +31,7 @@ pub struct BootstrapServerBinder { randomness_size_bytes: usize, size_field_len: usize, local_keypair: KeyPair, + // TODO: Reintroduce bandwidth limits duplex: TcpStream, prev_message: Option, version_serializer: VersionSerializer, From 748f7ac914b7d756a8b48e8b160b69a14d1642a5 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 14:47:07 +0200 Subject: [PATCH 79/83] Fix dependency management for when testing bootstrap server directly --- massa-bootstrap/Cargo.toml | 3 +++ massa-network-exports/Cargo.toml | 1 + 2 files changed, 4 insertions(+) diff --git a/massa-bootstrap/Cargo.toml b/massa-bootstrap/Cargo.toml index a5821e1f1c3..cc3309411b1 100644 --- a/massa-bootstrap/Cargo.toml +++ b/massa-bootstrap/Cargo.toml @@ -50,6 +50,9 @@ massa_pos_exports = { path = "../massa-pos-exports", features = ["testing"] } massa_consensus_exports = { path = "../massa-consensus-exports", features = [ "testing", ] } +massa_network_exports = { path = "../massa-network-exports", features = [ + "testing", +] } lazy_static = "1.4" tempfile = "3.3" diff --git a/massa-network-exports/Cargo.toml b/massa-network-exports/Cargo.toml index 38e9dc9e896..d76ec1de547 100644 --- a/massa-network-exports/Cargo.toml +++ b/massa-network-exports/Cargo.toml @@ -30,6 +30,7 @@ mockall = {version = "0.11.4", features = ["nightly"], optional = true } [dev-dependencies] massa_models = { path = "../massa-models", features = ["testing"] } +mockall = {version = "0.11.4", features = ["nightly"] } # for more information on what are the following features used for, see the cargo.toml at workspace level From 2a3b0f4090d599721c5025fe7b89a838d5dd9653 Mon Sep 17 00:00:00 2001 From: Ben Date: Wed, 12 Apr 2023 17:57:19 +0200 Subject: [PATCH 80/83] Bootstrap/peek streams (#3820) --- massa-bootstrap/src/client_binder.rs | 78 ++++++++++++++++++---------- 1 file changed, 51 insertions(+), 27 deletions(-) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index cd72a6434cf..eb82c281362 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -6,7 +6,7 @@ use crate::messages::{ BootstrapServerMessageDeserializer, }; use crate::settings::BootstrapClientConfig; -use massa_hash::{Hash, HASH_SIZE_BYTES}; +use massa_hash::Hash; use massa_models::serialization::{DeserializeMinBEInt, SerializeMinBEInt}; use massa_models::version::{Version, VersionSerializer}; use massa_serialization::{DeserializeError, Deserializer, Serializer}; @@ -75,43 +75,67 @@ impl BootstrapClientBinder { duration: Option, ) -> Result { self.duplex.set_read_timeout(duration)?; - // read signature - let sig = { - let mut sig_bytes = [0u8; SIGNATURE_SIZE_BYTES]; - self.duplex.read_exact(&mut sig_bytes)?; - Signature::from_bytes(&sig_bytes)? - }; - // read message length - let msg_len = { - let mut msg_len_bytes = vec![0u8; self.size_field_len]; - self.duplex.read_exact(&mut msg_len_bytes[..])?; - u32::from_be_bytes_min(&msg_len_bytes, self.cfg.max_bootstrap_message_size)?.0 - }; + // peek the signature and message len + let peek_len = SIGNATURE_SIZE_BYTES + self.size_field_len; + let mut peek_buff = vec![0u8; peek_len]; + // problematic if we can only peek some of it + while self.duplex.peek(&mut peek_buff)? < peek_len { + // TODO: Backoff spin of some sort + } + + // construct the signature from the peek + let sig_array = peek_buff.as_slice()[0..SIGNATURE_SIZE_BYTES] + .try_into() + .expect("logic error in array manipulations"); + let sig = Signature::from_bytes(&sig_array)?; - // read message, check signature and check signature of the message sent just before then deserialize it + // construct the message len from the peek + let msg_len = u32::from_be_bytes_min( + &peek_buff[SIGNATURE_SIZE_BYTES..], + self.cfg.max_bootstrap_message_size, + )? + .0; + + // Update this bindings "most recently received" message hash, retaining the replaced value let message_deserializer = BootstrapServerMessageDeserializer::new((&self.cfg).into()); + let legacy_msg = self + .prev_message + .replace(Hash::compute_from(&sig.to_bytes())); + let message = { - if let Some(prev_message) = self.prev_message { - self.prev_message = Some(Hash::compute_from(&sig.to_bytes())); - let mut sig_msg_bytes = vec![0u8; HASH_SIZE_BYTES + (msg_len as usize)]; - sig_msg_bytes[..HASH_SIZE_BYTES].copy_from_slice(prev_message.to_bytes()); - self.duplex - .read_exact(&mut sig_msg_bytes[HASH_SIZE_BYTES..])?; - let msg_hash = Hash::compute_from(&sig_msg_bytes); + if let Some(legacy_msg) = legacy_msg { + // Consume the stream, and discard the peek + let mut stream_bytes = vec![0u8; peek_len + (msg_len as usize)]; + // TODO: under the hood, this isn't actually atomic. For now, we use the ostrich algorithm. + self.duplex.read_exact(&mut stream_bytes[..])?; + let msg_bytes = &mut stream_bytes[peek_len..]; + + // prepend the received message with the previous messages hash, and derive the new hash. + // TODO: some sort of recovery if this fails? + let rehash_seed = &[legacy_msg.to_bytes().as_slice(), msg_bytes].concat(); + let msg_hash = Hash::compute_from(rehash_seed); self.remote_pubkey.verify_signature(&msg_hash, &sig)?; + + // ...And deserialize let (_, msg) = message_deserializer - .deserialize::(&sig_msg_bytes[HASH_SIZE_BYTES..]) + .deserialize::(&msg_bytes) .map_err(|err| BootstrapError::DeserializeError(format!("{}", err)))?; msg } else { - self.prev_message = Some(Hash::compute_from(&sig.to_bytes())); - let mut sig_msg_bytes = vec![0u8; msg_len as usize]; - self.duplex.read_exact(&mut sig_msg_bytes[..])?; - let msg_hash = Hash::compute_from(&sig_msg_bytes); + // Consume the stream and discard the peek + let mut stream_bytes = vec![0u8; peek_len + msg_len as usize]; + // TODO: under the hood, this isn't actually atomic. For now, we use the ostrich algorithm. + self.duplex.read_exact(&mut stream_bytes[..])?; + let sig_msg_bytes = &mut stream_bytes[peek_len..]; + + // Compute the hash and verify + let msg_hash = Hash::compute_from(sig_msg_bytes); self.remote_pubkey.verify_signature(&msg_hash, &sig)?; + + // ...And deserialize let (_, msg) = message_deserializer - .deserialize::(&sig_msg_bytes[..]) + .deserialize::(sig_msg_bytes) .map_err(|err| BootstrapError::DeserializeError(format!("{}", err)))?; msg } From 72d1f6b74210bca554d067a6cf3781114216feea Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 18:54:44 +0200 Subject: [PATCH 81/83] Set timeouts according to the time-window alloted --- massa-bootstrap/src/error.rs | 2 ++ massa-bootstrap/src/server.rs | 66 ++++++++++++++++++++++++----------- 2 files changed, 47 insertions(+), 21 deletions(-) diff --git a/massa-bootstrap/src/error.rs b/massa-bootstrap/src/error.rs index 3415b9c07d9..e5136ded0d2 100644 --- a/massa-bootstrap/src/error.rs +++ b/massa-bootstrap/src/error.rs @@ -62,6 +62,8 @@ pub enum BootstrapError { BlackListed(String), /// IP {0} is not in the whitelist WhiteListed(String), + /// The bootstrap process ended prematurely - e.g. too much time elapsed + Interupted(String), } /// # Platform-specific behavior diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index ad1c229ac87..a180eeb852f 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -513,6 +513,7 @@ pub fn stream_bootstrap_information( mut last_ops_step: StreamingStep, mut last_consensus_step: StreamingStep>, mut send_last_start_period: bool, + bs_deadline: &Instant, write_timeout: Duration, ) -> Result<(), BootstrapError> { loop { @@ -653,6 +654,7 @@ pub fn stream_bootstrap_information( } // If the consensus streaming is finished (also meaning that consensus slot == final state slot) exit + // We don't bother with the bs-deadline, as this is the last step of the bootstrap process - defer to general write-timeout if final_state_global_step.finished() && final_state_changes_step.finished() && last_consensus_step.finished() @@ -661,6 +663,9 @@ pub fn stream_bootstrap_information( break; } + let Some(write_timeout) = step_timeout_duration(&bs_deadline, &write_timeout) else { + return Err(BootstrapError::Interupted("insufficient time left to respond te request for mip-store".to_string())); + }; // At this point we know that consensus, final state or both are not finished server.send_msg( write_timeout, @@ -681,6 +686,17 @@ pub fn stream_bootstrap_information( Ok(()) } +// derives the duration allowed for a step in the bootstrap process. +// Returns None if the deadline for the entire bs-process has been reached +fn step_timeout_duration(bs_deadline: &Instant, step_timeout: &Duration) -> Option { + let now = Instant::now(); + if now >= *bs_deadline { + return None; + } + + let remaining = *bs_deadline - now; + Some(std::cmp::min(remaining, *step_timeout)) +} #[allow(clippy::too_many_arguments)] fn manage_bootstrap( bootstrap_config: &BootstrapConfig, @@ -689,50 +705,53 @@ fn manage_bootstrap( version: Version, consensus_controller: Box, network_command_sender: C, - _deadline: Instant, + deadline: Instant, mip_store: MipStore, ) -> Result<(), BootstrapError> { massa_trace!("bootstrap.lib.manage_bootstrap", {}); - let read_error_timeout: Duration = bootstrap_config.read_error_timeout.into(); - let rt_hack = massa_network_exports::make_runtime(); - server.handshake_timeout(version, Some(bootstrap_config.read_timeout.into()))?; + // TODO: make network_command_sender non-async and remove both the variable and the method + let rt_hack = massa_network_exports::make_runtime(); - match server.next_timeout(Some(read_error_timeout)) { - Err(BootstrapError::TimedOut(_)) => {} - Err(e) => return Err(e), - Ok(BootstrapClientMessage::BootstrapError { error }) => { - return Err(BootstrapError::GeneralError(error)); - } - Ok(msg) => return Err(BootstrapError::UnexpectedClientMessage(Box::new(msg))), + let Some(hs_deadline) = step_timeout_duration(&deadline, &bootstrap_config.read_timeout.to_duration()) else { + return Err(BootstrapError::Interupted("insufficient time left to begin handshake".to_string())); }; - let write_timeout: Duration = bootstrap_config.write_timeout.into(); - - // Sync clocks. - let server_time = MassaTime::now()?; + server.handshake_timeout(version, Some(hs_deadline))?; + let send_time_timeout = + step_timeout_duration(&deadline, &bootstrap_config.write_timeout.to_duration()); + let Some(next_step_timeout) = send_time_timeout else { + return Err(BootstrapError::Interupted("insufficient time left to send server time".to_string())); + }; server.send_msg( - write_timeout, + next_step_timeout, BootstrapServerMessage::BootstrapTime { - server_time, + server_time: MassaTime::now()?, version, }, )?; loop { - match server.next_timeout(Some(bootstrap_config.read_timeout.into())) { + let Some(read_timeout) = step_timeout_duration(&deadline, &bootstrap_config.read_timeout.to_duration()) else { + return Err(BootstrapError::Interupted("insufficient time left to process next message".to_string())); + }; + match dbg!(server.next_timeout(Some(read_timeout))) { Err(BootstrapError::TimedOut(_)) => break Ok(()), Err(e) => break Err(e), Ok(msg) => match msg { BootstrapClientMessage::AskBootstrapPeers => { - server.send_msg( + let Some(write_timeout) = step_timeout_duration(&deadline, &bootstrap_config.write_timeout.to_duration()) else { + return Err(BootstrapError::Interupted("insufficient time left to respond te request for peers".to_string())); + }; + + dbg!(server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { peers: rt_hack .block_on(network_command_sender.get_bootstrap_peers())?, }, - )?; + ))?; } BootstrapClientMessage::AskBootstrapPart { last_slot, @@ -756,11 +775,16 @@ fn manage_bootstrap( last_ops_step, last_consensus_step, send_last_start_period, - write_timeout, + &deadline, + bootstrap_config.write_timeout.to_duration(), )?; } BootstrapClientMessage::AskBootstrapMipStore => { let vs = mip_store.0.read().to_owned(); + let Some(write_timeout) = step_timeout_duration(&deadline, &bootstrap_config.write_timeout.to_duration()) else { + return Err(BootstrapError::Interupted("insufficient time left to respond te request for mip-store".to_string())); + }; + server.send_msg( write_timeout, BootstrapServerMessage::BootstrapMipStore { store: vs.clone() }, From a6e96533e82e10d8b44131d19e079da75b2132fe Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 19:02:19 +0200 Subject: [PATCH 82/83] Address self-reveiw comments --- massa-bootstrap/src/client_binder.rs | 2 +- massa-bootstrap/src/server.rs | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/massa-bootstrap/src/client_binder.rs b/massa-bootstrap/src/client_binder.rs index eb82c281362..a59bec23b28 100644 --- a/massa-bootstrap/src/client_binder.rs +++ b/massa-bootstrap/src/client_binder.rs @@ -119,7 +119,7 @@ impl BootstrapClientBinder { // ...And deserialize let (_, msg) = message_deserializer - .deserialize::(&msg_bytes) + .deserialize::(msg_bytes) .map_err(|err| BootstrapError::DeserializeError(format!("{}", err)))?; msg } else { diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index a180eeb852f..04a69546f8a 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -663,8 +663,8 @@ pub fn stream_bootstrap_information( break; } - let Some(write_timeout) = step_timeout_duration(&bs_deadline, &write_timeout) else { - return Err(BootstrapError::Interupted("insufficient time left to respond te request for mip-store".to_string())); + let Some(write_timeout) = step_timeout_duration(bs_deadline, &write_timeout) else { + return Err(BootstrapError::Interupted("insufficient time left to provide next bootstrap part".to_string())); }; // At this point we know that consensus, final state or both are not finished server.send_msg( @@ -713,11 +713,11 @@ fn manage_bootstrap( // TODO: make network_command_sender non-async and remove both the variable and the method let rt_hack = massa_network_exports::make_runtime(); - let Some(hs_deadline) = step_timeout_duration(&deadline, &bootstrap_config.read_timeout.to_duration()) else { + let Some(hs_timeout) = step_timeout_duration(&deadline, &bootstrap_config.read_timeout.to_duration()) else { return Err(BootstrapError::Interupted("insufficient time left to begin handshake".to_string())); }; - server.handshake_timeout(version, Some(hs_deadline))?; + server.handshake_timeout(version, Some(hs_timeout))?; let send_time_timeout = step_timeout_duration(&deadline, &bootstrap_config.write_timeout.to_duration()); @@ -736,7 +736,7 @@ fn manage_bootstrap( let Some(read_timeout) = step_timeout_duration(&deadline, &bootstrap_config.read_timeout.to_duration()) else { return Err(BootstrapError::Interupted("insufficient time left to process next message".to_string())); }; - match dbg!(server.next_timeout(Some(read_timeout))) { + match server.next_timeout(Some(read_timeout)) { Err(BootstrapError::TimedOut(_)) => break Ok(()), Err(e) => break Err(e), Ok(msg) => match msg { @@ -745,13 +745,13 @@ fn manage_bootstrap( return Err(BootstrapError::Interupted("insufficient time left to respond te request for peers".to_string())); }; - dbg!(server.send_msg( + server.send_msg( write_timeout, BootstrapServerMessage::BootstrapPeers { peers: rt_hack .block_on(network_command_sender.get_bootstrap_peers())?, }, - ))?; + )?; } BootstrapClientMessage::AskBootstrapPart { last_slot, From e0e80e2370dd87438773add9e5cf691f4df5f289 Mon Sep 17 00:00:00 2001 From: Ben PHL Date: Wed, 12 Apr 2023 19:09:58 +0200 Subject: [PATCH 83/83] Clear out resolved todo --- massa-bootstrap/src/server.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/massa-bootstrap/src/server.rs b/massa-bootstrap/src/server.rs index 04a69546f8a..b464ea05f5a 100644 --- a/massa-bootstrap/src/server.rs +++ b/massa-bootstrap/src/server.rs @@ -467,7 +467,7 @@ fn run_bootstrap_session( deadline, mip_store, ); - // TODO: handle the deadline management + // This drop allows the server to accept new connections before having to complete the error notifications // account for this session being finished, as well as the root-instance massa_trace!("bootstrap.session.finished", {