From 11ee1e431ee8bca56b00bd6e25fc5e4d1d0a777c Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 16:54:00 +0900 Subject: [PATCH 01/50] feat: seperate robust provider into own crate --- examples/historical_scanning/main.rs | 4 +- examples/latest_events_scanning/main.rs | 4 +- examples/live_scanning/main.rs | 4 +- examples/sync_from_block_scanning/main.rs | 4 +- examples/sync_from_latest_scanning/main.rs | 4 +- src/block_range_scanner.rs | 3 +- src/error.rs | 2 +- src/event_scanner/message.rs | 2 +- src/event_scanner/scanner/common.rs | 2 +- src/event_scanner/scanner/mod.rs | 3 +- src/lib.rs | 5 + src/robust_provider/builder.rs | 99 ++++++++ src/robust_provider/error.rs | 30 +++ src/robust_provider/mod.rs | 4 + .../provider.rs} | 220 +----------------- src/robust_provider/types.rs | 95 ++++++++ tests/common/mod.rs | 2 +- tests/common/setup_scanner.rs | 2 +- 18 files changed, 257 insertions(+), 232 deletions(-) create mode 100644 src/robust_provider/builder.rs create mode 100644 src/robust_provider/error.rs create mode 100644 src/robust_provider/mod.rs rename src/{robust_provider.rs => robust_provider/provider.rs} (72%) create mode 100644 src/robust_provider/types.rs diff --git a/examples/historical_scanning/main.rs b/examples/historical_scanning/main.rs index 1d2cc039..59cc88b6 100644 --- a/examples/historical_scanning/main.rs +++ b/examples/historical_scanning/main.rs @@ -1,9 +1,7 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, -}; +use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; use tokio_stream::StreamExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; diff --git a/examples/latest_events_scanning/main.rs b/examples/latest_events_scanning/main.rs index a51c404f..6f4ccad5 100644 --- a/examples/latest_events_scanning/main.rs +++ b/examples/latest_events_scanning/main.rs @@ -1,8 +1,6 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, -}; +use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; use tokio_stream::StreamExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; diff --git a/examples/live_scanning/main.rs b/examples/live_scanning/main.rs index dd888e20..4daa146b 100644 --- a/examples/live_scanning/main.rs +++ b/examples/live_scanning/main.rs @@ -1,8 +1,6 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, -}; +use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/sync_from_block_scanning/main.rs b/examples/sync_from_block_scanning/main.rs index 35c0c254..3d4fe177 100644 --- a/examples/sync_from_block_scanning/main.rs +++ b/examples/sync_from_block_scanning/main.rs @@ -2,9 +2,7 @@ use std::time::Duration; use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, -}; +use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; use tokio::time::sleep; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/sync_from_latest_scanning/main.rs b/examples/sync_from_latest_scanning/main.rs index c8ba8cfb..96f4e797 100644 --- a/examples/sync_from_latest_scanning/main.rs +++ b/examples/sync_from_latest_scanning/main.rs @@ -1,8 +1,6 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, -}; +use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index da742503..81d1578e 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -67,9 +67,8 @@ use tokio::{ use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ - ScannerMessage, + IntoRobustProvider, RobustProvider, RobustProviderError, ScannerMessage, error::ScannerError, - robust_provider::{Error as RobustProviderError, IntoRobustProvider, RobustProvider}, types::{ScannerStatus, TryStream}, }; use alloy::{ diff --git a/src/error.rs b/src/error.rs index e7ebedb5..44a1059d 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,7 +6,7 @@ use alloy::{ }; use thiserror::Error; -use crate::robust_provider::Error as RobustProviderError; +use crate::RobustProviderError; #[derive(Error, Debug, Clone)] pub enum ScannerError { diff --git a/src/event_scanner/message.rs b/src/event_scanner/message.rs index ebd1081a..cbe1ea86 100644 --- a/src/event_scanner/message.rs +++ b/src/event_scanner/message.rs @@ -1,6 +1,6 @@ use alloy::{rpc::types::Log, sol_types::SolEvent}; -use crate::{ScannerError, ScannerMessage, robust_provider::Error as RobustProviderError}; +use crate::{RobustProviderError, ScannerError, ScannerMessage}; pub type Message = ScannerMessage, ScannerError>; diff --git a/src/event_scanner/scanner/common.rs b/src/event_scanner/scanner/common.rs index 6bae3d9f..7c26d16b 100644 --- a/src/event_scanner/scanner/common.rs +++ b/src/event_scanner/scanner/common.rs @@ -1,9 +1,9 @@ use std::ops::RangeInclusive; use crate::{ + RobustProvider, RobustProviderError, block_range_scanner::{MAX_BUFFERED_MESSAGES, Message as BlockRangeMessage}, event_scanner::{filter::EventFilter, listener::EventListener}, - robust_provider::{Error as RobustProviderError, RobustProvider}, types::TryStream, }; use alloy::{ diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index 1a13a5f8..8b28f4dd 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -6,13 +6,12 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use crate::{ - EventFilter, Message, ScannerError, + EventFilter, IntoRobustProvider, Message, ScannerError, block_range_scanner::{ BlockRangeScanner, ConnectedBlockRangeScanner, DEFAULT_BLOCK_CONFIRMATIONS, MAX_BUFFERED_MESSAGES, }, event_scanner::listener::EventListener, - robust_provider::IntoRobustProvider, }; mod common; diff --git a/src/lib.rs b/src/lib.rs index 496051a7..d2b1480b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,3 +15,8 @@ pub use event_scanner::{ EventFilter, EventScanner, EventScannerBuilder, Historic, LatestEvents, Live, Message, SyncFromBlock, SyncFromLatestEvents, }; + +pub use robust_provider::{ + builder::RobustProviderBuilder, error::Error as RobustProviderError, provider::RobustProvider, + types::IntoRobustProvider, +}; diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs new file mode 100644 index 00000000..02b896e7 --- /dev/null +++ b/src/robust_provider/builder.rs @@ -0,0 +1,99 @@ +use std::{marker::PhantomData, time::Duration}; + +use alloy::{network::Network, providers::Provider}; + +use crate::{ + RobustProvider, + robust_provider::{error::Error, types::IntoProvider}, +}; + +// RPC retry and timeout settings +/// Default timeout used by `RobustProvider` +pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); +/// Default maximum number of retry attempts. +pub const DEFAULT_MAX_RETRIES: usize = 3; +/// Default base delay between retries. +pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); + +#[derive(Clone)] +pub struct RobustProviderBuilder> { + providers: Vec

, + max_timeout: Duration, + max_retries: usize, + min_delay: Duration, + _network: PhantomData, +} + +impl> RobustProviderBuilder { + /// Create a new `RobustProvider` with default settings. + /// + /// The provided provider is treated as the primary provider. + #[must_use] + pub fn new(provider: P) -> Self { + Self { + providers: vec![provider], + max_timeout: DEFAULT_MAX_TIMEOUT, + max_retries: DEFAULT_MAX_RETRIES, + min_delay: DEFAULT_MIN_DELAY, + _network: PhantomData, + } + } + + /// Create a new `RobustProvider` with no retry attempts and only timeout set. + /// + /// The provided provider is treated as the primary provider. + #[must_use] + pub fn fragile(provider: P) -> Self { + Self::new(provider).max_retries(0).min_delay(Duration::ZERO) + } + + /// Add a fallback provider to the list. + /// + /// Fallback providers are used when the primary provider times out or fails. + #[must_use] + pub fn fallback(mut self, provider: P) -> Self { + self.providers.push(provider); + self + } + + /// Set the maximum timeout for RPC operations. + #[must_use] + pub fn max_timeout(mut self, timeout: Duration) -> Self { + self.max_timeout = timeout; + self + } + + /// Set the maximum number of retry attempts. + #[must_use] + pub fn max_retries(mut self, max_retries: usize) -> Self { + self.max_retries = max_retries; + self + } + + /// Set the base delay for exponential backoff retries. + #[must_use] + pub fn min_delay(mut self, min_delay: Duration) -> Self { + self.min_delay = min_delay; + self + } + + /// Build the `RobustProvider`. + /// + /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. + /// + /// # Errors + /// + /// Returns an error if any of the providers fail to connect. + pub async fn build(self) -> Result, Error> { + let mut providers = vec![]; + for p in self.providers { + providers.push(p.into_provider().await?.root().to_owned()); + } + Ok(RobustProvider { + providers, + max_timeout: self.max_timeout, + max_retries: self.max_retries, + min_delay: self.min_delay, + }) + } +} diff --git a/src/robust_provider/error.rs b/src/robust_provider/error.rs new file mode 100644 index 00000000..ad52fd2b --- /dev/null +++ b/src/robust_provider/error.rs @@ -0,0 +1,30 @@ +use std::sync::Arc; + +use alloy::{ + eips::BlockId, + transports::{RpcError, TransportErrorKind}, +}; +use thiserror::Error; +use tokio::time::error as TokioError; + +#[derive(Error, Debug, Clone)] +pub enum Error { + #[error("Operation timed out")] + Timeout, + #[error("RPC call failed after exhausting all retry attempts: {0}")] + RpcError(Arc>), + #[error("Block not found, Block Id: {0}")] + BlockNotFound(BlockId), +} + +impl From> for Error { + fn from(err: RpcError) -> Self { + Error::RpcError(Arc::new(err)) + } +} + +impl From for Error { + fn from(_: TokioError::Elapsed) -> Self { + Error::Timeout + } +} diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs new file mode 100644 index 00000000..c3959735 --- /dev/null +++ b/src/robust_provider/mod.rs @@ -0,0 +1,4 @@ +pub mod builder; +pub mod error; +pub mod provider; +pub mod types; diff --git a/src/robust_provider.rs b/src/robust_provider/provider.rs similarity index 72% rename from src/robust_provider.rs rename to src/robust_provider/provider.rs index bada3fb2..d819e910 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider/provider.rs @@ -1,216 +1,18 @@ -use std::{fmt::Debug, future::Future, marker::PhantomData, sync::Arc, time::Duration}; +use std::{fmt::Debug, future::Future, time::Duration}; use alloy::{ - eips::{BlockId, BlockNumberOrTag}, + eips::BlockNumberOrTag, network::{Ethereum, Network}, - providers::{ - DynProvider, Provider, RootProvider, - fillers::{FillProvider, TxFiller}, - layers::{CacheProvider, CallBatchProvider}, - }, + providers::{Provider, RootProvider}, pubsub::Subscription, rpc::types::{Filter, Log}, - transports::{RpcError, TransportErrorKind, http::reqwest::Url}, + transports::{RpcError, TransportErrorKind}, }; use backon::{ExponentialBuilder, Retryable}; -use thiserror::Error; -use tokio::time::{error as TokioError, timeout}; +use tokio::time::timeout; use tracing::{error, info}; -#[derive(Error, Debug, Clone)] -pub enum Error { - #[error("Operation timed out")] - Timeout, - #[error("RPC call failed after exhausting all retry attempts: {0}")] - RpcError(Arc>), - #[error("Block not found, Block Id: {0}")] - BlockNotFound(BlockId), -} - -impl From> for Error { - fn from(err: RpcError) -> Self { - Error::RpcError(Arc::new(err)) - } -} - -impl From for Error { - fn from(_: TokioError::Elapsed) -> Self { - Error::Timeout - } -} - -pub trait IntoProvider { - fn into_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl IntoProvider for RobustProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self.primary().to_owned()) - } -} - -impl IntoProvider for RootProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for &str { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self).await?) - } -} - -impl IntoProvider for Url { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self.as_str()).await?) - } -} - -impl IntoProvider for FillProvider -where - F: TxFiller, - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CacheProvider -where - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for DynProvider -where - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CallBatchProvider -where - P: Provider + 'static, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -pub trait IntoRobustProvider { - fn into_robust_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl + Send> IntoRobustProvider for P { - async fn into_robust_provider(self) -> Result, Error> { - RobustProviderBuilder::new(self).build().await - } -} - -// RPC retry and timeout settings -/// Default timeout used by `RobustProvider` -pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); -/// Default maximum number of retry attempts. -pub const DEFAULT_MAX_RETRIES: usize = 3; -/// Default base delay between retries. -pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); - -#[derive(Clone)] -pub struct RobustProviderBuilder> { - providers: Vec

, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, - _network: PhantomData, -} - -impl> RobustProviderBuilder { - /// Create a new `RobustProvider` with default settings. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn new(provider: P) -> Self { - Self { - providers: vec![provider], - max_timeout: DEFAULT_MAX_TIMEOUT, - max_retries: DEFAULT_MAX_RETRIES, - min_delay: DEFAULT_MIN_DELAY, - _network: PhantomData, - } - } - - /// Create a new `RobustProvider` with no retry attempts and only timeout set. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn fragile(provider: P) -> Self { - Self::new(provider).max_retries(0).min_delay(Duration::ZERO) - } - - /// Add a fallback provider to the list. - /// - /// Fallback providers are used when the primary provider times out or fails. - #[must_use] - pub fn fallback(mut self, provider: P) -> Self { - self.providers.push(provider); - self - } - - /// Set the maximum timeout for RPC operations. - #[must_use] - pub fn max_timeout(mut self, timeout: Duration) -> Self { - self.max_timeout = timeout; - self - } - - /// Set the maximum number of retry attempts. - #[must_use] - pub fn max_retries(mut self, max_retries: usize) -> Self { - self.max_retries = max_retries; - self - } - - /// Set the base delay for exponential backoff retries. - #[must_use] - pub fn min_delay(mut self, min_delay: Duration) -> Self { - self.min_delay = min_delay; - self - } - - /// Build the `RobustProvider`. - /// - /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. - /// - /// # Errors - /// - /// Returns an error if any of the providers fail to connect. - pub async fn build(self) -> Result, Error> { - let mut providers = vec![]; - for p in self.providers { - providers.push(p.into_provider().await?.root().to_owned()); - } - Ok(RobustProvider { - providers, - max_timeout: self.max_timeout, - max_retries: self.max_retries, - min_delay: self.min_delay, - }) - } -} +use crate::robust_provider::error::Error; /// Provider wrapper with built-in retry and timeout mechanisms. /// @@ -219,10 +21,10 @@ impl> RobustProviderBuilder { /// The first provider in the vector is treated as the primary provider. #[derive(Clone)] pub struct RobustProvider { - providers: Vec>, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, + pub(crate) providers: Vec>, + pub(crate) max_timeout: Duration, + pub(crate) max_retries: usize, + pub(crate) min_delay: Duration, } impl RobustProvider { @@ -457,6 +259,8 @@ impl RobustProvider { #[cfg(test)] mod tests { + use crate::RobustProviderBuilder; + use super::*; use alloy::{ consensus::BlockHeader, diff --git a/src/robust_provider/types.rs b/src/robust_provider/types.rs new file mode 100644 index 00000000..0c6121c4 --- /dev/null +++ b/src/robust_provider/types.rs @@ -0,0 +1,95 @@ +use crate::{ + RobustProvider, + robust_provider::{builder::RobustProviderBuilder, error::Error}, +}; +use alloy::{ + network::{Ethereum, Network}, + providers::{ + DynProvider, Provider, RootProvider, + fillers::{FillProvider, TxFiller}, + layers::{CacheProvider, CallBatchProvider}, + }, + transports::http::reqwest::Url, +}; + +pub trait IntoProvider { + fn into_provider( + self, + ) -> impl std::future::Future, Error>> + Send; +} + +impl IntoProvider for RobustProvider { + async fn into_provider(self) -> Result, Error> { + Ok(self.primary().to_owned()) + } +} + +impl IntoProvider for RootProvider { + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for &str { + async fn into_provider(self) -> Result, Error> { + Ok(RootProvider::connect(self).await?) + } +} + +impl IntoProvider for Url { + async fn into_provider(self) -> Result, Error> { + Ok(RootProvider::connect(self.as_str()).await?) + } +} + +impl IntoProvider for FillProvider +where + F: TxFiller, + P: Provider, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for CacheProvider +where + P: Provider, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for DynProvider +where + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for CallBatchProvider +where + P: Provider + 'static, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +pub trait IntoRobustProvider { + fn into_robust_provider( + self, + ) -> impl std::future::Future, Error>> + Send; +} + +impl + Send> IntoRobustProvider for P { + async fn into_robust_provider(self) -> Result, Error> { + RobustProviderBuilder::new(self).build().await + } +} diff --git a/tests/common/mod.rs b/tests/common/mod.rs index d45a1046..ad550945 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -5,6 +5,7 @@ pub mod setup_scanner; pub mod test_counter; +use event_scanner::{RobustProvider, RobustProviderBuilder}; pub(crate) use setup_scanner::{ LiveScannerSetup, SyncScannerSetup, setup_common, setup_historic_scanner, setup_latest_scanner, setup_live_scanner, setup_sync_from_latest_scanner, setup_sync_scanner, @@ -13,7 +14,6 @@ pub(crate) use test_counter::{TestCounter, deploy_counter}; use alloy::{network::Ethereum, providers::ProviderBuilder}; use alloy_node_bindings::{Anvil, AnvilInstance}; -use event_scanner::robust_provider::{RobustProvider, RobustProviderBuilder}; pub fn spawn_anvil(block_time: Option) -> anyhow::Result { let mut anvil = Anvil::new(); diff --git a/tests/common/setup_scanner.rs b/tests/common/setup_scanner.rs index 7e9fdf9e..28d0f220 100644 --- a/tests/common/setup_scanner.rs +++ b/tests/common/setup_scanner.rs @@ -7,7 +7,7 @@ use alloy::{ use alloy_node_bindings::AnvilInstance; use event_scanner::{ EventFilter, EventScanner, EventScannerBuilder, Historic, LatestEvents, Live, Message, - SyncFromBlock, SyncFromLatestEvents, robust_provider::RobustProvider, + RobustProvider, SyncFromBlock, SyncFromLatestEvents, }; use tokio_stream::wrappers::ReceiverStream; From 0e5781d2d54f29445bdb20b94ee050ef7181098f Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 21:30:04 +0900 Subject: [PATCH 02/50] feat: internal retry fallback function --- src/robust_provider.rs | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index bada3fb2..f8787def 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -8,7 +8,6 @@ use alloy::{ fillers::{FillProvider, TxFiller}, layers::{CacheProvider, CallBatchProvider}, }, - pubsub::Subscription, rpc::types::{Filter, Log}, transports::{RpcError, TransportErrorKind, http::reqwest::Url}, }; @@ -17,6 +16,8 @@ use thiserror::Error; use tokio::time::{error as TokioError, timeout}; use tracing::{error, info}; +use crate::robust_subscription::{DEFAULT_RECONNECT_INTERVAL, RobustSubscription}; + #[derive(Error, Debug, Clone)] pub enum Error { #[error("Operation timed out")] @@ -217,7 +218,7 @@ impl> RobustProviderBuilder { /// This wrapper around Alloy providers automatically handles retries, /// timeouts, and error logging for RPC calls. /// The first provider in the vector is treated as the primary provider. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct RobustProvider { providers: Vec>, max_timeout: Duration, @@ -390,13 +391,35 @@ impl RobustProvider { let mut last_error = result.unwrap_err(); + // This loop starts at index 1 automatically + match self.try_fallback_providers(providers, &operation, last_error, require_pubsub).await { + Ok(value) => { + return Ok(value); + } + Err(e) => last_error = e, + } + + // Return the last error encountered + error!("All providers failed or timed out - returning the last providers attempt's error"); + Err(last_error) + } + + async fn try_fallback_providers( + &self, + fallback_providers: impl Iterator>, + operation: F, + mut last_error: Error, + require_pubsub: bool, + ) -> Result + where + F: Fn(RootProvider) -> Fut, + Fut: Future>>, + { let num_providers = self.providers.len(); if num_providers > 1 { info!("Primary provider failed, trying fallback provider(s)"); } - - // This loop starts at index 1 automatically - for (idx, provider) in providers.enumerate() { + for (idx, provider) in fallback_providers.enumerate() { let fallback_num = idx + 1; if require_pubsub && !Self::supports_pubsub(provider) { info!("Fallback provider {} doesn't support pubsub, skipping", fallback_num); @@ -415,9 +438,7 @@ impl RobustProvider { } } } - - // Return the last error encountered - error!("All providers failed or timed out - returning the last providers attempt's error"); + // All fallbacks failed / skipped, return the last error Err(last_error) } From fdd54beb3adc98141d6d413f8d43c4ecbc3f3052 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 22:06:11 +0900 Subject: [PATCH 03/50] ref: rename retry to try with failover --- src/robust_provider.rs | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index f8787def..5347387d 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -132,7 +132,7 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); #[derive(Clone)] pub struct RobustProviderBuilder> { - providers: Vec

, + pub(crate) providers: Vec

, max_timeout: Duration, max_retries: usize, min_delay: Duration, @@ -220,7 +220,7 @@ impl> RobustProviderBuilder { /// The first provider in the vector is treated as the primary provider. #[derive(Clone, Debug)] pub struct RobustProvider { - providers: Vec>, + pub(crate) providers: Vec>, max_timeout: Duration, max_retries: usize, min_delay: Duration, @@ -251,7 +251,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByNumber called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_by_number(number).await }, false, ) @@ -273,7 +273,7 @@ impl RobustProvider { pub async fn get_block_number(&self) -> Result { info!("eth_getBlockNumber called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_number().await }, false, ) @@ -297,7 +297,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByHash called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_by_hash(hash).await }, false, ) @@ -319,7 +319,7 @@ impl RobustProvider { pub async fn get_logs(&self, filter: &Filter) -> Result, Error> { info!("eth_getLogs called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_logs(filter).await }, false, ) @@ -330,7 +330,12 @@ impl RobustProvider { result } - /// Subscribe to new block headers with retry and timeout. + /// Subscribe to new block headers with automatic failover and reconnection. + /// + /// Returns a `RobustSubscription` that automatically: + /// - Handles connection errors by switching to fallback providers + /// - Detects and recovers from lagged subscriptions + /// - Periodically attempts to reconnect to the primary provider /// /// # Errors /// @@ -338,18 +343,23 @@ impl RobustProvider { /// call fails after exhausting all retry attempts, or if the call times out. /// When fallback providers are configured, the error returned will be from the /// final provider that was attempted. - pub async fn subscribe_blocks(&self) -> Result, Error> { + pub async fn subscribe_blocks(&self) -> Result, Error> { info!("eth_subscribe called"); - let result = self - .retry_with_total_timeout( + let subscription = self + .try_operation_with_failover( move |provider| async move { provider.subscribe_blocks().await }, true, ) .await; - if let Err(e) = &result { + + if let Err(e) = &subscription { error!(error = %e, "eth_subscribe failed"); + return Err(e.clone()); } - result + + let subscription = subscription?; + + Ok(RobustSubscription::new(subscription, self.clone(), DEFAULT_RECONNECT_INTERVAL)) } /// Execute `operation` with exponential backoff and a total timeout. @@ -371,7 +381,7 @@ impl RobustProvider { /// - Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` /// is true and all providers don't support pubsub. /// - Propagates any [`RpcError`] from the underlying retries. - async fn retry_with_total_timeout( + pub(crate) async fn try_operation_with_failover( &self, operation: F, require_pubsub: bool, @@ -391,8 +401,8 @@ impl RobustProvider { let mut last_error = result.unwrap_err(); - // This loop starts at index 1 automatically - match self.try_fallback_providers(providers, &operation, last_error, require_pubsub).await { + // providers are just fallback + match self.try_fallback_providers(providers, &operation, require_pubsub, last_error).await { Ok(value) => { return Ok(value); } @@ -404,12 +414,12 @@ impl RobustProvider { Err(last_error) } - async fn try_fallback_providers( + pub(crate) async fn try_fallback_providers( &self, fallback_providers: impl Iterator>, operation: F, - mut last_error: Error, require_pubsub: bool, + mut last_error: Error, ) -> Result where F: Fn(RootProvider) -> Fut, @@ -503,7 +513,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -523,7 +533,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -546,7 +556,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result: Result<(), Error> = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); Err(TransportErrorKind::BackendGone.into()) @@ -565,7 +575,7 @@ mod tests { let provider = test_provider(max_timeout, 10, 1); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( move |_provider| async move { sleep(Duration::from_millis(max_timeout + 10)).await; Ok(42) From c22c0c3c7f99dff0a40d46819894b471607c1621 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 22:21:42 +0900 Subject: [PATCH 04/50] feat: crate modifier --- src/robust_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 5347387d..305c92cc 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -453,7 +453,7 @@ impl RobustProvider { } /// Try executing an operation with a specific provider with retry and timeout. - async fn try_provider_with_timeout( + pub(crate) async fn try_provider_with_timeout( &self, provider: &RootProvider, operation: F, From 8934fecfcc719e2ac0db1f2b8f289102861986c3 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 22:21:51 +0900 Subject: [PATCH 05/50] feat: robust subscription --- src/robust_subscription.rs | 176 +++++++++++++++++++++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 src/robust_subscription.rs diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs new file mode 100644 index 00000000..231b878e --- /dev/null +++ b/src/robust_subscription.rs @@ -0,0 +1,176 @@ +use std::time::{Duration, Instant}; + +use alloy::{ + network::Network, + providers::{Provider, RootProvider}, + pubsub::Subscription, + transports::{RpcError, TransportErrorKind}, +}; +use tokio::sync::broadcast::error::RecvError; +use tracing::{error, info, warn}; + +use crate::robust_provider::{Error, RobustProvider}; + +/// Default time interval between primary provider reconnection attempts (30 seconds) +pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); + +/// Maximum number of consecutive lags before switching providers +const MAX_LAG_COUNT: usize = 3; + +/// A robust subscription wrapper that automatically handles provider failover +/// and periodic reconnection attempts to the primary provider. +#[derive(Debug)] +pub struct RobustSubscription { + subscription: Option>, + robust_provider: RobustProvider, + reconnect_interval: Duration, + last_reconnect_attempt: Option, + consecutive_lags: usize, +} + +impl RobustSubscription { + /// Create a new [`RobustSubscription`] + pub(crate) fn new( + subscription: Subscription, + robust_provider: RobustProvider, + reconnect_interval: Duration, + ) -> Self { + Self { + subscription: Some(subscription), + robust_provider, + reconnect_interval, + last_reconnect_attempt: None, + consecutive_lags: 0, + } + } + + /// Receive the next item from the subscription with automatic failover. + /// + /// This method will: + /// - Attempt to receive from the current subscription + /// - Handle errors by switching to fallback providers + /// - Periodically attempt to reconnect to the primary provider + /// + /// # Errors + /// + /// Returns an error if all providers have been exhausted and failed. + pub async fn recv(&mut self) -> Result { + loop { + // Check if we should attempt to reconnect to primary + if self.should_reconnect_to_primary() { + info!("Attempting to reconnect to primary provider"); + if let Err(e) = self.try_reconnect_to_primary().await { + warn!(error = %e, "Failed to reconnect to primary provider"); + } else { + info!("Successfully reconnected to primary provider"); + } + } + + // Try to receive from current subscription + if let Some(subscription) = &mut self.subscription { + match subscription.recv().await { + Ok(header) => { + self.consecutive_lags = 0; + return Ok(header); + } + Err(recv_error) => match recv_error { + RecvError::Closed => { + error!("Subscription channel closed, switching provider"); + // NOTE: Not sure what error to pass here + let error = RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + RecvError::Lagged(skipped) => { + self.consecutive_lags += 1; + warn!( + skipped = skipped, + consecutive_lags = self.consecutive_lags, + "Subscription lagged" + ); + + if self.consecutive_lags >= MAX_LAG_COUNT { + error!("Too many consecutive lags, switching provider"); + // NOTE: Not sure what error to pass here + let error = + RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + } + }, + } + } else { + // No subscription available + return Err(RpcError::Transport(TransportErrorKind::BackendGone).into()); + } + } + } + + /// Check if we should attempt to reconnect to the primary provider + fn should_reconnect_to_primary(&self) -> bool { + // Only attempt reconnection if enough time has passed since last attempt + // The RobustProvider will try the primary provider first automatically + match self.last_reconnect_attempt { + None => false, + Some(last_attempt) => last_attempt.elapsed() >= self.reconnect_interval, + } + } + + /// Attempt to reconnect to the primary provider + async fn try_reconnect_to_primary(&mut self) -> Result<(), Error> { + self.last_reconnect_attempt = Some(Instant::now()); + + let operation = + move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + + let primary = self.robust_provider.primary(); + let subscription = + self.robust_provider.try_provider_with_timeout(primary, &operation).await; + + if let Err(e) = &subscription { + error!(error = %e, "eth_subscribe failed"); + return Err(e.clone()); + } + + let subscription = subscription?; + self.subscription = Some(subscription); + Ok(()) + } + + /// Switch to a fallback provider + async fn switch_to_fallback(&mut self, last_error: Error) -> Result<(), Error> { + // Mark that we need reconnection attempts + if self.last_reconnect_attempt.is_none() { + self.last_reconnect_attempt = Some(Instant::now()); + } + + let operation = + move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + let subscription = self + .robust_provider + .try_fallback_providers( + self.robust_provider.providers.iter().skip(1), + &operation, + true, + last_error, + ) + .await; + + if let Err(e) = &subscription { + error!(error = %e, "eth_subscribe failed"); + return Err(e.clone()); + } + + let subscription = subscription?; + self.subscription = Some(subscription); + Ok(()) + } + + /// Check if the subscription channel is empty (no pending messages) + #[must_use] + pub fn is_empty(&self) -> bool { + match &self.subscription { + Some(sub) => sub.is_empty(), + None => true, + } + } +} From 193baa567d81ddca3ce4eec9b7f5af884c601355 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 22:22:13 +0900 Subject: [PATCH 06/50] feat: add robust subscription to block range scanner --- src/block_range_scanner.rs | 26 +++++++++++++++++++++----- src/lib.rs | 1 + 2 files changed, 22 insertions(+), 5 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index da742503..3854e723 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -64,12 +64,13 @@ use tokio::{ sync::{mpsc, oneshot}, try_join, }; -use tokio_stream::{StreamExt, wrappers::ReceiverStream}; +use tokio_stream::wrappers::ReceiverStream; use crate::{ ScannerMessage, error::ScannerError, robust_provider::{Error as RobustProviderError, IntoRobustProvider, RobustProvider}, + robust_subscription::RobustSubscription, types::{ScannerStatus, TryStream}, }; use alloy::{ @@ -77,7 +78,6 @@ use alloy::{ eips::BlockNumberOrTag, network::{BlockResponse, Network, primitives::HeaderResponse}, primitives::{B256, BlockNumber}, - pubsub::Subscription, transports::{RpcError, TransportErrorKind}, }; use tracing::{debug, error, info, warn}; @@ -611,17 +611,33 @@ impl Service { async fn stream_live_blocks( mut range_start: BlockNumber, - subscription: Subscription, + mut subscription: RobustSubscription, sender: mpsc::Sender, block_confirmations: u64, max_block_range: u64, ) { // ensure we start streaming only after the expected_next_block cutoff let cutoff = range_start; - let mut stream = subscription.into_stream().skip_while(|header| header.number() < cutoff); - while let Some(incoming_block) = stream.next().await { + loop { + // Use recv() to get the next block with automatic failover + let incoming_block = match subscription.recv().await { + Ok(block) => block, + Err(e) => { + error!(error = %e, "Failed to receive block from subscription"); + // Send error to subscriber and terminate + _ = sender.try_stream(e).await; + return; + } + }; + let incoming_block_num = incoming_block.number(); + + // Skip blocks before the cutoff + if incoming_block_num < cutoff { + continue; + } + info!(block_number = incoming_block_num, "Received block header"); if incoming_block_num < range_start { diff --git a/src/lib.rs b/src/lib.rs index 496051a7..b797eb4a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,7 @@ pub mod block_range_scanner; pub mod robust_provider; +pub mod robust_subscription; #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; From 97ca46cdfdf828a73ed4521dff53ba0ee3230f74 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 22:51:32 +0900 Subject: [PATCH 07/50] feat: merge latest sub --- src/block_range_scanner.rs | 5 +- src/error.rs | 2 +- src/event_scanner/message.rs | 2 +- src/event_scanner/scanner/common.rs | 3 +- src/lib.rs | 5 +- src/robust_provider/mod.rs | 3 + src/robust_provider/provider.rs | 222 ++---------------- .../robust_subscription.rs | 2 +- 8 files changed, 27 insertions(+), 217 deletions(-) rename src/{ => robust_provider}/robust_subscription.rs (99%) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index e4f411e9..3331ee00 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -67,10 +67,9 @@ use tokio::{ use tokio_stream::wrappers::ReceiverStream; use crate::{ - IntoRobustProvider, RobustProvider, RobustProviderError, ScannerMessage, + IntoRobustProvider, RobustProvider, RobustSubscription, ScannerMessage, error::ScannerError, - robust_provider::{Error as RobustProviderError, IntoRobustProvider, RobustProvider}, - robust_subscription::RobustSubscription, + robust_provider::Error as RobustProviderError, types::{ScannerStatus, TryStream}, }; use alloy::{ diff --git a/src/error.rs b/src/error.rs index 44a1059d..e7ebedb5 100644 --- a/src/error.rs +++ b/src/error.rs @@ -6,7 +6,7 @@ use alloy::{ }; use thiserror::Error; -use crate::RobustProviderError; +use crate::robust_provider::Error as RobustProviderError; #[derive(Error, Debug, Clone)] pub enum ScannerError { diff --git a/src/event_scanner/message.rs b/src/event_scanner/message.rs index cbe1ea86..ebd1081a 100644 --- a/src/event_scanner/message.rs +++ b/src/event_scanner/message.rs @@ -1,6 +1,6 @@ use alloy::{rpc::types::Log, sol_types::SolEvent}; -use crate::{RobustProviderError, ScannerError, ScannerMessage}; +use crate::{ScannerError, ScannerMessage, robust_provider::Error as RobustProviderError}; pub type Message = ScannerMessage, ScannerError>; diff --git a/src/event_scanner/scanner/common.rs b/src/event_scanner/scanner/common.rs index 7c26d16b..89b379b3 100644 --- a/src/event_scanner/scanner/common.rs +++ b/src/event_scanner/scanner/common.rs @@ -1,9 +1,10 @@ use std::ops::RangeInclusive; use crate::{ - RobustProvider, RobustProviderError, + RobustProvider, block_range_scanner::{MAX_BUFFERED_MESSAGES, Message as BlockRangeMessage}, event_scanner::{filter::EventFilter, listener::EventListener}, + robust_provider::Error as RobustProviderError, types::TryStream, }; use alloy::{ diff --git a/src/lib.rs b/src/lib.rs index 47397aca..7da4b438 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,7 +1,6 @@ pub mod block_range_scanner; pub mod robust_provider; -pub mod robust_subscription; #[cfg(any(test, feature = "test-utils"))] pub mod test_utils; @@ -18,6 +17,6 @@ pub use event_scanner::{ }; pub use robust_provider::{ - builder::RobustProviderBuilder, error::Error as RobustProviderError, provider::RobustProvider, - types::IntoRobustProvider, + builder::RobustProviderBuilder, provider::RobustProvider, + robust_subscription::RobustSubscription, types::IntoRobustProvider, }; diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index c3959735..86f35ed2 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -1,4 +1,7 @@ pub mod builder; pub mod error; pub mod provider; +pub mod robust_subscription; pub mod types; + +pub use error::Error; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index d970b26f..8ac2e927 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -1,13 +1,9 @@ -use std::{fmt::Debug, future::Future, time::Duration}; +use std::{fmt::Debug, time::Duration}; use alloy::{ eips::BlockNumberOrTag, network::{Ethereum, Network}, - providers::{ - DynProvider, Provider, RootProvider, - fillers::{FillProvider, TxFiller}, - layers::{CacheProvider, CallBatchProvider}, - }, + providers::{Provider, RootProvider}, rpc::types::{Filter, Log}, transports::{RpcError, TransportErrorKind}, }; @@ -15,202 +11,10 @@ use backon::{ExponentialBuilder, Retryable}; use tokio::time::timeout; use tracing::{error, info}; -use crate::robust_subscription::{DEFAULT_RECONNECT_INTERVAL, RobustSubscription}; - -#[derive(Error, Debug, Clone)] -pub enum Error { - #[error("Operation timed out")] - Timeout, - #[error("RPC call failed after exhausting all retry attempts: {0}")] - RpcError(Arc>), - #[error("Block not found, Block Id: {0}")] - BlockNotFound(BlockId), -} - -impl From> for Error { - fn from(err: RpcError) -> Self { - Error::RpcError(Arc::new(err)) - } -} - -impl From for Error { - fn from(_: TokioError::Elapsed) -> Self { - Error::Timeout - } -} - -pub trait IntoProvider { - fn into_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl IntoProvider for RobustProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self.primary().to_owned()) - } -} - -impl IntoProvider for RootProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for &str { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self).await?) - } -} - -impl IntoProvider for Url { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self.as_str()).await?) - } -} - -impl IntoProvider for FillProvider -where - F: TxFiller, - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CacheProvider -where - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for DynProvider -where - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CallBatchProvider -where - P: Provider + 'static, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -pub trait IntoRobustProvider { - fn into_robust_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl + Send> IntoRobustProvider for P { - async fn into_robust_provider(self) -> Result, Error> { - RobustProviderBuilder::new(self).build().await - } -} - -// RPC retry and timeout settings -/// Default timeout used by `RobustProvider` -pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); -/// Default maximum number of retry attempts. -pub const DEFAULT_MAX_RETRIES: usize = 3; -/// Default base delay between retries. -pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); - -#[derive(Clone)] -pub struct RobustProviderBuilder> { - pub(crate) providers: Vec

, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, - _network: PhantomData, -} - -impl> RobustProviderBuilder { - /// Create a new `RobustProvider` with default settings. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn new(provider: P) -> Self { - Self { - providers: vec![provider], - max_timeout: DEFAULT_MAX_TIMEOUT, - max_retries: DEFAULT_MAX_RETRIES, - min_delay: DEFAULT_MIN_DELAY, - _network: PhantomData, - } - } - - /// Create a new `RobustProvider` with no retry attempts and only timeout set. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn fragile(provider: P) -> Self { - Self::new(provider).max_retries(0).min_delay(Duration::ZERO) - } - - /// Add a fallback provider to the list. - /// - /// Fallback providers are used when the primary provider times out or fails. - #[must_use] - pub fn fallback(mut self, provider: P) -> Self { - self.providers.push(provider); - self - } - - /// Set the maximum timeout for RPC operations. - #[must_use] - pub fn max_timeout(mut self, timeout: Duration) -> Self { - self.max_timeout = timeout; - self - } - - /// Set the maximum number of retry attempts. - #[must_use] - pub fn max_retries(mut self, max_retries: usize) -> Self { - self.max_retries = max_retries; - self - } - - /// Set the base delay for exponential backoff retries. - #[must_use] - pub fn min_delay(mut self, min_delay: Duration) -> Self { - self.min_delay = min_delay; - self - } - - /// Build the `RobustProvider`. - /// - /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. - /// - /// # Errors - /// - /// Returns an error if any of the providers fail to connect. - pub async fn build(self) -> Result, Error> { - let mut providers = vec![]; - for p in self.providers { - providers.push(p.into_provider().await?.root().to_owned()); - } - Ok(RobustProvider { - providers, - max_timeout: self.max_timeout, - max_retries: self.max_retries, - min_delay: self.min_delay, - }) - } -} +use crate::{ + RobustSubscription, + robust_provider::{Error, robust_subscription::DEFAULT_RECONNECT_INTERVAL}, +}; /// Provider wrapper with built-in retry and timeout mechanisms. /// @@ -220,9 +24,9 @@ impl> RobustProviderBuilder { #[derive(Clone, Debug)] pub struct RobustProvider { pub(crate) providers: Vec>, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, + pub(crate) max_timeout: Duration, + pub(crate) max_retries: usize, + pub(crate) min_delay: Duration, } impl RobustProvider { @@ -487,15 +291,19 @@ impl RobustProvider { #[cfg(test)] mod tests { - use crate::RobustProviderBuilder; + use crate::{RobustProviderBuilder, robust_provider::Error}; use super::*; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, + transports::{RpcError, TransportErrorKind}, }; use alloy_node_bindings::Anvil; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::{ + sync::atomic::{AtomicUsize, Ordering}, + time::Duration, + }; use tokio::time::sleep; fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { diff --git a/src/robust_subscription.rs b/src/robust_provider/robust_subscription.rs similarity index 99% rename from src/robust_subscription.rs rename to src/robust_provider/robust_subscription.rs index 231b878e..f812fee0 100644 --- a/src/robust_subscription.rs +++ b/src/robust_provider/robust_subscription.rs @@ -9,7 +9,7 @@ use alloy::{ use tokio::sync::broadcast::error::RecvError; use tracing::{error, info, warn}; -use crate::robust_provider::{Error, RobustProvider}; +use crate::{RobustProvider, robust_provider::Error}; /// Default time interval between primary provider reconnection attempts (30 seconds) pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); From 70fc3003a75ef3fc2dd9fb60bead12c604c604f4 Mon Sep 17 00:00:00 2001 From: Leo Date: Mon, 10 Nov 2025 23:03:15 +0900 Subject: [PATCH 08/50] feat: impl stream on RobustSuscscription --- src/block_range_scanner.rs | 19 +++++++------- src/robust_subscription.rs | 53 +++++++++++++++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 11 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 3854e723..5a709791 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -64,7 +64,7 @@ use tokio::{ sync::{mpsc, oneshot}, try_join, }; -use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ ScannerMessage, @@ -611,7 +611,7 @@ impl Service { async fn stream_live_blocks( mut range_start: BlockNumber, - mut subscription: RobustSubscription, + subscription: RobustSubscription, sender: mpsc::Sender, block_confirmations: u64, max_block_range: u64, @@ -619,9 +619,13 @@ impl Service { // ensure we start streaming only after the expected_next_block cutoff let cutoff = range_start; - loop { - // Use recv() to get the next block with automatic failover - let incoming_block = match subscription.recv().await { + let mut stream = subscription.into_stream().skip_while(|result| match result { + Ok(header) => header.number() < cutoff, + Err(_) => false, + }); + + while let Some(result) = stream.next().await { + let incoming_block = match result { Ok(block) => block, Err(e) => { error!(error = %e, "Failed to receive block from subscription"); @@ -633,11 +637,6 @@ impl Service { let incoming_block_num = incoming_block.number(); - // Skip blocks before the cutoff - if incoming_block_num < cutoff { - continue; - } - info!(block_number = incoming_block_num, "Received block header"); if incoming_block_num < range_start { diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index 231b878e..f8c2de73 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -1,4 +1,8 @@ -use std::time::{Duration, Instant}; +use std::{ + pin::Pin, + task::{Context, Poll}, + time::{Duration, Instant}, +}; use alloy::{ network::Network, @@ -7,6 +11,7 @@ use alloy::{ transports::{RpcError, TransportErrorKind}, }; use tokio::sync::broadcast::error::RecvError; +use tokio_stream::Stream; use tracing::{error, info, warn}; use crate::robust_provider::{Error, RobustProvider}; @@ -173,4 +178,50 @@ impl RobustSubscription { None => true, } } + + /// Convert this `RobustSubscription` into a `Stream`. + /// + /// This allows using standard stream combinators like `skip_while`, `filter`, etc. + /// The stream will automatically handle failover and reconnection attempts. + /// + /// # Example + /// + /// ```ignore + /// use tokio_stream::StreamExt; + /// + /// let mut stream = subscription.into_stream() + /// .skip_while(|result| matches!(result, Ok(header) if header.number() < cutoff)) + /// .filter_map(|result| result.ok()); + /// + /// while let Some(block) = stream.next().await { + /// // Process block + /// } + /// ``` + #[must_use] + pub fn into_stream(self) -> RobustSubscriptionStream { + RobustSubscriptionStream { inner: self } + } +} + +/// A `Stream` wrapper around `RobustSubscription`. +/// +/// This struct implements the `Stream` trait, allowing you to use standard stream +/// combinators from `tokio_stream::StreamExt`. +pub struct RobustSubscriptionStream { + inner: RobustSubscription, +} + +impl Stream for RobustSubscriptionStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + let fut = self.inner.recv(); + tokio::pin!(fut); + + match fut.poll(cx) { + Poll::Ready(Ok(header)) => Poll::Ready(Some(Ok(header))), + Poll::Ready(Err(e)) => Poll::Ready(Some(Err(e))), + Poll::Pending => Poll::Pending, + } + } } From 77ce10c3f2a78e49ddadbe25c8f85ea3168b1c62 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 17:13:57 +0900 Subject: [PATCH 09/50] feat: impl stream on robust subsription --- src/robust_subscription.rs | 52 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index 231b878e..4d2510ae 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -1,4 +1,8 @@ -use std::time::{Duration, Instant}; +use std::{ + pin::Pin, + task::{Context, Poll}, + time::{Duration, Instant}, +}; use alloy::{ network::Network, @@ -6,7 +10,8 @@ use alloy::{ pubsub::Subscription, transports::{RpcError, TransportErrorKind}, }; -use tokio::sync::broadcast::error::RecvError; +use tokio::sync::{broadcast::error::RecvError, mpsc}; +use tokio_stream::{Stream, wrappers::UnboundedReceiverStream}; use tracing::{error, info, warn}; use crate::robust_provider::{Error, RobustProvider}; @@ -173,4 +178,47 @@ impl RobustSubscription { None => true, } } + + /// Convert the subscription into a stream. + /// + /// This spawns a background task that continuously receives from the subscription + /// and forwards items to a channel, which is then wrapped in a Stream. + #[must_use] + pub fn into_stream(mut self) -> RobustSubscriptionStream { + let (tx, rx) = mpsc::unbounded_channel(); + + // Spawn a background task to handle the recv loop + tokio::spawn(async move { + loop { + match self.recv().await { + Ok(item) => { + if tx.send(Ok(item)).is_err() { + // Receiver dropped, exit the loop + break; + } + } + Err(e) => { + // Send the error and exit + let _ = tx.send(Err(e)); + break; + } + } + } + }); + + RobustSubscriptionStream { inner: UnboundedReceiverStream::new(rx) } + } +} + +/// A stream wrapper around [`RobustSubscription`] that implements the [`Stream`] trait. +pub struct RobustSubscriptionStream { + inner: UnboundedReceiverStream>, +} + +impl Stream for RobustSubscriptionStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_next(cx) + } } From 3443501f4462523df188a1587574e06ed571ea60 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 17:39:00 +0900 Subject: [PATCH 10/50] feat: add robust stream logic to block range scanner --- src/block_range_scanner.rs | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 3854e723..eb48a63f 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -64,7 +64,7 @@ use tokio::{ sync::{mpsc, oneshot}, try_join, }; -use tokio_stream::wrappers::ReceiverStream; +use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ ScannerMessage, @@ -611,33 +611,30 @@ impl Service { async fn stream_live_blocks( mut range_start: BlockNumber, - mut subscription: RobustSubscription, + subscription: RobustSubscription, sender: mpsc::Sender, block_confirmations: u64, max_block_range: u64, ) { // ensure we start streaming only after the expected_next_block cutoff let cutoff = range_start; + let mut stream = subscription.into_stream().skip_while(|result| match result { + Ok(header) => header.number() < cutoff, + Err(_) => false, + }); - loop { - // Use recv() to get the next block with automatic failover - let incoming_block = match subscription.recv().await { + while let Some(result) = stream.next().await { + let incoming_block = match result { Ok(block) => block, Err(e) => { - error!(error = %e, "Failed to receive block from subscription"); - // Send error to subscriber and terminate - _ = sender.try_stream(e).await; + error!(error = %e, "Error receiving block from stream"); + // Error from subscription, exit the stream + _ = sender.try_stream(e); return; } }; let incoming_block_num = incoming_block.number(); - - // Skip blocks before the cutoff - if incoming_block_num < cutoff { - continue; - } - info!(block_number = incoming_block_num, "Received block header"); if incoming_block_num < range_start { From 19aeb9c6556613d04c1ab2ef92339dbe8f7124c1 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 17:39:12 +0900 Subject: [PATCH 11/50] ref: add todo about unbounded channel ref: remove comment --- src/robust_subscription.rs | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index 4d2510ae..913190c4 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -110,7 +110,6 @@ impl RobustSubscription { } } - /// Check if we should attempt to reconnect to the primary provider fn should_reconnect_to_primary(&self) -> bool { // Only attempt reconnection if enough time has passed since last attempt // The RobustProvider will try the primary provider first automatically @@ -120,7 +119,6 @@ impl RobustSubscription { } } - /// Attempt to reconnect to the primary provider async fn try_reconnect_to_primary(&mut self) -> Result<(), Error> { self.last_reconnect_attempt = Some(Instant::now()); @@ -141,9 +139,7 @@ impl RobustSubscription { Ok(()) } - /// Switch to a fallback provider async fn switch_to_fallback(&mut self, last_error: Error) -> Result<(), Error> { - // Mark that we need reconnection attempts if self.last_reconnect_attempt.is_none() { self.last_reconnect_attempt = Some(Instant::now()); } @@ -185,9 +181,9 @@ impl RobustSubscription { /// and forwards items to a channel, which is then wrapped in a Stream. #[must_use] pub fn into_stream(mut self) -> RobustSubscriptionStream { + // TODO: This shouldnt be unbounded let (tx, rx) = mpsc::unbounded_channel(); - // Spawn a background task to handle the recv loop tokio::spawn(async move { loop { match self.recv().await { From e65f5d18fd3e33ed2f9638b2cf9313b4902d4c12 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 18:02:21 +0900 Subject: [PATCH 12/50] feat: add sub timeout (seperate to rpc timeout) --- src/robust_provider.rs | 23 ++++++++++++-- src/robust_subscription.rs | 63 +++++++++++++++++++++++--------------- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 305c92cc..2c4fdb28 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -125,6 +125,8 @@ impl + Send> IntoRobustProvider for P { // RPC retry and timeout settings /// Default timeout used by `RobustProvider` pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); +/// Default timeout for subscriptions (longer to accommodate slow block times) +pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_mins(2); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; /// Default base delay between retries. @@ -134,6 +136,7 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); pub struct RobustProviderBuilder> { pub(crate) providers: Vec

, max_timeout: Duration, + subscription_timeout: Duration, max_retries: usize, min_delay: Duration, _network: PhantomData, @@ -148,6 +151,7 @@ impl> RobustProviderBuilder { Self { providers: vec![provider], max_timeout: DEFAULT_MAX_TIMEOUT, + subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, min_delay: DEFAULT_MIN_DELAY, _network: PhantomData, @@ -178,6 +182,16 @@ impl> RobustProviderBuilder { self } + /// Set the timeout for subscription operations. + /// + /// This should be set higher than `max_timeout` to accommodate chains with slow block times. + /// Default is 2 minutes. + #[must_use] + pub fn subscription_timeout(mut self, timeout: Duration) -> Self { + self.subscription_timeout = timeout; + self + } + /// Set the maximum number of retry attempts. #[must_use] pub fn max_retries(mut self, max_retries: usize) -> Self { @@ -207,6 +221,7 @@ impl> RobustProviderBuilder { Ok(RobustProvider { providers, max_timeout: self.max_timeout, + subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, min_delay: self.min_delay, }) @@ -221,9 +236,10 @@ impl> RobustProviderBuilder { #[derive(Clone, Debug)] pub struct RobustProvider { pub(crate) providers: Vec>, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, + pub(crate) max_timeout: Duration, + pub(crate) subscription_timeout: Duration, + pub(crate) max_retries: usize, + pub(crate) min_delay: Duration, } impl RobustProvider { @@ -501,6 +517,7 @@ mod tests { RobustProvider { providers: vec![RootProvider::new_http("http://localhost:8545".parse().unwrap())], max_timeout: Duration::from_millis(timeout), + subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, min_delay: Duration::from_millis(min_delay), } diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index 913190c4..a6a2d2ae 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -10,7 +10,10 @@ use alloy::{ pubsub::Subscription, transports::{RpcError, TransportErrorKind}, }; -use tokio::sync::{broadcast::error::RecvError, mpsc}; +use tokio::{ + sync::{broadcast::error::RecvError, mpsc}, + time::timeout, +}; use tokio_stream::{Stream, wrappers::UnboundedReceiverStream}; use tracing::{error, info, warn}; @@ -71,37 +74,47 @@ impl RobustSubscription { } } - // Try to receive from current subscription + // Try to receive from current subscription with timeout if let Some(subscription) = &mut self.subscription { - match subscription.recv().await { - Ok(header) => { - self.consecutive_lags = 0; - return Ok(header); - } - Err(recv_error) => match recv_error { - RecvError::Closed => { - error!("Subscription channel closed, switching provider"); - // NOTE: Not sure what error to pass here - let error = RpcError::Transport(TransportErrorKind::BackendGone).into(); - self.switch_to_fallback(error).await?; + let subscription_timeout = self.robust_provider.subscription_timeout; + match timeout(subscription_timeout, subscription.recv()).await { + Ok(recv_result) => match recv_result { + Ok(header) => { + self.consecutive_lags = 0; + return Ok(header); } - RecvError::Lagged(skipped) => { - self.consecutive_lags += 1; - warn!( - skipped = skipped, - consecutive_lags = self.consecutive_lags, - "Subscription lagged" - ); - - if self.consecutive_lags >= MAX_LAG_COUNT { - error!("Too many consecutive lags, switching provider"); - // NOTE: Not sure what error to pass here + Err(recv_error) => match recv_error { + RecvError::Closed => { + error!("Subscription channel closed, switching provider"); let error = RpcError::Transport(TransportErrorKind::BackendGone).into(); self.switch_to_fallback(error).await?; } - } + RecvError::Lagged(skipped) => { + self.consecutive_lags += 1; + warn!( + skipped = skipped, + consecutive_lags = self.consecutive_lags, + "Subscription lagged" + ); + + if self.consecutive_lags >= MAX_LAG_COUNT { + error!("Too many consecutive lags, switching provider"); + let error = + RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + } + }, }, + Err(e) => { + // Timeout occurred - no block received within subscription_timeout + error!( + timeout_secs = subscription_timeout.as_secs(), + "Subscription timeout - no block received, switching provider" + ); + self.switch_to_fallback(e.into()).await?; + } } } else { // No subscription available From 4c62598fa26c3e72833c154a9da03125bbb174a7 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 19:08:13 +0900 Subject: [PATCH 13/50] ref: better comment --- src/robust_subscription.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index a6a2d2ae..ea61ac90 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -194,7 +194,8 @@ impl RobustSubscription { /// and forwards items to a channel, which is then wrapped in a Stream. #[must_use] pub fn into_stream(mut self) -> RobustSubscriptionStream { - // TODO: This shouldnt be unbounded + // TODO: This shouldnt be unbounded need choose an appropriate bound (Maybe same as max + // buffer probably should be bigger) let (tx, rx) = mpsc::unbounded_channel(); tokio::spawn(async move { From c64d700d07ae426a895d255a60980fbf98abc279 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 19:11:51 +0900 Subject: [PATCH 14/50] test: test subscription failover --- src/robust_provider.rs | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 2c4fdb28..d4d730da 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -512,6 +512,7 @@ mod tests { use alloy_node_bindings::Anvil; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::time::sleep; + use tokio_stream::StreamExt; fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { RobustProvider { @@ -718,4 +719,48 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_stream_with_failover() -> anyhow::Result<()> { + let mut anvil_1 = Some(Anvil::new().block_time(1).try_spawn()?); + + let ws_provider = ProviderBuilder::new() + .connect(anvil_1.as_ref().unwrap().ws_endpoint_url().as_str()) + .await?; + + let anvil_2 = Anvil::new().block_time(1).try_spawn()?; + + let ws_provider_2 = + ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; + + let robust = RobustProviderBuilder::fragile(ws_provider.clone()) + .fallback(ws_provider_2) + .subscription_timeout(Duration::from_secs(3)) + .build() + .await?; + + let subscription = robust.subscribe_blocks().await?; + let mut stream = subscription.into_stream(); + + while let Some(result) = stream.next().await { + let Ok(block) = result else { + break; + }; + + let block_number = block.number(); + + // At block 10, drop the primary provider to test failover + if block_number == 10 && + let Some(anvil) = anvil_1.take() + { + drop(anvil); + } + + if block_number >= 20 { + break; + } + } + + Ok(()) + } } From 2d53712b1d2f317f639b0b0704bed1202e7062a6 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 19:14:02 +0900 Subject: [PATCH 15/50] fix: increase test speed --- src/robust_provider.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index d4d730da..da69f203 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -722,20 +722,20 @@ mod tests { #[tokio::test] async fn test_stream_with_failover() -> anyhow::Result<()> { - let mut anvil_1 = Some(Anvil::new().block_time(1).try_spawn()?); + let mut anvil_1 = Some(Anvil::new().block_time_f64(0.1).try_spawn()?); let ws_provider = ProviderBuilder::new() .connect(anvil_1.as_ref().unwrap().ws_endpoint_url().as_str()) .await?; - let anvil_2 = Anvil::new().block_time(1).try_spawn()?; + let anvil_2 = Anvil::new().block_time_f64(0.1).try_spawn()?; let ws_provider_2 = ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(ws_provider_2) - .subscription_timeout(Duration::from_secs(3)) + .subscription_timeout(Duration::from_millis(500)) .build() .await?; @@ -756,7 +756,7 @@ mod tests { drop(anvil); } - if block_number >= 20 { + if block_number >= 11 { break; } } From efd857d3ae7cca5f82c5bb950f2da908b802cebd Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 20:17:51 +0900 Subject: [PATCH 16/50] fix: from_min --> from_sec --- src/robust_provider.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index da69f203..9198b01e 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -126,7 +126,7 @@ impl + Send> IntoRobustProvider for P { /// Default timeout used by `RobustProvider` pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); /// Default timeout for subscriptions (longer to accommodate slow block times) -pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_mins(2); +pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(120); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; /// Default base delay between retries. From 4cb95b157c7e3b3b119c852277f0d775851353e9 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 20:28:22 +0900 Subject: [PATCH 17/50] fix: imports --- src/robust_provider/types.rs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/src/robust_provider/types.rs b/src/robust_provider/types.rs index 74ff3c4c..292e7560 100644 --- a/src/robust_provider/types.rs +++ b/src/robust_provider/types.rs @@ -1,3 +1,15 @@ +use alloy::{ + network::{Ethereum, Network}, + providers::{ + DynProvider, Provider, RootProvider, + fillers::{FillProvider, TxFiller}, + layers::{CacheProvider, CallBatchProvider}, + }, + transports::http::reqwest::Url, +}; + +use crate::{RobustProvider, RobustProviderBuilder, robust_provider::Error}; + pub trait IntoProvider { fn into_provider( self, From 71a3b3b825af6adfbbc623a91fdc800be47ed6a9 Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 20:17:51 +0900 Subject: [PATCH 18/50] fix: from_min --> from_sec ref: better comments fix: remove bracket --- src/robust_provider.rs | 6 +++--- src/robust_subscription.rs | 7 ++----- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index da69f203..c5794dcd 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -126,7 +126,7 @@ impl + Send> IntoRobustProvider for P { /// Default timeout used by `RobustProvider` pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); /// Default timeout for subscriptions (longer to accommodate slow block times) -pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_mins(2); +pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(120); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; /// Default base delay between retries. @@ -143,7 +143,7 @@ pub struct RobustProviderBuilder> { } impl> RobustProviderBuilder { - /// Create a new `RobustProvider` with default settings. + /// Create a new [`RobustProvider`] with default settings. /// /// The provided provider is treated as the primary provider. #[must_use] @@ -158,7 +158,7 @@ impl> RobustProviderBuilder { } } - /// Create a new `RobustProvider` with no retry attempts and only timeout set. + /// Create a new [`RobustProvider`] with no retry attempts and only timeout set. /// /// The provided provider is treated as the primary provider. #[must_use] diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index ea61ac90..aca0131a 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -19,7 +19,7 @@ use tracing::{error, info, warn}; use crate::robust_provider::{Error, RobustProvider}; -/// Default time interval between primary provider reconnection attempts (30 seconds) +/// Default time interval between primary provider reconnection attempts pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); /// Maximum number of consecutive lags before switching providers @@ -58,13 +58,13 @@ impl RobustSubscription { /// - Attempt to receive from the current subscription /// - Handle errors by switching to fallback providers /// - Periodically attempt to reconnect to the primary provider + /// - Will switch to fallback providers if subscription timeout is exhausted /// /// # Errors /// /// Returns an error if all providers have been exhausted and failed. pub async fn recv(&mut self) -> Result { loop { - // Check if we should attempt to reconnect to primary if self.should_reconnect_to_primary() { info!("Attempting to reconnect to primary provider"); if let Err(e) = self.try_reconnect_to_primary().await { @@ -74,7 +74,6 @@ impl RobustSubscription { } } - // Try to receive from current subscription with timeout if let Some(subscription) = &mut self.subscription { let subscription_timeout = self.robust_provider.subscription_timeout; match timeout(subscription_timeout, subscription.recv()).await { @@ -108,7 +107,6 @@ impl RobustSubscription { }, }, Err(e) => { - // Timeout occurred - no block received within subscription_timeout error!( timeout_secs = subscription_timeout.as_secs(), "Subscription timeout - no block received, switching provider" @@ -125,7 +123,6 @@ impl RobustSubscription { fn should_reconnect_to_primary(&self) -> bool { // Only attempt reconnection if enough time has passed since last attempt - // The RobustProvider will try the primary provider first automatically match self.last_reconnect_attempt { None => false, Some(last_attempt) => last_attempt.elapsed() >= self.reconnect_interval, From 6c5932197f2abe176cd588e2c47630391b79afee Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 20:39:42 +0900 Subject: [PATCH 19/50] fix: doc --- src/block_range_scanner.rs | 2 +- src/event_scanner/scanner/mod.rs | 10 +++++----- src/event_scanner/scanner/sync/mod.rs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index ed4fc5a9..3cf86148 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -12,7 +12,7 @@ //! BlockRangeScanner, BlockRangeScannerClient, DEFAULT_BLOCK_CONFIRMATIONS, //! DEFAULT_MAX_BLOCK_RANGE, Message, //! }, -//! robust_provider::RobustProviderBuilder, +//! RobustProviderBuilder, //! }; //! use tokio::time::Duration; //! use tracing::{error, info}; diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index 8b28f4dd..e65af101 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -77,7 +77,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -103,7 +103,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Stream events between blocks [1_000_000, 2_000_000] @@ -143,7 +143,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -231,7 +231,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, primitives::Address, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -258,7 +258,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Collect the latest 5 events between blocks [1_000_000, 1_100_000] diff --git a/src/event_scanner/scanner/sync/mod.rs b/src/event_scanner/scanner/sync/mod.rs index ffdae097..c51607c4 100644 --- a/src/event_scanner/scanner/sync/mod.rs +++ b/src/event_scanner/scanner/sync/mod.rs @@ -25,7 +25,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -123,7 +123,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -163,7 +163,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, eips::BlockNumberOrTag, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Sync from genesis block From 212ef73c683fa6d732dcfe8bdff5f35b0e1461ae Mon Sep 17 00:00:00 2001 From: Leo Date: Tue, 11 Nov 2025 21:10:50 +0900 Subject: [PATCH 20/50] fix: format --- src/block_range_scanner.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 3cf86148..c39c95e9 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -7,12 +7,11 @@ //! //! use alloy::providers::{Provider, ProviderBuilder}; //! use event_scanner::{ -//! ScannerError, +//! RobustProviderBuilder, ScannerError, //! block_range_scanner::{ //! BlockRangeScanner, BlockRangeScannerClient, DEFAULT_BLOCK_CONFIRMATIONS, //! DEFAULT_MAX_BLOCK_RANGE, Message, //! }, -//! RobustProviderBuilder, //! }; //! use tokio::time::Duration; //! use tracing::{error, info}; From 8a47d928715d893b2057ee4f9f9d861ef09b5a70 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 18:54:39 +0900 Subject: [PATCH 21/50] ref: rename type and robust_subscription --- src/lib.rs | 2 +- src/robust_provider/mod.rs | 4 ++-- src/robust_provider/provider.rs | 2 +- src/robust_provider/{types.rs => provider_conversion.rs} | 0 .../{robust_subscription.rs => subscription.rs} | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename src/robust_provider/{types.rs => provider_conversion.rs} (100%) rename src/robust_provider/{robust_subscription.rs => subscription.rs} (100%) diff --git a/src/lib.rs b/src/lib.rs index 7da4b438..07b953dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,5 +18,5 @@ pub use event_scanner::{ pub use robust_provider::{ builder::RobustProviderBuilder, provider::RobustProvider, - robust_subscription::RobustSubscription, types::IntoRobustProvider, + provider_conversions::IntoRobustProvider, subscription::RobustSubscription, }; diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 86f35ed2..bd87784c 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -1,7 +1,7 @@ pub mod builder; pub mod error; pub mod provider; -pub mod robust_subscription; -pub mod types; +pub mod provider_conversion; +pub mod subscription; pub use error::Error; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 6c36e5c8..e6cffeb3 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -13,7 +13,7 @@ use tracing::{error, info}; use crate::{ RobustSubscription, - robust_provider::{Error, robust_subscription::DEFAULT_RECONNECT_INTERVAL}, + robust_provider::{Error, subscription::DEFAULT_RECONNECT_INTERVAL}, }; /// Provider wrapper with built-in retry and timeout mechanisms. diff --git a/src/robust_provider/types.rs b/src/robust_provider/provider_conversion.rs similarity index 100% rename from src/robust_provider/types.rs rename to src/robust_provider/provider_conversion.rs diff --git a/src/robust_provider/robust_subscription.rs b/src/robust_provider/subscription.rs similarity index 100% rename from src/robust_provider/robust_subscription.rs rename to src/robust_provider/subscription.rs From a0cdda91564f660f934304af76c6ea2c36a762b8 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:00:23 +0900 Subject: [PATCH 22/50] fix: all subsequent imports due to rename --- examples/historical_scanning/main.rs | 4 +++- examples/latest_events_scanning/main.rs | 4 +++- examples/live_scanning/main.rs | 4 +++- examples/sync_from_block_scanning/main.rs | 4 +++- examples/sync_from_latest_scanning/main.rs | 4 +++- src/lib.rs | 4 ++-- src/robust_provider/builder.rs | 2 +- src/robust_provider/provider.rs | 6 +++--- src/robust_provider/provider_conversion.rs | 5 ++++- tests/common/mod.rs | 2 +- 10 files changed, 26 insertions(+), 13 deletions(-) diff --git a/examples/historical_scanning/main.rs b/examples/historical_scanning/main.rs index 59cc88b6..d5633ee0 100644 --- a/examples/historical_scanning/main.rs +++ b/examples/historical_scanning/main.rs @@ -1,7 +1,9 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; +use event_scanner::{ + EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, +}; use tokio_stream::StreamExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; diff --git a/examples/latest_events_scanning/main.rs b/examples/latest_events_scanning/main.rs index 6f4ccad5..b3e323a5 100644 --- a/examples/latest_events_scanning/main.rs +++ b/examples/latest_events_scanning/main.rs @@ -1,6 +1,8 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; +use event_scanner::{ + EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, +}; use tokio_stream::StreamExt; use tracing::{error, info}; use tracing_subscriber::EnvFilter; diff --git a/examples/live_scanning/main.rs b/examples/live_scanning/main.rs index 4daa146b..a4c66d00 100644 --- a/examples/live_scanning/main.rs +++ b/examples/live_scanning/main.rs @@ -1,6 +1,8 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; +use event_scanner::{ + EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, +}; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/sync_from_block_scanning/main.rs b/examples/sync_from_block_scanning/main.rs index 3d4fe177..beb77998 100644 --- a/examples/sync_from_block_scanning/main.rs +++ b/examples/sync_from_block_scanning/main.rs @@ -2,7 +2,9 @@ use std::time::Duration; use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; +use event_scanner::{ + EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, +}; use tokio::time::sleep; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/sync_from_latest_scanning/main.rs b/examples/sync_from_latest_scanning/main.rs index 96f4e797..a83ffbec 100644 --- a/examples/sync_from_latest_scanning/main.rs +++ b/examples/sync_from_latest_scanning/main.rs @@ -1,6 +1,8 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; -use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; +use event_scanner::{ + EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, +}; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/src/lib.rs b/src/lib.rs index 07b953dd..02ec1141 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -17,6 +17,6 @@ pub use event_scanner::{ }; pub use robust_provider::{ - builder::RobustProviderBuilder, provider::RobustProvider, - provider_conversions::IntoRobustProvider, subscription::RobustSubscription, + provider::RobustProvider, provider_conversion::IntoRobustProvider, + subscription::RobustSubscription, }; diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 2631e649..38182386 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -4,7 +4,7 @@ use alloy::{network::Network, providers::Provider}; use crate::{ RobustProvider, - robust_provider::{error::Error, types::IntoProvider}, + robust_provider::{error::Error, provider_conversion::IntoProvider}, }; // RPC retry and timeout settings diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index e6cffeb3..a9b1cc59 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -292,9 +292,9 @@ impl RobustProvider { #[cfg(test)] mod tests { - use crate::{ - RobustProviderBuilder, - robust_provider::{Error, builder::DEFAULT_SUBSCRIPTION_TIMEOUT}, + use crate::robust_provider::{ + Error, + builder::{DEFAULT_SUBSCRIPTION_TIMEOUT, RobustProviderBuilder}, }; use super::*; diff --git a/src/robust_provider/provider_conversion.rs b/src/robust_provider/provider_conversion.rs index 292e7560..3a9845a2 100644 --- a/src/robust_provider/provider_conversion.rs +++ b/src/robust_provider/provider_conversion.rs @@ -8,7 +8,10 @@ use alloy::{ transports::http::reqwest::Url, }; -use crate::{RobustProvider, RobustProviderBuilder, robust_provider::Error}; +use crate::{ + RobustProvider, + robust_provider::{Error, builder::RobustProviderBuilder}, +}; pub trait IntoProvider { fn into_provider( diff --git a/tests/common/mod.rs b/tests/common/mod.rs index ad550945..69136bb3 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -5,7 +5,7 @@ pub mod setup_scanner; pub mod test_counter; -use event_scanner::{RobustProvider, RobustProviderBuilder}; +use event_scanner::{RobustProvider, robust_provider::builder::RobustProviderBuilder}; pub(crate) use setup_scanner::{ LiveScannerSetup, SyncScannerSetup, setup_common, setup_historic_scanner, setup_latest_scanner, setup_live_scanner, setup_sync_from_latest_scanner, setup_sync_scanner, From c5690484d480a24032c1031e9d1d7e20d43580fa Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:09:59 +0900 Subject: [PATCH 23/50] fix: doc test ref: format --- src/block_range_scanner.rs | 3 ++- src/event_scanner/scanner/mod.rs | 14 +++++++------- src/event_scanner/scanner/sync/mod.rs | 6 +++--- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index c39c95e9..15e32828 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -7,11 +7,12 @@ //! //! use alloy::providers::{Provider, ProviderBuilder}; //! use event_scanner::{ -//! RobustProviderBuilder, ScannerError, +//! ScannerError, //! block_range_scanner::{ //! BlockRangeScanner, BlockRangeScannerClient, DEFAULT_BLOCK_CONFIRMATIONS, //! DEFAULT_MAX_BLOCK_RANGE, Message, //! }, +//! robust_provider::builder::RobustProviderBuilder, //! }; //! use tokio::time::Duration; //! use tracing::{error, info}; diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index e65af101..568e2f0b 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -76,8 +76,8 @@ impl EventScannerBuilder { /// # Example /// /// ```no_run - /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; + /// # use alloy::{network::Ethereum, primitives::Address, providers::{Provider, ProviderBuilder}}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -103,7 +103,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Stream events between blocks [1_000_000, 2_000_000] @@ -143,7 +143,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -230,8 +230,8 @@ impl EventScannerBuilder { /// # Example /// /// ```no_run - /// # use alloy::{network::Ethereum, primitives::Address, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; + /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -258,7 +258,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Collect the latest 5 events between blocks [1_000_000, 1_100_000] diff --git a/src/event_scanner/scanner/sync/mod.rs b/src/event_scanner/scanner/sync/mod.rs index c51607c4..7d1944f6 100644 --- a/src/event_scanner/scanner/sync/mod.rs +++ b/src/event_scanner/scanner/sync/mod.rs @@ -25,7 +25,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -123,7 +123,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -163,7 +163,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, eips::BlockNumberOrTag, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Sync from genesis block From 782b9cf9cb02cc0aca54b02e21399ea63dd72c0e Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:13:07 +0900 Subject: [PATCH 24/50] fix: unwaited try_stream --- src/block_range_scanner.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index eb48a63f..fa8220e7 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -629,7 +629,7 @@ impl Service { Err(e) => { error!(error = %e, "Error receiving block from stream"); // Error from subscription, exit the stream - _ = sender.try_stream(e); + _ = sender.try_stream(e).await; return; } }; From d471b164df615f46578cf39b79ce5d98bbbdfb4a Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:23:15 +0900 Subject: [PATCH 25/50] ref: add comment link --- src/robust_provider.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index c5794dcd..715f6340 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -184,8 +184,8 @@ impl> RobustProviderBuilder { /// Set the timeout for subscription operations. /// - /// This should be set higher than `max_timeout` to accommodate chains with slow block times. - /// Default is 2 minutes. + /// This should be set higher than [`max_timeout`](Self::max_timeout) to accommodate chains with + /// slow block times. Default is [`DEFAULT_SUBSCRIPTION_TIMEOUT`]. #[must_use] pub fn subscription_timeout(mut self, timeout: Duration) -> Self { self.subscription_timeout = timeout; From 9793875fceab4309d49099356a292fde46a92468 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:27:42 +0900 Subject: [PATCH 26/50] ref: rewrite as match --- src/robust_provider.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 715f6340..74f7f763 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -368,14 +368,13 @@ impl RobustProvider { ) .await; - if let Err(e) = &subscription { - error!(error = %e, "eth_subscribe failed"); - return Err(e.clone()); + match subscription { + Ok(sub) => Ok(RobustSubscription::new(sub, self.clone(), DEFAULT_RECONNECT_INTERVAL)), + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } } - - let subscription = subscription?; - - Ok(RobustSubscription::new(subscription, self.clone(), DEFAULT_RECONNECT_INTERVAL)) } /// Execute `operation` with exponential backoff and a total timeout. From 9d6bd1112462d057843198175df7465af0b0179b Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:37:08 +0900 Subject: [PATCH 27/50] ref: use match --- src/robust_provider.rs | 15 +++------------ src/robust_subscription.rs | 34 +++++++++++++++++++--------------- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 74f7f763..23020c24 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -414,19 +414,9 @@ impl RobustProvider { return result; } - let mut last_error = result.unwrap_err(); + let last_error = result.unwrap_err(); - // providers are just fallback - match self.try_fallback_providers(providers, &operation, require_pubsub, last_error).await { - Ok(value) => { - return Ok(value); - } - Err(e) => last_error = e, - } - - // Return the last error encountered - error!("All providers failed or timed out - returning the last providers attempt's error"); - Err(last_error) + self.try_fallback_providers(providers, &operation, require_pubsub, last_error).await } pub(crate) async fn try_fallback_providers( @@ -464,6 +454,7 @@ impl RobustProvider { } } // All fallbacks failed / skipped, return the last error + error!("All providers failed or timed out - returning the last providers attempt's error"); Err(last_error) } diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index aca0131a..f89a91e9 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -134,19 +134,20 @@ impl RobustSubscription { let operation = move |provider: RootProvider| async move { provider.subscribe_blocks().await }; - let primary = self.robust_provider.primary(); let subscription = self.robust_provider.try_provider_with_timeout(primary, &operation).await; - if let Err(e) = &subscription { - error!(error = %e, "eth_subscribe failed"); - return Err(e.clone()); + match subscription { + Ok(sub) => { + self.subscription = Some(sub); + Ok(()) + } + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } } - - let subscription = subscription?; - self.subscription = Some(subscription); - Ok(()) } async fn switch_to_fallback(&mut self, last_error: Error) -> Result<(), Error> { @@ -156,6 +157,7 @@ impl RobustSubscription { let operation = move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + let subscription = self .robust_provider .try_fallback_providers( @@ -166,14 +168,16 @@ impl RobustSubscription { ) .await; - if let Err(e) = &subscription { - error!(error = %e, "eth_subscribe failed"); - return Err(e.clone()); + match subscription { + Ok(sub) => { + self.subscription = Some(sub); + Ok(()) + } + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } } - - let subscription = subscription?; - self.subscription = Some(subscription); - Ok(()) } /// Check if the subscription channel is empty (no pending messages) From 18a6dd7ad7ce41c69c4f8832e11c059dfa0320dd Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:48:01 +0900 Subject: [PATCH 28/50] ref: try fallback providers fn --- src/robust_provider.rs | 8 +++----- src/robust_subscription.rs | 13 +++---------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 23020c24..78415652 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -405,9 +405,7 @@ impl RobustProvider { F: Fn(RootProvider) -> Fut, Fut: Future>>, { - let mut providers = self.providers.iter(); - let primary = providers.next().expect("should have primary provider"); - + let primary = self.primary(); let result = self.try_provider_with_timeout(primary, &operation).await; if result.is_ok() { @@ -416,12 +414,11 @@ impl RobustProvider { let last_error = result.unwrap_err(); - self.try_fallback_providers(providers, &operation, require_pubsub, last_error).await + self.try_fallback_providers(&operation, require_pubsub, last_error).await } pub(crate) async fn try_fallback_providers( &self, - fallback_providers: impl Iterator>, operation: F, require_pubsub: bool, mut last_error: Error, @@ -434,6 +431,7 @@ impl RobustProvider { if num_providers > 1 { info!("Primary provider failed, trying fallback provider(s)"); } + let fallback_providers = self.providers.iter().skip(1); for (idx, provider) in fallback_providers.enumerate() { let fallback_num = idx + 1; if require_pubsub && !Self::supports_pubsub(provider) { diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index f89a91e9..c8b5145a 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -134,6 +134,7 @@ impl RobustSubscription { let operation = move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + let primary = self.robust_provider.primary(); let subscription = self.robust_provider.try_provider_with_timeout(primary, &operation).await; @@ -157,16 +158,8 @@ impl RobustSubscription { let operation = move |provider: RootProvider| async move { provider.subscribe_blocks().await }; - - let subscription = self - .robust_provider - .try_fallback_providers( - self.robust_provider.providers.iter().skip(1), - &operation, - true, - last_error, - ) - .await; + let subscription = + self.robust_provider.try_fallback_providers(&operation, true, last_error).await; match subscription { Ok(sub) => { From 26ed89399f98197bad813f51f83debcb381a6001 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 19:50:09 +0900 Subject: [PATCH 29/50] ref: - to * --- src/robust_provider.rs | 12 ++++++------ src/robust_subscription.rs | 8 ++++---- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 78415652..9056fb87 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -349,9 +349,9 @@ impl RobustProvider { /// Subscribe to new block headers with automatic failover and reconnection. /// /// Returns a `RobustSubscription` that automatically: - /// - Handles connection errors by switching to fallback providers - /// - Detects and recovers from lagged subscriptions - /// - Periodically attempts to reconnect to the primary provider + /// * Handles connection errors by switching to fallback providers + /// * Detects and recovers from lagged subscriptions + /// * Periodically attempts to reconnect to the primary provider /// /// # Errors /// @@ -390,12 +390,12 @@ impl RobustProvider { /// /// # Errors /// - /// - Returns [`RpcError`] with message "total operation timeout exceeded + /// * Returns [`RpcError`] with message "total operation timeout exceeded /// and all fallback providers failed" if the overall timeout elapses and no fallback /// providers succeed. - /// - Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` + /// * Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` /// is true and all providers don't support pubsub. - /// - Propagates any [`RpcError`] from the underlying retries. + /// * Propagates any [`RpcError`] from the underlying retries. pub(crate) async fn try_operation_with_failover( &self, operation: F, diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index c8b5145a..73494539 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -55,10 +55,10 @@ impl RobustSubscription { /// Receive the next item from the subscription with automatic failover. /// /// This method will: - /// - Attempt to receive from the current subscription - /// - Handle errors by switching to fallback providers - /// - Periodically attempt to reconnect to the primary provider - /// - Will switch to fallback providers if subscription timeout is exhausted + /// * Attempt to receive from the current subscription + /// * Handle errors by switching to fallback providers + /// * Periodically attempt to reconnect to the primary provider + /// * Will switch to fallback providers if subscription timeout is exhausted /// /// # Errors /// From 36ff7529107bdee0f101572616d49354daf15fab Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 20:00:31 +0900 Subject: [PATCH 30/50] ref: update to use bounded channel comment --- src/robust_subscription.rs | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/robust_subscription.rs b/src/robust_subscription.rs index 73494539..bb10dde9 100644 --- a/src/robust_subscription.rs +++ b/src/robust_subscription.rs @@ -14,7 +14,7 @@ use tokio::{ sync::{broadcast::error::RecvError, mpsc}, time::timeout, }; -use tokio_stream::{Stream, wrappers::UnboundedReceiverStream}; +use tokio_stream::{Stream, wrappers::ReceiverStream}; use tracing::{error, info, warn}; use crate::robust_provider::{Error, RobustProvider}; @@ -25,6 +25,9 @@ pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); /// Maximum number of consecutive lags before switching providers const MAX_LAG_COUNT: usize = 3; +/// Max amount of buffered blocks stream can hold +pub const MAX_BUFFERED_BLOCKS: usize = 50000; + /// A robust subscription wrapper that automatically handles provider failover /// and periodic reconnection attempts to the primary provider. #[derive(Debug)] @@ -188,35 +191,35 @@ impl RobustSubscription { /// and forwards items to a channel, which is then wrapped in a Stream. #[must_use] pub fn into_stream(mut self) -> RobustSubscriptionStream { - // TODO: This shouldnt be unbounded need choose an appropriate bound (Maybe same as max - // buffer probably should be bigger) - let (tx, rx) = mpsc::unbounded_channel(); + let (tx, rx) = mpsc::channel(MAX_BUFFERED_BLOCKS); tokio::spawn(async move { loop { match self.recv().await { Ok(item) => { - if tx.send(Ok(item)).is_err() { - // Receiver dropped, exit the loop + if let Err(err) = tx.send(Ok(item)).await { + warn!(error = %err, "Downstream channel closed, stopping stream"); break; } } Err(e) => { // Send the error and exit - let _ = tx.send(Err(e)); + if let Err(err) = tx.send(Err(e)).await { + warn!(error = %err, "Downstream channel closed, stopping stream"); + } break; } } } }); - RobustSubscriptionStream { inner: UnboundedReceiverStream::new(rx) } + RobustSubscriptionStream { inner: ReceiverStream::new(rx) } } } /// A stream wrapper around [`RobustSubscription`] that implements the [`Stream`] trait. pub struct RobustSubscriptionStream { - inner: UnboundedReceiverStream>, + inner: ReceiverStream>, } impl Stream for RobustSubscriptionStream { From c3eae825ff74a39d000560031f19d66b20a09ae1 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 20:45:36 +0900 Subject: [PATCH 31/50] fix: add timeout for flaky test --- src/robust_provider/provider.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index fc42250f..6d37a5d7 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -484,6 +484,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(http_provider) .max_timeout(Duration::from_millis(500)) + .subscription_timeout(Duration::from_millis(500)) .build() .await?; From 3fc2173c13b6e98995cf0d92cbc01d9c027dda42 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 20:56:28 +0900 Subject: [PATCH 32/50] test: add sub timeout to test --- src/robust_provider.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/robust_provider.rs b/src/robust_provider.rs index 9056fb87..aca824ef 100644 --- a/src/robust_provider.rs +++ b/src/robust_provider.rs @@ -608,6 +608,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) .max_timeout(Duration::from_secs(1)) + .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -687,6 +688,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(http_provider) .max_timeout(Duration::from_millis(500)) + .subscription_timeout(Duration::from_secs(1)) .build() .await?; From 8304024f643e8935a278c4b213994da4232397f1 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 22:31:35 +0900 Subject: [PATCH 33/50] ref: revert changes --- src/robust_provider/provider.rs | 361 +++++++++++++++++++--------- src/robust_provider/subscription.rs | 231 ------------------ 2 files changed, 241 insertions(+), 351 deletions(-) delete mode 100644 src/robust_provider/subscription.rs diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 3945ebf1..89482dbc 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -1,33 +1,228 @@ -use std::{fmt::Debug, time::Duration}; +use std::{fmt::Debug, future::Future, marker::PhantomData, sync::Arc, time::Duration}; use alloy::{ - eips::BlockNumberOrTag, + eips::{BlockId, BlockNumberOrTag}, network::{Ethereum, Network}, - providers::{Provider, RootProvider}, + providers::{ + DynProvider, Provider, RootProvider, + fillers::{FillProvider, TxFiller}, + layers::{CacheProvider, CallBatchProvider}, + }, + pubsub::Subscription, rpc::types::{Filter, Log}, - transports::{RpcError, TransportErrorKind}, + transports::{RpcError, TransportErrorKind, http::reqwest::Url}, }; use backon::{ExponentialBuilder, Retryable}; -use tokio::time::timeout; +use thiserror::Error; +use tokio::time::{error as TokioError, timeout}; use tracing::{error, info}; -use crate::{ - RobustSubscription, - robust_provider::{Error, subscription::DEFAULT_RECONNECT_INTERVAL}, -}; +#[derive(Error, Debug, Clone)] +pub enum Error { + #[error("Operation timed out")] + Timeout, + #[error("RPC call failed after exhausting all retry attempts: {0}")] + RpcError(Arc>), + #[error("Block not found, Block Id: {0}")] + BlockNotFound(BlockId), +} + +impl From> for Error { + fn from(err: RpcError) -> Self { + Error::RpcError(Arc::new(err)) + } +} + +impl From for Error { + fn from(_: TokioError::Elapsed) -> Self { + Error::Timeout + } +} + +pub trait IntoProvider { + fn into_provider( + self, + ) -> impl std::future::Future, Error>> + Send; +} + +impl IntoProvider for RobustProvider { + async fn into_provider(self) -> Result, Error> { + Ok(self.primary().to_owned()) + } +} + +impl IntoProvider for RootProvider { + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for &str { + async fn into_provider(self) -> Result, Error> { + Ok(RootProvider::connect(self).await?) + } +} + +impl IntoProvider for Url { + async fn into_provider(self) -> Result, Error> { + Ok(RootProvider::connect(self.as_str()).await?) + } +} + +impl IntoProvider for FillProvider +where + F: TxFiller, + P: Provider, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for CacheProvider +where + P: Provider, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for DynProvider +where + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +impl IntoProvider for CallBatchProvider +where + P: Provider + 'static, + N: Network, +{ + async fn into_provider(self) -> Result, Error> { + Ok(self) + } +} + +pub trait IntoRobustProvider { + fn into_robust_provider( + self, + ) -> impl std::future::Future, Error>> + Send; +} + +impl + Send> IntoRobustProvider for P { + async fn into_robust_provider(self) -> Result, Error> { + RobustProviderBuilder::new(self).build().await + } +} + +// RPC retry and timeout settings +/// Default timeout used by `RobustProvider` +pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); +/// Default maximum number of retry attempts. +pub const DEFAULT_MAX_RETRIES: usize = 3; +/// Default base delay between retries. +pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); + +#[derive(Clone)] +pub struct RobustProviderBuilder> { + providers: Vec

, + max_timeout: Duration, + max_retries: usize, + min_delay: Duration, + _network: PhantomData, +} + +impl> RobustProviderBuilder { + /// Create a new `RobustProvider` with default settings. + /// + /// The provided provider is treated as the primary provider. + #[must_use] + pub fn new(provider: P) -> Self { + Self { + providers: vec![provider], + max_timeout: DEFAULT_MAX_TIMEOUT, + max_retries: DEFAULT_MAX_RETRIES, + min_delay: DEFAULT_MIN_DELAY, + _network: PhantomData, + } + } + + /// Create a new `RobustProvider` with no retry attempts and only timeout set. + /// + /// The provided provider is treated as the primary provider. + #[must_use] + pub fn fragile(provider: P) -> Self { + Self::new(provider).max_retries(0).min_delay(Duration::ZERO) + } + + /// Add a fallback provider to the list. + /// + /// Fallback providers are used when the primary provider times out or fails. + #[must_use] + pub fn fallback(mut self, provider: P) -> Self { + self.providers.push(provider); + self + } + + /// Set the maximum timeout for RPC operations. + #[must_use] + pub fn max_timeout(mut self, timeout: Duration) -> Self { + self.max_timeout = timeout; + self + } + + /// Set the maximum number of retry attempts. + #[must_use] + pub fn max_retries(mut self, max_retries: usize) -> Self { + self.max_retries = max_retries; + self + } + + /// Set the base delay for exponential backoff retries. + #[must_use] + pub fn min_delay(mut self, min_delay: Duration) -> Self { + self.min_delay = min_delay; + self + } + + /// Build the `RobustProvider`. + /// + /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. + /// + /// # Errors + /// + /// Returns an error if any of the providers fail to connect. + pub async fn build(self) -> Result, Error> { + let mut providers = vec![]; + for p in self.providers { + providers.push(p.into_provider().await?.root().to_owned()); + } + Ok(RobustProvider { + providers, + max_timeout: self.max_timeout, + max_retries: self.max_retries, + min_delay: self.min_delay, + }) + } +} /// Provider wrapper with built-in retry and timeout mechanisms. /// /// This wrapper around Alloy providers automatically handles retries, /// timeouts, and error logging for RPC calls. /// The first provider in the vector is treated as the primary provider. -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct RobustProvider { - pub(crate) providers: Vec>, - pub(crate) max_timeout: Duration, - pub(crate) subscription_timeout: Duration, - pub(crate) max_retries: usize, - pub(crate) min_delay: Duration, + providers: Vec>, + max_timeout: Duration, + max_retries: usize, + min_delay: Duration, } impl RobustProvider { @@ -55,7 +250,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByNumber called"); let result = self - .try_operation_with_failover( + .retry_with_total_timeout( move |provider| async move { provider.get_block_by_number(number).await }, false, ) @@ -77,7 +272,7 @@ impl RobustProvider { pub async fn get_block_number(&self) -> Result { info!("eth_getBlockNumber called"); let result = self - .try_operation_with_failover( + .retry_with_total_timeout( move |provider| async move { provider.get_block_number().await }, false, ) @@ -101,7 +296,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByHash called"); let result = self - .try_operation_with_failover( + .retry_with_total_timeout( move |provider| async move { provider.get_block_by_hash(hash).await }, false, ) @@ -123,7 +318,7 @@ impl RobustProvider { pub async fn get_logs(&self, filter: &Filter) -> Result, Error> { info!("eth_getLogs called"); let result = self - .try_operation_with_failover( + .retry_with_total_timeout( move |provider| async move { provider.get_logs(filter).await }, false, ) @@ -134,12 +329,7 @@ impl RobustProvider { result } - /// Subscribe to new block headers with automatic failover and reconnection. - /// - /// Returns a `RobustSubscription` that automatically: - /// * Handles connection errors by switching to fallback providers - /// * Detects and recovers from lagged subscriptions - /// * Periodically attempts to reconnect to the primary provider + /// Subscribe to new block headers with retry and timeout. /// /// # Errors /// @@ -147,22 +337,18 @@ impl RobustProvider { /// call fails after exhausting all retry attempts, or if the call times out. /// When fallback providers are configured, the error returned will be from the /// final provider that was attempted. - pub async fn subscribe_blocks(&self) -> Result, Error> { + pub async fn subscribe_blocks(&self) -> Result, Error> { info!("eth_subscribe called"); - let subscription = self - .try_operation_with_failover( + let result = self + .retry_with_total_timeout( move |provider| async move { provider.subscribe_blocks().await }, true, ) .await; - - match subscription { - Ok(sub) => Ok(RobustSubscription::new(sub, self.clone(), DEFAULT_RECONNECT_INTERVAL)), - Err(e) => { - error!(error = %e, "eth_subscribe failed"); - Err(e) - } + if let Err(e) = &result { + error!(error = %e, "eth_subscribe failed"); } + result } /// Execute `operation` with exponential backoff and a total timeout. @@ -178,13 +364,13 @@ impl RobustProvider { /// /// # Errors /// - /// * Returns [`RpcError`] with message "total operation timeout exceeded + /// - Returns [`RpcError`] with message "total operation timeout exceeded /// and all fallback providers failed" if the overall timeout elapses and no fallback /// providers succeed. - /// * Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` + /// - Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` /// is true and all providers don't support pubsub. - /// * Propagates any [`RpcError`] from the underlying retries. - pub(crate) async fn try_operation_with_failover( + /// - Propagates any [`RpcError`] from the underlying retries. + async fn retry_with_total_timeout( &self, operation: F, require_pubsub: bool, @@ -193,34 +379,24 @@ impl RobustProvider { F: Fn(RootProvider) -> Fut, Fut: Future>>, { - let primary = self.primary(); + let mut providers = self.providers.iter(); + let primary = providers.next().expect("should have primary provider"); + let result = self.try_provider_with_timeout(primary, &operation).await; if result.is_ok() { return result; } - let last_error = result.unwrap_err(); + let mut last_error = result.unwrap_err(); - self.try_fallback_providers(&operation, require_pubsub, last_error).await - } - - pub(crate) async fn try_fallback_providers( - &self, - operation: F, - require_pubsub: bool, - mut last_error: Error, - ) -> Result - where - F: Fn(RootProvider) -> Fut, - Fut: Future>>, - { let num_providers = self.providers.len(); if num_providers > 1 { info!("Primary provider failed, trying fallback provider(s)"); } - let fallback_providers = self.providers.iter().skip(1); - for (idx, provider) in fallback_providers.enumerate() { + + // This loop starts at index 1 automatically + for (idx, provider) in providers.enumerate() { let fallback_num = idx + 1; if require_pubsub && !Self::supports_pubsub(provider) { info!("Fallback provider {} doesn't support pubsub, skipping", fallback_num); @@ -239,13 +415,14 @@ impl RobustProvider { } } } - // All fallbacks failed / skipped, return the last error + + // Return the last error encountered error!("All providers failed or timed out - returning the last providers attempt's error"); Err(last_error) } /// Try executing an operation with a specific provider with retry and timeout. - pub(crate) async fn try_provider_with_timeout( + async fn try_provider_with_timeout( &self, provider: &RootProvider, operation: F, @@ -280,30 +457,19 @@ impl RobustProvider { #[cfg(test)] mod tests { - use crate::robust_provider::{ - Error, - builder::{DEFAULT_SUBSCRIPTION_TIMEOUT, RobustProviderBuilder}, - }; - use super::*; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, - transports::{RpcError, TransportErrorKind}, }; use alloy_node_bindings::Anvil; - use std::{ - sync::atomic::{AtomicUsize, Ordering}, - time::Duration, - }; + use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::time::sleep; - use tokio_stream::StreamExt; fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { RobustProvider { providers: vec![RootProvider::new_http("http://localhost:8545".parse().unwrap())], max_timeout: Duration::from_millis(timeout), - subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, min_delay: Duration::from_millis(min_delay), } @@ -316,7 +482,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .try_operation_with_failover( + .retry_with_total_timeout( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -336,7 +502,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .try_operation_with_failover( + .retry_with_total_timeout( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -359,7 +525,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result: Result<(), Error> = provider - .try_operation_with_failover( + .retry_with_total_timeout( |_| async { call_count.fetch_add(1, Ordering::SeqCst); Err(TransportErrorKind::BackendGone.into()) @@ -378,7 +544,7 @@ mod tests { let provider = test_provider(max_timeout, 10, 1); let result = provider - .try_operation_with_failover( + .retry_with_total_timeout( move |_provider| async move { sleep(Duration::from_millis(max_timeout + 10)).await; Ok(42) @@ -391,6 +557,7 @@ mod tests { } #[tokio::test] + #[ignore = "Either anvil or the failover for subscription is flaky so best to ignore for now"] async fn test_subscribe_fails_causes_backup_to_be_used() -> anyhow::Result<()> { let anvil_1 = Anvil::new().try_spawn()?; @@ -405,7 +572,6 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) .max_timeout(Duration::from_secs(1)) - .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -485,7 +651,6 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(http_provider) .max_timeout(Duration::from_millis(500)) - .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -506,48 +671,4 @@ mod tests { Ok(()) } - - #[tokio::test] - async fn test_stream_with_failover() -> anyhow::Result<()> { - let mut anvil_1 = Some(Anvil::new().block_time_f64(0.1).try_spawn()?); - - let ws_provider = ProviderBuilder::new() - .connect(anvil_1.as_ref().unwrap().ws_endpoint_url().as_str()) - .await?; - - let anvil_2 = Anvil::new().block_time_f64(0.1).try_spawn()?; - - let ws_provider_2 = - ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; - - let robust = RobustProviderBuilder::fragile(ws_provider.clone()) - .fallback(ws_provider_2) - .subscription_timeout(Duration::from_millis(500)) - .build() - .await?; - - let subscription = robust.subscribe_blocks().await?; - let mut stream = subscription.into_stream(); - - while let Some(result) = stream.next().await { - let Ok(block) = result else { - break; - }; - - let block_number = block.number(); - - // At block 10, drop the primary provider to test failover - if block_number == 10 && - let Some(anvil) = anvil_1.take() - { - drop(anvil); - } - - if block_number >= 11 { - break; - } - } - - Ok(()) - } } diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs deleted file mode 100644 index a234b759..00000000 --- a/src/robust_provider/subscription.rs +++ /dev/null @@ -1,231 +0,0 @@ -use std::{ - pin::Pin, - task::{Context, Poll}, - time::{Duration, Instant}, -}; - -use alloy::{ - network::Network, - providers::{Provider, RootProvider}, - pubsub::Subscription, - transports::{RpcError, TransportErrorKind}, -}; -use tokio::{ - sync::{broadcast::error::RecvError, mpsc}, - time::timeout, -}; -use tokio_stream::{Stream, wrappers::ReceiverStream}; -use tracing::{error, info, warn}; - -use crate::{RobustProvider, robust_provider::Error}; - -/// Default time interval between primary provider reconnection attempts -pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); - -/// Maximum number of consecutive lags before switching providers -const MAX_LAG_COUNT: usize = 3; - -/// Max amount of buffered blocks stream can hold -pub const MAX_BUFFERED_BLOCKS: usize = 50000; - -/// A robust subscription wrapper that automatically handles provider failover -/// and periodic reconnection attempts to the primary provider. -#[derive(Debug)] -pub struct RobustSubscription { - subscription: Option>, - robust_provider: RobustProvider, - reconnect_interval: Duration, - last_reconnect_attempt: Option, - consecutive_lags: usize, -} - -impl RobustSubscription { - /// Create a new [`RobustSubscription`] - pub(crate) fn new( - subscription: Subscription, - robust_provider: RobustProvider, - reconnect_interval: Duration, - ) -> Self { - Self { - subscription: Some(subscription), - robust_provider, - reconnect_interval, - last_reconnect_attempt: None, - consecutive_lags: 0, - } - } - - /// Receive the next item from the subscription with automatic failover. - /// - /// This method will: - /// * Attempt to receive from the current subscription - /// * Handle errors by switching to fallback providers - /// * Periodically attempt to reconnect to the primary provider - /// * Will switch to fallback providers if subscription timeout is exhausted - /// - /// # Errors - /// - /// Returns an error if all providers have been exhausted and failed. - pub async fn recv(&mut self) -> Result { - loop { - if self.should_reconnect_to_primary() { - info!("Attempting to reconnect to primary provider"); - if let Err(e) = self.try_reconnect_to_primary().await { - warn!(error = %e, "Failed to reconnect to primary provider"); - } else { - info!("Successfully reconnected to primary provider"); - } - } - - if let Some(subscription) = &mut self.subscription { - let subscription_timeout = self.robust_provider.subscription_timeout; - match timeout(subscription_timeout, subscription.recv()).await { - Ok(recv_result) => match recv_result { - Ok(header) => { - self.consecutive_lags = 0; - return Ok(header); - } - Err(recv_error) => match recv_error { - RecvError::Closed => { - error!("Subscription channel closed, switching provider"); - let error = - RpcError::Transport(TransportErrorKind::BackendGone).into(); - self.switch_to_fallback(error).await?; - } - RecvError::Lagged(skipped) => { - self.consecutive_lags += 1; - warn!( - skipped = skipped, - consecutive_lags = self.consecutive_lags, - "Subscription lagged" - ); - - if self.consecutive_lags >= MAX_LAG_COUNT { - error!("Too many consecutive lags, switching provider"); - let error = - RpcError::Transport(TransportErrorKind::BackendGone).into(); - self.switch_to_fallback(error).await?; - } - } - }, - }, - Err(e) => { - error!( - timeout_secs = subscription_timeout.as_secs(), - "Subscription timeout - no block received, switching provider" - ); - self.switch_to_fallback(e.into()).await?; - } - } - } else { - // No subscription available - return Err(RpcError::Transport(TransportErrorKind::BackendGone).into()); - } - } - } - - fn should_reconnect_to_primary(&self) -> bool { - // Only attempt reconnection if enough time has passed since last attempt - match self.last_reconnect_attempt { - None => false, - Some(last_attempt) => last_attempt.elapsed() >= self.reconnect_interval, - } - } - - async fn try_reconnect_to_primary(&mut self) -> Result<(), Error> { - self.last_reconnect_attempt = Some(Instant::now()); - - let operation = - move |provider: RootProvider| async move { provider.subscribe_blocks().await }; - - let primary = self.robust_provider.primary(); - let subscription = - self.robust_provider.try_provider_with_timeout(primary, &operation).await; - - match subscription { - Ok(sub) => { - self.subscription = Some(sub); - Ok(()) - } - Err(e) => { - error!(error = %e, "eth_subscribe failed"); - Err(e) - } - } - } - - async fn switch_to_fallback(&mut self, last_error: Error) -> Result<(), Error> { - if self.last_reconnect_attempt.is_none() { - self.last_reconnect_attempt = Some(Instant::now()); - } - - let operation = - move |provider: RootProvider| async move { provider.subscribe_blocks().await }; - let subscription = - self.robust_provider.try_fallback_providers(&operation, true, last_error).await; - - match subscription { - Ok(sub) => { - self.subscription = Some(sub); - Ok(()) - } - Err(e) => { - error!(error = %e, "eth_subscribe failed"); - Err(e) - } - } - } - - /// Check if the subscription channel is empty (no pending messages) - #[must_use] - pub fn is_empty(&self) -> bool { - match &self.subscription { - Some(sub) => sub.is_empty(), - None => true, - } - } - - /// Convert the subscription into a stream. - /// - /// This spawns a background task that continuously receives from the subscription - /// and forwards items to a channel, which is then wrapped in a Stream. - #[must_use] - pub fn into_stream(mut self) -> RobustSubscriptionStream { - let (tx, rx) = mpsc::channel(MAX_BUFFERED_BLOCKS); - - tokio::spawn(async move { - loop { - match self.recv().await { - Ok(item) => { - if let Err(err) = tx.send(Ok(item)).await { - warn!(error = %err, "Downstream channel closed, stopping stream"); - break; - } - } - Err(e) => { - // Send the error and exit - if let Err(err) = tx.send(Err(e)).await { - warn!(error = %err, "Downstream channel closed, stopping stream"); - } - break; - } - } - } - }); - - RobustSubscriptionStream { inner: ReceiverStream::new(rx) } - } -} - -/// A stream wrapper around [`RobustSubscription`] that implements the [`Stream`] trait. -pub struct RobustSubscriptionStream { - inner: ReceiverStream>, -} - -impl Stream for RobustSubscriptionStream { - type Item = Result; - - fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.inner).poll_next(cx) - } -} From 4ba97ee768b2d3ea0f54a2adda75a85025633f0a Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 22:56:07 +0900 Subject: [PATCH 34/50] ref: rebase onto main --- src/block_range_scanner.rs | 22 +--- src/lib.rs | 5 +- src/robust_provider/builder.rs | 21 +-- src/robust_provider/mod.rs | 1 - src/robust_provider/provider.rs | 219 ++------------------------------ 5 files changed, 20 insertions(+), 248 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 19930ead..bc4cbda7 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -67,7 +67,7 @@ use tokio::{ use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ - IntoRobustProvider, RobustProvider, RobustSubscription, ScannerMessage, + IntoRobustProvider, RobustProvider, ScannerMessage, error::ScannerError, robust_provider::Error as RobustProviderError, types::{ScannerStatus, TryStream}, @@ -77,6 +77,7 @@ use alloy::{ eips::BlockNumberOrTag, network::{BlockResponse, Network, primitives::HeaderResponse}, primitives::{B256, BlockNumber}, + pubsub::Subscription, transports::{RpcError, TransportErrorKind}, }; use tracing::{debug, error, info, warn}; @@ -610,29 +611,16 @@ impl Service { async fn stream_live_blocks( mut range_start: BlockNumber, - subscription: RobustSubscription, + subscription: Subscription, sender: mpsc::Sender, block_confirmations: u64, max_block_range: u64, ) { // ensure we start streaming only after the expected_next_block cutoff let cutoff = range_start; - let mut stream = subscription.into_stream().skip_while(|result| match result { - Ok(header) => header.number() < cutoff, - Err(_) => false, - }); - - while let Some(result) = stream.next().await { - let incoming_block = match result { - Ok(block) => block, - Err(e) => { - error!(error = %e, "Error receiving block from stream"); - // Error from subscription, exit the stream - _ = sender.try_stream(e).await; - return; - } - }; + let mut stream = subscription.into_stream().skip_while(|header| header.number() < cutoff); + while let Some(incoming_block) = stream.next().await { let incoming_block_num = incoming_block.number(); info!(block_number = incoming_block_num, "Received block header"); diff --git a/src/lib.rs b/src/lib.rs index 02ec1141..5460c178 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,7 +16,4 @@ pub use event_scanner::{ SyncFromBlock, SyncFromLatestEvents, }; -pub use robust_provider::{ - provider::RobustProvider, provider_conversion::IntoRobustProvider, - subscription::RobustSubscription, -}; +pub use robust_provider::{provider::RobustProvider, provider_conversion::IntoRobustProvider}; diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 38182386..0fa18625 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -10,8 +10,6 @@ use crate::{ // RPC retry and timeout settings /// Default timeout used by `RobustProvider` pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); -/// Default timeout for subscriptions (longer to accommodate slow block times) -pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(120); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; /// Default base delay between retries. @@ -19,16 +17,15 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); #[derive(Clone)] pub struct RobustProviderBuilder> { - pub(crate) providers: Vec

, + providers: Vec

, max_timeout: Duration, - subscription_timeout: Duration, max_retries: usize, min_delay: Duration, _network: PhantomData, } impl> RobustProviderBuilder { - /// Create a new [`RobustProvider`] with default settings. + /// Create a new `RobustProvider` with default settings. /// /// The provided provider is treated as the primary provider. #[must_use] @@ -36,14 +33,13 @@ impl> RobustProviderBuilder { Self { providers: vec![provider], max_timeout: DEFAULT_MAX_TIMEOUT, - subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, min_delay: DEFAULT_MIN_DELAY, _network: PhantomData, } } - /// Create a new [`RobustProvider`] with no retry attempts and only timeout set. + /// Create a new `RobustProvider` with no retry attempts and only timeout set. /// /// The provided provider is treated as the primary provider. #[must_use] @@ -67,16 +63,6 @@ impl> RobustProviderBuilder { self } - /// Set the timeout for subscription operations. - /// - /// This should be set higher than `max_timeout` to accommodate chains with slow block times. - /// Default is 2 minutes. - #[must_use] - pub fn subscription_timeout(mut self, timeout: Duration) -> Self { - self.subscription_timeout = timeout; - self - } - /// Set the maximum number of retry attempts. #[must_use] pub fn max_retries(mut self, max_retries: usize) -> Self { @@ -106,7 +92,6 @@ impl> RobustProviderBuilder { Ok(RobustProvider { providers, max_timeout: self.max_timeout, - subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, min_delay: self.min_delay, }) diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index bd87784c..642b4f9f 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -2,6 +2,5 @@ pub mod builder; pub mod error; pub mod provider; pub mod provider_conversion; -pub mod subscription; pub use error::Error; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 89482dbc..6ee7ad69 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -1,216 +1,18 @@ -use std::{fmt::Debug, future::Future, marker::PhantomData, sync::Arc, time::Duration}; +use std::{fmt::Debug, time::Duration}; use alloy::{ - eips::{BlockId, BlockNumberOrTag}, + eips::BlockNumberOrTag, network::{Ethereum, Network}, - providers::{ - DynProvider, Provider, RootProvider, - fillers::{FillProvider, TxFiller}, - layers::{CacheProvider, CallBatchProvider}, - }, + providers::{Provider, RootProvider}, pubsub::Subscription, rpc::types::{Filter, Log}, - transports::{RpcError, TransportErrorKind, http::reqwest::Url}, + transports::{RpcError, TransportErrorKind}, }; use backon::{ExponentialBuilder, Retryable}; -use thiserror::Error; -use tokio::time::{error as TokioError, timeout}; +use tokio::time::timeout; use tracing::{error, info}; -#[derive(Error, Debug, Clone)] -pub enum Error { - #[error("Operation timed out")] - Timeout, - #[error("RPC call failed after exhausting all retry attempts: {0}")] - RpcError(Arc>), - #[error("Block not found, Block Id: {0}")] - BlockNotFound(BlockId), -} - -impl From> for Error { - fn from(err: RpcError) -> Self { - Error::RpcError(Arc::new(err)) - } -} - -impl From for Error { - fn from(_: TokioError::Elapsed) -> Self { - Error::Timeout - } -} - -pub trait IntoProvider { - fn into_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl IntoProvider for RobustProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self.primary().to_owned()) - } -} - -impl IntoProvider for RootProvider { - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for &str { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self).await?) - } -} - -impl IntoProvider for Url { - async fn into_provider(self) -> Result, Error> { - Ok(RootProvider::connect(self.as_str()).await?) - } -} - -impl IntoProvider for FillProvider -where - F: TxFiller, - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CacheProvider -where - P: Provider, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for DynProvider -where - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -impl IntoProvider for CallBatchProvider -where - P: Provider + 'static, - N: Network, -{ - async fn into_provider(self) -> Result, Error> { - Ok(self) - } -} - -pub trait IntoRobustProvider { - fn into_robust_provider( - self, - ) -> impl std::future::Future, Error>> + Send; -} - -impl + Send> IntoRobustProvider for P { - async fn into_robust_provider(self) -> Result, Error> { - RobustProviderBuilder::new(self).build().await - } -} - -// RPC retry and timeout settings -/// Default timeout used by `RobustProvider` -pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); -/// Default maximum number of retry attempts. -pub const DEFAULT_MAX_RETRIES: usize = 3; -/// Default base delay between retries. -pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); - -#[derive(Clone)] -pub struct RobustProviderBuilder> { - providers: Vec

, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, - _network: PhantomData, -} - -impl> RobustProviderBuilder { - /// Create a new `RobustProvider` with default settings. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn new(provider: P) -> Self { - Self { - providers: vec![provider], - max_timeout: DEFAULT_MAX_TIMEOUT, - max_retries: DEFAULT_MAX_RETRIES, - min_delay: DEFAULT_MIN_DELAY, - _network: PhantomData, - } - } - - /// Create a new `RobustProvider` with no retry attempts and only timeout set. - /// - /// The provided provider is treated as the primary provider. - #[must_use] - pub fn fragile(provider: P) -> Self { - Self::new(provider).max_retries(0).min_delay(Duration::ZERO) - } - - /// Add a fallback provider to the list. - /// - /// Fallback providers are used when the primary provider times out or fails. - #[must_use] - pub fn fallback(mut self, provider: P) -> Self { - self.providers.push(provider); - self - } - - /// Set the maximum timeout for RPC operations. - #[must_use] - pub fn max_timeout(mut self, timeout: Duration) -> Self { - self.max_timeout = timeout; - self - } - - /// Set the maximum number of retry attempts. - #[must_use] - pub fn max_retries(mut self, max_retries: usize) -> Self { - self.max_retries = max_retries; - self - } - - /// Set the base delay for exponential backoff retries. - #[must_use] - pub fn min_delay(mut self, min_delay: Duration) -> Self { - self.min_delay = min_delay; - self - } - - /// Build the `RobustProvider`. - /// - /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. - /// - /// # Errors - /// - /// Returns an error if any of the providers fail to connect. - pub async fn build(self) -> Result, Error> { - let mut providers = vec![]; - for p in self.providers { - providers.push(p.into_provider().await?.root().to_owned()); - } - Ok(RobustProvider { - providers, - max_timeout: self.max_timeout, - max_retries: self.max_retries, - min_delay: self.min_delay, - }) - } -} +use crate::robust_provider::Error; /// Provider wrapper with built-in retry and timeout mechanisms. /// @@ -219,10 +21,10 @@ impl> RobustProviderBuilder { /// The first provider in the vector is treated as the primary provider. #[derive(Clone)] pub struct RobustProvider { - providers: Vec>, - max_timeout: Duration, - max_retries: usize, - min_delay: Duration, + pub(crate) providers: Vec>, + pub(crate) max_timeout: Duration, + pub(crate) max_retries: usize, + pub(crate) min_delay: Duration, } impl RobustProvider { @@ -458,6 +260,7 @@ impl RobustProvider { #[cfg(test)] mod tests { use super::*; + use crate::robust_provider::builder::RobustProviderBuilder; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, From 41a891d660392120fcb1377aec0716f4e6ce2ba8 Mon Sep 17 00:00:00 2001 From: Leo Date: Wed, 12 Nov 2025 23:08:41 +0900 Subject: [PATCH 35/50] fix: rebase past changes --- src/block_range_scanner.rs | 22 ++- src/lib.rs | 5 +- src/robust_provider/builder.rs | 19 ++- src/robust_provider/mod.rs | 1 + src/robust_provider/provider.rs | 137 +++++++++++++---- src/robust_provider/subscription.rs | 231 ++++++++++++++++++++++++++++ 6 files changed, 373 insertions(+), 42 deletions(-) create mode 100644 src/robust_provider/subscription.rs diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index bc4cbda7..19930ead 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -67,7 +67,7 @@ use tokio::{ use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ - IntoRobustProvider, RobustProvider, ScannerMessage, + IntoRobustProvider, RobustProvider, RobustSubscription, ScannerMessage, error::ScannerError, robust_provider::Error as RobustProviderError, types::{ScannerStatus, TryStream}, @@ -77,7 +77,6 @@ use alloy::{ eips::BlockNumberOrTag, network::{BlockResponse, Network, primitives::HeaderResponse}, primitives::{B256, BlockNumber}, - pubsub::Subscription, transports::{RpcError, TransportErrorKind}, }; use tracing::{debug, error, info, warn}; @@ -611,16 +610,29 @@ impl Service { async fn stream_live_blocks( mut range_start: BlockNumber, - subscription: Subscription, + subscription: RobustSubscription, sender: mpsc::Sender, block_confirmations: u64, max_block_range: u64, ) { // ensure we start streaming only after the expected_next_block cutoff let cutoff = range_start; - let mut stream = subscription.into_stream().skip_while(|header| header.number() < cutoff); + let mut stream = subscription.into_stream().skip_while(|result| match result { + Ok(header) => header.number() < cutoff, + Err(_) => false, + }); + + while let Some(result) = stream.next().await { + let incoming_block = match result { + Ok(block) => block, + Err(e) => { + error!(error = %e, "Error receiving block from stream"); + // Error from subscription, exit the stream + _ = sender.try_stream(e).await; + return; + } + }; - while let Some(incoming_block) = stream.next().await { let incoming_block_num = incoming_block.number(); info!(block_number = incoming_block_num, "Received block header"); diff --git a/src/lib.rs b/src/lib.rs index 5460c178..02ec1141 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -16,4 +16,7 @@ pub use event_scanner::{ SyncFromBlock, SyncFromLatestEvents, }; -pub use robust_provider::{provider::RobustProvider, provider_conversion::IntoRobustProvider}; +pub use robust_provider::{ + provider::RobustProvider, provider_conversion::IntoRobustProvider, + subscription::RobustSubscription, +}; diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 0fa18625..9ec181e6 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -10,6 +10,8 @@ use crate::{ // RPC retry and timeout settings /// Default timeout used by `RobustProvider` pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); +/// Default timeout for subscriptions (longer to accommodate slow block times) +pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(120); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; /// Default base delay between retries. @@ -19,13 +21,14 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); pub struct RobustProviderBuilder> { providers: Vec

, max_timeout: Duration, + subscription_timeout: Duration, max_retries: usize, min_delay: Duration, _network: PhantomData, } impl> RobustProviderBuilder { - /// Create a new `RobustProvider` with default settings. + /// Create a new [`RobustProvider`] with default settings. /// /// The provided provider is treated as the primary provider. #[must_use] @@ -33,13 +36,14 @@ impl> RobustProviderBuilder { Self { providers: vec![provider], max_timeout: DEFAULT_MAX_TIMEOUT, + subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, min_delay: DEFAULT_MIN_DELAY, _network: PhantomData, } } - /// Create a new `RobustProvider` with no retry attempts and only timeout set. + /// Create a new [`RobustProvider`] with no retry attempts and only timeout set. /// /// The provided provider is treated as the primary provider. #[must_use] @@ -63,6 +67,16 @@ impl> RobustProviderBuilder { self } + /// Set the timeout for subscription operations. + /// + /// This should be set higher than [`max_timeout`](Self::max_timeout) to accommodate chains with + /// slow block times. Default is [`DEFAULT_SUBSCRIPTION_TIMEOUT`]. + #[must_use] + pub fn subscription_timeout(mut self, timeout: Duration) -> Self { + self.subscription_timeout = timeout; + self + } + /// Set the maximum number of retry attempts. #[must_use] pub fn max_retries(mut self, max_retries: usize) -> Self { @@ -92,6 +106,7 @@ impl> RobustProviderBuilder { Ok(RobustProvider { providers, max_timeout: self.max_timeout, + subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, min_delay: self.min_delay, }) diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 642b4f9f..bd87784c 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -2,5 +2,6 @@ pub mod builder; pub mod error; pub mod provider; pub mod provider_conversion; +pub mod subscription; pub use error::Error; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 6ee7ad69..2cbac1de 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -1,10 +1,10 @@ use std::{fmt::Debug, time::Duration}; use alloy::{ + self, eips::BlockNumberOrTag, network::{Ethereum, Network}, providers::{Provider, RootProvider}, - pubsub::Subscription, rpc::types::{Filter, Log}, transports::{RpcError, TransportErrorKind}, }; @@ -12,17 +12,21 @@ use backon::{ExponentialBuilder, Retryable}; use tokio::time::timeout; use tracing::{error, info}; -use crate::robust_provider::Error; +use crate::{ + RobustSubscription, + robust_provider::{Error, subscription::DEFAULT_RECONNECT_INTERVAL}, +}; /// Provider wrapper with built-in retry and timeout mechanisms. /// /// This wrapper around Alloy providers automatically handles retries, /// timeouts, and error logging for RPC calls. /// The first provider in the vector is treated as the primary provider. -#[derive(Clone)] +#[derive(Clone, Debug)] pub struct RobustProvider { pub(crate) providers: Vec>, pub(crate) max_timeout: Duration, + pub(crate) subscription_timeout: Duration, pub(crate) max_retries: usize, pub(crate) min_delay: Duration, } @@ -52,7 +56,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByNumber called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_by_number(number).await }, false, ) @@ -74,7 +78,7 @@ impl RobustProvider { pub async fn get_block_number(&self) -> Result { info!("eth_getBlockNumber called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_number().await }, false, ) @@ -98,7 +102,7 @@ impl RobustProvider { ) -> Result { info!("eth_getBlockByHash called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_block_by_hash(hash).await }, false, ) @@ -120,7 +124,7 @@ impl RobustProvider { pub async fn get_logs(&self, filter: &Filter) -> Result, Error> { info!("eth_getLogs called"); let result = self - .retry_with_total_timeout( + .try_operation_with_failover( move |provider| async move { provider.get_logs(filter).await }, false, ) @@ -131,7 +135,12 @@ impl RobustProvider { result } - /// Subscribe to new block headers with retry and timeout. + /// Subscribe to new block headers with automatic failover and reconnection. + /// + /// Returns a `RobustSubscription` that automatically: + /// * Handles connection errors by switching to fallback providers + /// * Detects and recovers from lagged subscriptions + /// * Periodically attempts to reconnect to the primary provider /// /// # Errors /// @@ -139,18 +148,22 @@ impl RobustProvider { /// call fails after exhausting all retry attempts, or if the call times out. /// When fallback providers are configured, the error returned will be from the /// final provider that was attempted. - pub async fn subscribe_blocks(&self) -> Result, Error> { + pub async fn subscribe_blocks(&self) -> Result, Error> { info!("eth_subscribe called"); - let result = self - .retry_with_total_timeout( + let subscription = self + .try_operation_with_failover( move |provider| async move { provider.subscribe_blocks().await }, true, ) .await; - if let Err(e) = &result { - error!(error = %e, "eth_subscribe failed"); + + match subscription { + Ok(sub) => Ok(RobustSubscription::new(sub, self.clone(), DEFAULT_RECONNECT_INTERVAL)), + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } } - result } /// Execute `operation` with exponential backoff and a total timeout. @@ -166,13 +179,13 @@ impl RobustProvider { /// /// # Errors /// - /// - Returns [`RpcError`] with message "total operation timeout exceeded + /// * Returns [`RpcError`] with message "total operation timeout exceeded /// and all fallback providers failed" if the overall timeout elapses and no fallback /// providers succeed. - /// - Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` + /// * Returns [`RpcError::Transport(TransportErrorKind::PubsubUnavailable)`] if `require_pubsub` /// is true and all providers don't support pubsub. - /// - Propagates any [`RpcError`] from the underlying retries. - async fn retry_with_total_timeout( + /// * Propagates any [`RpcError`] from the underlying retries. + pub(crate) async fn try_operation_with_failover( &self, operation: F, require_pubsub: bool, @@ -181,24 +194,34 @@ impl RobustProvider { F: Fn(RootProvider) -> Fut, Fut: Future>>, { - let mut providers = self.providers.iter(); - let primary = providers.next().expect("should have primary provider"); - + let primary = self.primary(); let result = self.try_provider_with_timeout(primary, &operation).await; if result.is_ok() { return result; } - let mut last_error = result.unwrap_err(); + let last_error = result.unwrap_err(); + self.try_fallback_providers(&operation, require_pubsub, last_error).await + } + + pub(crate) async fn try_fallback_providers( + &self, + operation: F, + require_pubsub: bool, + mut last_error: Error, + ) -> Result + where + F: Fn(RootProvider) -> Fut, + Fut: Future>>, + { let num_providers = self.providers.len(); if num_providers > 1 { info!("Primary provider failed, trying fallback provider(s)"); } - - // This loop starts at index 1 automatically - for (idx, provider) in providers.enumerate() { + let fallback_providers = self.providers.iter().skip(1); + for (idx, provider) in fallback_providers.enumerate() { let fallback_num = idx + 1; if require_pubsub && !Self::supports_pubsub(provider) { info!("Fallback provider {} doesn't support pubsub, skipping", fallback_num); @@ -217,14 +240,13 @@ impl RobustProvider { } } } - - // Return the last error encountered + // All fallbacks failed / skipped, return the last error error!("All providers failed or timed out - returning the last providers attempt's error"); Err(last_error) } /// Try executing an operation with a specific provider with retry and timeout. - async fn try_provider_with_timeout( + pub(crate) async fn try_provider_with_timeout( &self, provider: &RootProvider, operation: F, @@ -260,7 +282,7 @@ impl RobustProvider { #[cfg(test)] mod tests { use super::*; - use crate::robust_provider::builder::RobustProviderBuilder; + use crate::robust_provider::builder::{DEFAULT_SUBSCRIPTION_TIMEOUT, RobustProviderBuilder}; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, @@ -268,11 +290,13 @@ mod tests { use alloy_node_bindings::Anvil; use std::sync::atomic::{AtomicUsize, Ordering}; use tokio::time::sleep; + use tokio_stream::StreamExt; fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { RobustProvider { providers: vec![RootProvider::new_http("http://localhost:8545".parse().unwrap())], max_timeout: Duration::from_millis(timeout), + subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, min_delay: Duration::from_millis(min_delay), } @@ -285,7 +309,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -305,7 +329,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); let count = call_count.load(Ordering::SeqCst); @@ -328,7 +352,7 @@ mod tests { let call_count = AtomicUsize::new(0); let result: Result<(), Error> = provider - .retry_with_total_timeout( + .try_operation_with_failover( |_| async { call_count.fetch_add(1, Ordering::SeqCst); Err(TransportErrorKind::BackendGone.into()) @@ -347,7 +371,7 @@ mod tests { let provider = test_provider(max_timeout, 10, 1); let result = provider - .retry_with_total_timeout( + .try_operation_with_failover( move |_provider| async move { sleep(Duration::from_millis(max_timeout + 10)).await; Ok(42) @@ -360,7 +384,6 @@ mod tests { } #[tokio::test] - #[ignore = "Either anvil or the failover for subscription is flaky so best to ignore for now"] async fn test_subscribe_fails_causes_backup_to_be_used() -> anyhow::Result<()> { let anvil_1 = Anvil::new().try_spawn()?; @@ -375,6 +398,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) .max_timeout(Duration::from_secs(1)) + .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -454,6 +478,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(http_provider) .max_timeout(Duration::from_millis(500)) + .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -474,4 +499,48 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_stream_with_failover() -> anyhow::Result<()> { + let mut anvil_1 = Some(Anvil::new().block_time_f64(0.1).try_spawn()?); + + let ws_provider = ProviderBuilder::new() + .connect(anvil_1.as_ref().unwrap().ws_endpoint_url().as_str()) + .await?; + + let anvil_2 = Anvil::new().block_time_f64(0.1).try_spawn()?; + + let ws_provider_2 = + ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; + + let robust = RobustProviderBuilder::fragile(ws_provider.clone()) + .fallback(ws_provider_2) + .subscription_timeout(Duration::from_millis(500)) + .build() + .await?; + + let subscription = robust.subscribe_blocks().await?; + let mut stream = subscription.into_stream(); + + while let Some(result) = stream.next().await { + let Ok(block) = result else { + break; + }; + + let block_number = block.number(); + + // At block 10, drop the primary provider to test failover + if block_number == 10 && + let Some(anvil) = anvil_1.take() + { + drop(anvil); + } + + if block_number >= 11 { + break; + } + } + + Ok(()) + } } diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs new file mode 100644 index 00000000..a234b759 --- /dev/null +++ b/src/robust_provider/subscription.rs @@ -0,0 +1,231 @@ +use std::{ + pin::Pin, + task::{Context, Poll}, + time::{Duration, Instant}, +}; + +use alloy::{ + network::Network, + providers::{Provider, RootProvider}, + pubsub::Subscription, + transports::{RpcError, TransportErrorKind}, +}; +use tokio::{ + sync::{broadcast::error::RecvError, mpsc}, + time::timeout, +}; +use tokio_stream::{Stream, wrappers::ReceiverStream}; +use tracing::{error, info, warn}; + +use crate::{RobustProvider, robust_provider::Error}; + +/// Default time interval between primary provider reconnection attempts +pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); + +/// Maximum number of consecutive lags before switching providers +const MAX_LAG_COUNT: usize = 3; + +/// Max amount of buffered blocks stream can hold +pub const MAX_BUFFERED_BLOCKS: usize = 50000; + +/// A robust subscription wrapper that automatically handles provider failover +/// and periodic reconnection attempts to the primary provider. +#[derive(Debug)] +pub struct RobustSubscription { + subscription: Option>, + robust_provider: RobustProvider, + reconnect_interval: Duration, + last_reconnect_attempt: Option, + consecutive_lags: usize, +} + +impl RobustSubscription { + /// Create a new [`RobustSubscription`] + pub(crate) fn new( + subscription: Subscription, + robust_provider: RobustProvider, + reconnect_interval: Duration, + ) -> Self { + Self { + subscription: Some(subscription), + robust_provider, + reconnect_interval, + last_reconnect_attempt: None, + consecutive_lags: 0, + } + } + + /// Receive the next item from the subscription with automatic failover. + /// + /// This method will: + /// * Attempt to receive from the current subscription + /// * Handle errors by switching to fallback providers + /// * Periodically attempt to reconnect to the primary provider + /// * Will switch to fallback providers if subscription timeout is exhausted + /// + /// # Errors + /// + /// Returns an error if all providers have been exhausted and failed. + pub async fn recv(&mut self) -> Result { + loop { + if self.should_reconnect_to_primary() { + info!("Attempting to reconnect to primary provider"); + if let Err(e) = self.try_reconnect_to_primary().await { + warn!(error = %e, "Failed to reconnect to primary provider"); + } else { + info!("Successfully reconnected to primary provider"); + } + } + + if let Some(subscription) = &mut self.subscription { + let subscription_timeout = self.robust_provider.subscription_timeout; + match timeout(subscription_timeout, subscription.recv()).await { + Ok(recv_result) => match recv_result { + Ok(header) => { + self.consecutive_lags = 0; + return Ok(header); + } + Err(recv_error) => match recv_error { + RecvError::Closed => { + error!("Subscription channel closed, switching provider"); + let error = + RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + RecvError::Lagged(skipped) => { + self.consecutive_lags += 1; + warn!( + skipped = skipped, + consecutive_lags = self.consecutive_lags, + "Subscription lagged" + ); + + if self.consecutive_lags >= MAX_LAG_COUNT { + error!("Too many consecutive lags, switching provider"); + let error = + RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + } + }, + }, + Err(e) => { + error!( + timeout_secs = subscription_timeout.as_secs(), + "Subscription timeout - no block received, switching provider" + ); + self.switch_to_fallback(e.into()).await?; + } + } + } else { + // No subscription available + return Err(RpcError::Transport(TransportErrorKind::BackendGone).into()); + } + } + } + + fn should_reconnect_to_primary(&self) -> bool { + // Only attempt reconnection if enough time has passed since last attempt + match self.last_reconnect_attempt { + None => false, + Some(last_attempt) => last_attempt.elapsed() >= self.reconnect_interval, + } + } + + async fn try_reconnect_to_primary(&mut self) -> Result<(), Error> { + self.last_reconnect_attempt = Some(Instant::now()); + + let operation = + move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + + let primary = self.robust_provider.primary(); + let subscription = + self.robust_provider.try_provider_with_timeout(primary, &operation).await; + + match subscription { + Ok(sub) => { + self.subscription = Some(sub); + Ok(()) + } + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } + } + } + + async fn switch_to_fallback(&mut self, last_error: Error) -> Result<(), Error> { + if self.last_reconnect_attempt.is_none() { + self.last_reconnect_attempt = Some(Instant::now()); + } + + let operation = + move |provider: RootProvider| async move { provider.subscribe_blocks().await }; + let subscription = + self.robust_provider.try_fallback_providers(&operation, true, last_error).await; + + match subscription { + Ok(sub) => { + self.subscription = Some(sub); + Ok(()) + } + Err(e) => { + error!(error = %e, "eth_subscribe failed"); + Err(e) + } + } + } + + /// Check if the subscription channel is empty (no pending messages) + #[must_use] + pub fn is_empty(&self) -> bool { + match &self.subscription { + Some(sub) => sub.is_empty(), + None => true, + } + } + + /// Convert the subscription into a stream. + /// + /// This spawns a background task that continuously receives from the subscription + /// and forwards items to a channel, which is then wrapped in a Stream. + #[must_use] + pub fn into_stream(mut self) -> RobustSubscriptionStream { + let (tx, rx) = mpsc::channel(MAX_BUFFERED_BLOCKS); + + tokio::spawn(async move { + loop { + match self.recv().await { + Ok(item) => { + if let Err(err) = tx.send(Ok(item)).await { + warn!(error = %err, "Downstream channel closed, stopping stream"); + break; + } + } + Err(e) => { + // Send the error and exit + if let Err(err) = tx.send(Err(e)).await { + warn!(error = %err, "Downstream channel closed, stopping stream"); + } + break; + } + } + } + }); + + RobustSubscriptionStream { inner: ReceiverStream::new(rx) } + } +} + +/// A stream wrapper around [`RobustSubscription`] that implements the [`Stream`] trait. +pub struct RobustSubscriptionStream { + inner: ReceiverStream>, +} + +impl Stream for RobustSubscriptionStream { + type Item = Result; + + fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + Pin::new(&mut self.inner).poll_next(cx) + } +} From e76b5f7e7e578f30833c1c6b237309b10066a85f Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 16:56:36 +0900 Subject: [PATCH 36/50] ref: move all imports to nested in robust_provider --- src/block_range_scanner.rs | 7 +++++-- src/event_scanner/scanner/common.rs | 3 +-- src/event_scanner/scanner/mod.rs | 3 ++- src/lib.rs | 2 -- src/robust_provider/builder.rs | 5 +---- src/robust_provider/mod.rs | 3 +++ src/robust_provider/provider_conversion.rs | 5 +---- tests/common/mod.rs | 2 +- tests/common/setup_scanner.rs | 2 +- 9 files changed, 15 insertions(+), 17 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index bc4cbda7..812bdb5c 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -67,9 +67,12 @@ use tokio::{ use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ - IntoRobustProvider, RobustProvider, ScannerMessage, + ScannerMessage, error::ScannerError, - robust_provider::Error as RobustProviderError, + robust_provider::{ + Error as RobustProviderError, provider::RobustProvider, + provider_conversion::IntoRobustProvider, + }, types::{ScannerStatus, TryStream}, }; use alloy::{ diff --git a/src/event_scanner/scanner/common.rs b/src/event_scanner/scanner/common.rs index 89b379b3..6bae3d9f 100644 --- a/src/event_scanner/scanner/common.rs +++ b/src/event_scanner/scanner/common.rs @@ -1,10 +1,9 @@ use std::ops::RangeInclusive; use crate::{ - RobustProvider, block_range_scanner::{MAX_BUFFERED_MESSAGES, Message as BlockRangeMessage}, event_scanner::{filter::EventFilter, listener::EventListener}, - robust_provider::Error as RobustProviderError, + robust_provider::{Error as RobustProviderError, RobustProvider}, types::TryStream, }; use alloy::{ diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index 568e2f0b..dd5a94d8 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -6,12 +6,13 @@ use tokio::sync::mpsc; use tokio_stream::wrappers::ReceiverStream; use crate::{ - EventFilter, IntoRobustProvider, Message, ScannerError, + EventFilter, Message, ScannerError, block_range_scanner::{ BlockRangeScanner, ConnectedBlockRangeScanner, DEFAULT_BLOCK_CONFIRMATIONS, MAX_BUFFERED_MESSAGES, }, event_scanner::listener::EventListener, + robust_provider::provider_conversion::IntoRobustProvider, }; mod common; diff --git a/src/lib.rs b/src/lib.rs index 5460c178..496051a7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -15,5 +15,3 @@ pub use event_scanner::{ EventFilter, EventScanner, EventScannerBuilder, Historic, LatestEvents, Live, Message, SyncFromBlock, SyncFromLatestEvents, }; - -pub use robust_provider::{provider::RobustProvider, provider_conversion::IntoRobustProvider}; diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 0fa18625..855f6755 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -2,10 +2,7 @@ use std::{marker::PhantomData, time::Duration}; use alloy::{network::Network, providers::Provider}; -use crate::{ - RobustProvider, - robust_provider::{error::Error, provider_conversion::IntoProvider}, -}; +use crate::robust_provider::{Error, IntoProvider, RobustProvider}; // RPC retry and timeout settings /// Default timeout used by `RobustProvider` diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 642b4f9f..1952ce5e 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -3,4 +3,7 @@ pub mod error; pub mod provider; pub mod provider_conversion; +pub use builder::RobustProviderBuilder; pub use error::Error; +pub use provider::RobustProvider; +pub use provider_conversion::IntoProvider; diff --git a/src/robust_provider/provider_conversion.rs b/src/robust_provider/provider_conversion.rs index 3a9845a2..129919be 100644 --- a/src/robust_provider/provider_conversion.rs +++ b/src/robust_provider/provider_conversion.rs @@ -8,10 +8,7 @@ use alloy::{ transports::http::reqwest::Url, }; -use crate::{ - RobustProvider, - robust_provider::{Error, builder::RobustProviderBuilder}, -}; +use crate::robust_provider::{Error, RobustProvider, RobustProviderBuilder}; pub trait IntoProvider { fn into_provider( diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 69136bb3..b12e9f3f 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -5,7 +5,7 @@ pub mod setup_scanner; pub mod test_counter; -use event_scanner::{RobustProvider, robust_provider::builder::RobustProviderBuilder}; +use event_scanner::robust_provider::{RobustProvider, RobustProviderBuilder}; pub(crate) use setup_scanner::{ LiveScannerSetup, SyncScannerSetup, setup_common, setup_historic_scanner, setup_latest_scanner, setup_live_scanner, setup_sync_from_latest_scanner, setup_sync_scanner, diff --git a/tests/common/setup_scanner.rs b/tests/common/setup_scanner.rs index 28d0f220..7e9fdf9e 100644 --- a/tests/common/setup_scanner.rs +++ b/tests/common/setup_scanner.rs @@ -7,7 +7,7 @@ use alloy::{ use alloy_node_bindings::AnvilInstance; use event_scanner::{ EventFilter, EventScanner, EventScannerBuilder, Historic, LatestEvents, Live, Message, - RobustProvider, SyncFromBlock, SyncFromLatestEvents, + SyncFromBlock, SyncFromLatestEvents, robust_provider::RobustProvider, }; use tokio_stream::wrappers::ReceiverStream; From c5980caaddbfceb15d80ab84139b97a471faeb2e Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 16:58:48 +0900 Subject: [PATCH 37/50] ref: use new import --- src/block_range_scanner.rs | 1 + src/robust_provider/mod.rs | 1 + src/robust_provider/provider.rs | 5 +---- src/robust_provider/subscription.rs | 2 +- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 50cab570..921013a5 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -59,6 +59,7 @@ //! } //! ``` +use crate::robust_provider::subscription::RobustSubscription; use std::{cmp::Ordering, ops::RangeInclusive}; use tokio::{ sync::{mpsc, oneshot}, diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 574df8e1..d8d71285 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -8,3 +8,4 @@ pub use builder::RobustProviderBuilder; pub use error::Error; pub use provider::RobustProvider; pub use provider_conversion::IntoProvider; +pub use subscription::RobustSubscription; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 2cbac1de..17dbbcbf 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -12,10 +12,7 @@ use backon::{ExponentialBuilder, Retryable}; use tokio::time::timeout; use tracing::{error, info}; -use crate::{ - RobustSubscription, - robust_provider::{Error, subscription::DEFAULT_RECONNECT_INTERVAL}, -}; +use crate::robust_provider::{Error, RobustSubscription, subscription::DEFAULT_RECONNECT_INTERVAL}; /// Provider wrapper with built-in retry and timeout mechanisms. /// diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs index a234b759..bb10dde9 100644 --- a/src/robust_provider/subscription.rs +++ b/src/robust_provider/subscription.rs @@ -17,7 +17,7 @@ use tokio::{ use tokio_stream::{Stream, wrappers::ReceiverStream}; use tracing::{error, info, warn}; -use crate::{RobustProvider, robust_provider::Error}; +use crate::robust_provider::{Error, RobustProvider}; /// Default time interval between primary provider reconnection attempts pub const DEFAULT_RECONNECT_INTERVAL: Duration = Duration::from_secs(30); From ee5857caa54d58f4315efc54df1ac35b8e17900f Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 17:30:15 +0900 Subject: [PATCH 38/50] ref: rename max timeout to call timeout --- examples/historical_scanning/main.rs | 2 +- examples/latest_events_scanning/main.rs | 2 +- examples/live_scanning/main.rs | 2 +- examples/sync_from_block_scanning/main.rs | 2 +- examples/sync_from_latest_scanning/main.rs | 2 +- src/robust_provider/builder.rs | 18 +++++++-------- src/robust_provider/provider.rs | 26 +++++++++++----------- 7 files changed, 27 insertions(+), 27 deletions(-) diff --git a/examples/historical_scanning/main.rs b/examples/historical_scanning/main.rs index d5633ee0..8c596c6f 100644 --- a/examples/historical_scanning/main.rs +++ b/examples/historical_scanning/main.rs @@ -55,7 +55,7 @@ async fn main() -> anyhow::Result<()> { let _ = counter_contract.increase().send().await?.get_receipt().await?; let robust_provider = RobustProviderBuilder::new(provider) - .max_timeout(std::time::Duration::from_secs(30)) + .call_timeout(std::time::Duration::from_secs(30)) .max_retries(5) .min_delay(std::time::Duration::from_millis(500)) .build() diff --git a/examples/latest_events_scanning/main.rs b/examples/latest_events_scanning/main.rs index b3e323a5..83838ecb 100644 --- a/examples/latest_events_scanning/main.rs +++ b/examples/latest_events_scanning/main.rs @@ -52,7 +52,7 @@ async fn main() -> anyhow::Result<()> { .event(Counter::CountIncreased::SIGNATURE); let robust_provider = RobustProviderBuilder::new(provider) - .max_timeout(std::time::Duration::from_secs(30)) + .call_timeout(std::time::Duration::from_secs(30)) .max_retries(5) .min_delay(std::time::Duration::from_millis(500)) .build() diff --git a/examples/live_scanning/main.rs b/examples/live_scanning/main.rs index a4c66d00..bbdd8839 100644 --- a/examples/live_scanning/main.rs +++ b/examples/live_scanning/main.rs @@ -53,7 +53,7 @@ async fn main() -> anyhow::Result<()> { .event(Counter::CountIncreased::SIGNATURE); let robust_provider = RobustProviderBuilder::new(provider) - .max_timeout(std::time::Duration::from_secs(30)) + .call_timeout(std::time::Duration::from_secs(30)) .max_retries(5) .min_delay(std::time::Duration::from_millis(500)) .build() diff --git a/examples/sync_from_block_scanning/main.rs b/examples/sync_from_block_scanning/main.rs index beb77998..99c3e968 100644 --- a/examples/sync_from_block_scanning/main.rs +++ b/examples/sync_from_block_scanning/main.rs @@ -61,7 +61,7 @@ async fn main() -> anyhow::Result<()> { } let robust_provider = RobustProviderBuilder::new(provider) - .max_timeout(Duration::from_secs(30)) + .call_timeout(Duration::from_secs(30)) .max_retries(5) .min_delay(Duration::from_millis(500)) .build() diff --git a/examples/sync_from_latest_scanning/main.rs b/examples/sync_from_latest_scanning/main.rs index a83ffbec..7bdcc2f0 100644 --- a/examples/sync_from_latest_scanning/main.rs +++ b/examples/sync_from_latest_scanning/main.rs @@ -53,7 +53,7 @@ async fn main() -> anyhow::Result<()> { .event(Counter::CountIncreased::SIGNATURE); let robust_provider = RobustProviderBuilder::new(provider) - .max_timeout(std::time::Duration::from_secs(30)) + .call_timeout(std::time::Duration::from_secs(30)) .max_retries(5) .min_delay(std::time::Duration::from_millis(500)) .build() diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 15e52763..4ca299ea 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -6,8 +6,8 @@ use crate::robust_provider::{Error, IntoProvider, RobustProvider}; // RPC retry and timeout settings /// Default timeout used by `RobustProvider` -pub const DEFAULT_MAX_TIMEOUT: Duration = Duration::from_secs(60); -/// Default timeout for subscriptions (longer to accommodate slow block times) +pub const DEFAULT_CALL_TIMEOUT: Duration = Duration::from_secs(60); +/// Default timeout for subscriptions pub const DEFAULT_SUBSCRIPTION_TIMEOUT: Duration = Duration::from_secs(120); /// Default maximum number of retry attempts. pub const DEFAULT_MAX_RETRIES: usize = 3; @@ -17,7 +17,7 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); #[derive(Clone)] pub struct RobustProviderBuilder> { providers: Vec

, - max_timeout: Duration, + call_timeout: Duration, subscription_timeout: Duration, max_retries: usize, min_delay: Duration, @@ -32,7 +32,7 @@ impl> RobustProviderBuilder { pub fn new(provider: P) -> Self { Self { providers: vec![provider], - max_timeout: DEFAULT_MAX_TIMEOUT, + call_timeout: DEFAULT_CALL_TIMEOUT, subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, min_delay: DEFAULT_MIN_DELAY, @@ -59,15 +59,15 @@ impl> RobustProviderBuilder { /// Set the maximum timeout for RPC operations. #[must_use] - pub fn max_timeout(mut self, timeout: Duration) -> Self { - self.max_timeout = timeout; + pub fn call_timeout(mut self, timeout: Duration) -> Self { + self.call_timeout = timeout; self } /// Set the timeout for subscription operations. /// - /// This should be set higher than [`max_timeout`](Self::max_timeout) to accommodate chains with - /// slow block times. Default is [`DEFAULT_SUBSCRIPTION_TIMEOUT`]. + /// This should be set higher than [`call_timeout`](Self::call_timeout) to accommodate chains + /// with slow block times. Default is [`DEFAULT_SUBSCRIPTION_TIMEOUT`]. #[must_use] pub fn subscription_timeout(mut self, timeout: Duration) -> Self { self.subscription_timeout = timeout; @@ -102,7 +102,7 @@ impl> RobustProviderBuilder { } Ok(RobustProvider { providers, - max_timeout: self.max_timeout, + call_timeout: self.call_timeout, subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, min_delay: self.min_delay, diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 7f5277d4..dff2521d 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -22,7 +22,7 @@ use crate::robust_provider::{Error, RobustSubscription, subscription::DEFAULT_RE #[derive(Clone, Debug)] pub struct RobustProvider { pub(crate) providers: Vec>, - pub(crate) max_timeout: Duration, + pub(crate) call_timeout: Duration, pub(crate) subscription_timeout: Duration, pub(crate) max_retries: usize, pub(crate) min_delay: Duration, @@ -174,9 +174,9 @@ impl RobustProvider { /// Execute `operation` with exponential backoff and a total timeout. /// - /// Wraps the retry logic with `tokio::time::timeout(self.max_timeout, ...)` so + /// Wraps the retry logic with `tokio::time::timeout(self.call_timeout, ...)` so /// the entire operation (including time spent inside the RPC call) cannot exceed - /// `max_timeout`. + /// `call_timeout`. /// /// If the timeout is exceeded and fallback providers are available, it will /// attempt to use each fallback provider in sequence. @@ -267,7 +267,7 @@ impl RobustProvider { .with_min_delay(self.min_delay); timeout( - self.max_timeout, + self.call_timeout, (|| operation(provider.clone())) .retry(retry_strategy) .notify(|err: &RpcError, dur: Duration| { @@ -302,7 +302,7 @@ mod tests { fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { RobustProvider { providers: vec![RootProvider::new_http("http://localhost:8545".parse().unwrap())], - max_timeout: Duration::from_millis(timeout), + call_timeout: Duration::from_millis(timeout), subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, min_delay: Duration::from_millis(min_delay), @@ -373,14 +373,14 @@ mod tests { } #[tokio::test] - async fn test_retry_with_timeout_respects_max_timeout() { - let max_timeout = 50; - let provider = test_provider(max_timeout, 10, 1); + async fn test_retry_with_timeout_respects_call_timeout() { + let call_timeout = 50; + let provider = test_provider(call_timeout, 10, 1); let result = provider .try_operation_with_failover( move |_provider| async move { - sleep(Duration::from_millis(max_timeout + 10)).await; + sleep(Duration::from_millis(call_timeout + 10)).await; Ok(42) }, false, @@ -404,7 +404,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) - .max_timeout(Duration::from_secs(1)) + .call_timeout(Duration::from_secs(1)) .subscription_timeout(Duration::from_secs(1)) .build() .await?; @@ -430,7 +430,7 @@ mod tests { let robust = RobustProviderBuilder::new(http_provider.clone()) .fallback(http_provider) - .max_timeout(Duration::from_secs(5)) + .call_timeout(Duration::from_secs(5)) .min_delay(Duration::from_millis(100)) .build() .await?; @@ -462,7 +462,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(http_provider) .fallback(ws_provider) - .max_timeout(Duration::from_secs(5)) + .call_timeout(Duration::from_secs(5)) .build() .await?; @@ -484,7 +484,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider.clone()) .fallback(http_provider) - .max_timeout(Duration::from_millis(500)) + .call_timeout(Duration::from_millis(500)) .subscription_timeout(Duration::from_secs(1)) .build() .await?; From 6cc293c65c900198a533e80b02af6f27ecaf311f Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 17:35:53 +0900 Subject: [PATCH 39/50] ref: add sleep --- src/robust_provider/provider.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index dff2521d..67846521 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -411,6 +411,8 @@ mod tests { drop(anvil_1); + sleep(Duration::from_secs(1)).await; + let mut subscription = robust.subscribe_blocks().await?; ws_provider_2.anvil_mine(Some(2), None).await?; From ce4c640224a842cb74122b43a683f0468ba6593c Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 17:48:34 +0900 Subject: [PATCH 40/50] ref: simpler test --- src/robust_provider/provider.rs | 98 +++++++++++++-------------------- 1 file changed, 39 insertions(+), 59 deletions(-) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 67846521..91394795 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -390,40 +390,6 @@ mod tests { assert!(matches!(result, Err(Error::Timeout))); } - #[tokio::test] - async fn test_subscribe_fails_causes_backup_to_be_used() -> anyhow::Result<()> { - let anvil_1 = Anvil::new().try_spawn()?; - - let ws_provider_1 = - ProviderBuilder::new().connect(anvil_1.ws_endpoint_url().as_str()).await?; - - let anvil_2 = Anvil::new().try_spawn()?; - - let ws_provider_2 = - ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; - - let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) - .fallback(ws_provider_2.clone()) - .call_timeout(Duration::from_secs(1)) - .subscription_timeout(Duration::from_secs(1)) - .build() - .await?; - - drop(anvil_1); - - sleep(Duration::from_secs(1)).await; - - let mut subscription = robust.subscribe_blocks().await?; - - ws_provider_2.anvil_mine(Some(2), None).await?; - - assert_eq!(1, subscription.recv().await?.number()); - assert_eq!(2, subscription.recv().await?.number()); - assert!(subscription.is_empty()); - - Ok(()) - } - #[tokio::test] async fn test_subscribe_fails_when_all_providers_lack_pubsub() -> anyhow::Result<()> { let anvil = Anvil::new().try_spawn()?; @@ -485,16 +451,27 @@ mod tests { let http_provider = ProviderBuilder::new().connect_http(anvil_2.endpoint_url()); let robust = RobustProviderBuilder::fragile(ws_provider.clone()) - .fallback(http_provider) + .fallback(http_provider.clone()) .call_timeout(Duration::from_millis(500)) .subscription_timeout(Duration::from_secs(1)) .build() .await?; - // force ws_provider to fail and return BackendGone + let mut subscription = robust.subscribe_blocks().await?; + + ws_provider.anvil_mine(Some(1), None).await?; + assert_eq!(1, subscription.recv().await?.number()); + + ws_provider.anvil_mine(Some(1), None).await?; + assert_eq!(2, subscription.recv().await?.number()); + drop(anvil_1); - let err = robust.subscribe_blocks().await.unwrap_err(); + sleep(Duration::from_millis(100)).await; + + http_provider.anvil_mine(Some(1), None).await?; + + let err = subscription.recv().await.unwrap_err(); // The error should be either a Timeout or BackendGone from the primary WS provider, // NOT a PubsubUnavailable error (which would indicate HTTP fallback was attempted) @@ -510,20 +487,19 @@ mod tests { } #[tokio::test] - async fn test_stream_with_failover() -> anyhow::Result<()> { - let mut anvil_1 = Some(Anvil::new().block_time_f64(0.1).try_spawn()?); + async fn test_robust_subscription_stream_with_failover() -> anyhow::Result<()> { + let anvil_1 = Anvil::new().try_spawn()?; - let ws_provider = ProviderBuilder::new() - .connect(anvil_1.as_ref().unwrap().ws_endpoint_url().as_str()) - .await?; + let ws_provider = + ProviderBuilder::new().connect(anvil_1.ws_endpoint_url().as_str()).await?; - let anvil_2 = Anvil::new().block_time_f64(0.1).try_spawn()?; + let anvil_2 = Anvil::new().try_spawn()?; let ws_provider_2 = ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; let robust = RobustProviderBuilder::fragile(ws_provider.clone()) - .fallback(ws_provider_2) + .fallback(ws_provider_2.clone()) .subscription_timeout(Duration::from_millis(500)) .build() .await?; @@ -531,24 +507,28 @@ mod tests { let subscription = robust.subscribe_blocks().await?; let mut stream = subscription.into_stream(); - while let Some(result) = stream.next().await { - let Ok(block) = result else { - break; - }; + ws_provider.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); - let block_number = block.number(); + ws_provider.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(2, block.number()); - // At block 10, drop the primary provider to test failover - if block_number == 10 && - let Some(anvil) = anvil_1.take() - { - drop(anvil); - } + // Drop the primary provider to trigger failover + drop(anvil_1); - if block_number >= 11 { - break; - } - } + // Wait a bit for the connection to be detected as closed + sleep(Duration::from_millis(800)).await; + + ws_provider_2.anvil_mine(Some(1), None).await?; + + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + ws_provider_2.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(2, block.number()); Ok(()) } From a2b0f544e2f81f2754d4e2ff070caf6052972977 Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 18:04:22 +0900 Subject: [PATCH 41/50] ref: imports --- examples/historical_scanning/main.rs | 2 +- examples/latest_events_scanning/main.rs | 2 +- examples/live_scanning/main.rs | 2 +- examples/sync_from_block_scanning/main.rs | 2 +- examples/sync_from_latest_scanning/main.rs | 2 +- src/block_range_scanner.rs | 7 ++----- src/event_scanner/scanner/mod.rs | 12 ++++++------ src/event_scanner/scanner/sync/mod.rs | 6 +++--- src/robust_provider/mod.rs | 2 +- src/robust_provider/provider.rs | 2 +- 10 files changed, 18 insertions(+), 21 deletions(-) diff --git a/examples/historical_scanning/main.rs b/examples/historical_scanning/main.rs index d5633ee0..1d2cc039 100644 --- a/examples/historical_scanning/main.rs +++ b/examples/historical_scanning/main.rs @@ -2,7 +2,7 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, + EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, }; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/latest_events_scanning/main.rs b/examples/latest_events_scanning/main.rs index b3e323a5..a51c404f 100644 --- a/examples/latest_events_scanning/main.rs +++ b/examples/latest_events_scanning/main.rs @@ -1,7 +1,7 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, + EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, }; use tokio_stream::StreamExt; use tracing::{error, info}; diff --git a/examples/live_scanning/main.rs b/examples/live_scanning/main.rs index a4c66d00..dd888e20 100644 --- a/examples/live_scanning/main.rs +++ b/examples/live_scanning/main.rs @@ -1,7 +1,7 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, + EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, }; use tokio_stream::StreamExt; diff --git a/examples/sync_from_block_scanning/main.rs b/examples/sync_from_block_scanning/main.rs index beb77998..35c0c254 100644 --- a/examples/sync_from_block_scanning/main.rs +++ b/examples/sync_from_block_scanning/main.rs @@ -3,7 +3,7 @@ use std::time::Duration; use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, + EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, }; use tokio::time::sleep; use tokio_stream::StreamExt; diff --git a/examples/sync_from_latest_scanning/main.rs b/examples/sync_from_latest_scanning/main.rs index a83ffbec..c8ba8cfb 100644 --- a/examples/sync_from_latest_scanning/main.rs +++ b/examples/sync_from_latest_scanning/main.rs @@ -1,7 +1,7 @@ use alloy::{providers::ProviderBuilder, sol, sol_types::SolEvent}; use alloy_node_bindings::Anvil; use event_scanner::{ - EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder, + EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder, }; use tokio_stream::StreamExt; diff --git a/src/block_range_scanner.rs b/src/block_range_scanner.rs index 39ef3b2d..6886ffc3 100644 --- a/src/block_range_scanner.rs +++ b/src/block_range_scanner.rs @@ -12,7 +12,7 @@ //! BlockRangeScanner, BlockRangeScannerClient, DEFAULT_BLOCK_CONFIRMATIONS, //! DEFAULT_MAX_BLOCK_RANGE, Message, //! }, -//! robust_provider::builder::RobustProviderBuilder, +//! robust_provider::RobustProviderBuilder, //! }; //! use tokio::time::Duration; //! use tracing::{error, info}; @@ -69,10 +69,7 @@ use tokio_stream::{StreamExt, wrappers::ReceiverStream}; use crate::{ ScannerMessage, error::ScannerError, - robust_provider::{ - Error as RobustProviderError, provider::RobustProvider, - provider_conversion::IntoRobustProvider, - }, + robust_provider::{Error as RobustProviderError, IntoRobustProvider, RobustProvider}, types::{ScannerStatus, TryStream}, }; use alloy::{ diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index dd5a94d8..5fa19550 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -12,7 +12,7 @@ use crate::{ MAX_BUFFERED_MESSAGES, }, event_scanner::listener::EventListener, - robust_provider::provider_conversion::IntoRobustProvider, + robust_provider::IntoRobustProvider, }; mod common; @@ -78,7 +78,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, primitives::Address, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -104,7 +104,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Stream events between blocks [1_000_000, 2_000_000] @@ -144,7 +144,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -232,7 +232,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -259,7 +259,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Collect the latest 5 events between blocks [1_000_000, 1_100_000] diff --git a/src/event_scanner/scanner/sync/mod.rs b/src/event_scanner/scanner/sync/mod.rs index 7d1944f6..ffdae097 100644 --- a/src/event_scanner/scanner/sync/mod.rs +++ b/src/event_scanner/scanner/sync/mod.rs @@ -25,7 +25,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -123,7 +123,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # /// # async fn example() -> Result<(), Box> { @@ -163,7 +163,7 @@ impl EventScannerBuilder { /// /// ```no_run /// # use alloy::{network::Ethereum, eips::BlockNumberOrTag, providers::{Provider, ProviderBuilder}}; - /// # use event_scanner::{EventScannerBuilder, robust_provider::builder::RobustProviderBuilder}; + /// # use event_scanner::{EventScannerBuilder, robust_provider::RobustProviderBuilder}; /// # /// # async fn example() -> Result<(), Box> { /// // Sync from genesis block diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 1952ce5e..874486aa 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -6,4 +6,4 @@ pub mod provider_conversion; pub use builder::RobustProviderBuilder; pub use error::Error; pub use provider::RobustProvider; -pub use provider_conversion::IntoProvider; +pub use provider_conversion::{IntoProvider, IntoRobustProvider}; diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 1cf45dfa..fecb9e79 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -270,7 +270,7 @@ impl RobustProvider { #[cfg(test)] mod tests { use super::*; - use crate::robust_provider::builder::RobustProviderBuilder; + use crate::robust_provider::RobustProviderBuilder; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, From f6c242576ecbbe15eb707307682586f77a713fa6 Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 18:08:33 +0900 Subject: [PATCH 42/50] ref: export all builder --- src/robust_provider/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/robust_provider/mod.rs b/src/robust_provider/mod.rs index 874486aa..8aa800a1 100644 --- a/src/robust_provider/mod.rs +++ b/src/robust_provider/mod.rs @@ -3,7 +3,7 @@ pub mod error; pub mod provider; pub mod provider_conversion; -pub use builder::RobustProviderBuilder; +pub use builder::*; pub use error::Error; pub use provider::RobustProvider; pub use provider_conversion::{IntoProvider, IntoRobustProvider}; From 12e3a91f5b276f6da0104f43da6bee439b43147e Mon Sep 17 00:00:00 2001 From: Leo Date: Thu, 13 Nov 2025 19:06:48 +0900 Subject: [PATCH 43/50] ref: remove primitive import --- src/event_scanner/scanner/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/event_scanner/scanner/mod.rs b/src/event_scanner/scanner/mod.rs index 5fa19550..c8d87640 100644 --- a/src/event_scanner/scanner/mod.rs +++ b/src/event_scanner/scanner/mod.rs @@ -77,7 +77,7 @@ impl EventScannerBuilder { /// # Example /// /// ```no_run - /// # use alloy::{network::Ethereum, primitives::Address, providers::{Provider, ProviderBuilder}}; + /// # use alloy::{network::Ethereum, providers::{Provider, ProviderBuilder}}; /// # use event_scanner::{EventFilter, EventScannerBuilder, Message, robust_provider::RobustProviderBuilder}; /// # use tokio_stream::StreamExt; /// # From 8570f69e42097606f793bf314915987d8d972bbe Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 17:35:25 +0900 Subject: [PATCH 44/50] test: reconnect to primary --- src/robust_provider/builder.rs | 18 +++++++- src/robust_provider/provider.rs | 68 +++++++++++++++++++++++++++-- src/robust_provider/subscription.rs | 5 +-- 3 files changed, 82 insertions(+), 9 deletions(-) diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 4ca299ea..3c41ffd0 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -2,7 +2,9 @@ use std::{marker::PhantomData, time::Duration}; use alloy::{network::Network, providers::Provider}; -use crate::robust_provider::{Error, IntoProvider, RobustProvider}; +use crate::robust_provider::{ + Error, IntoProvider, RobustProvider, subscription::DEFAULT_RECONNECT_INTERVAL, +}; // RPC retry and timeout settings /// Default timeout used by `RobustProvider` @@ -21,6 +23,7 @@ pub struct RobustProviderBuilder> { subscription_timeout: Duration, max_retries: usize, min_delay: Duration, + reconnect_interval: Duration, _network: PhantomData, } @@ -36,6 +39,7 @@ impl> RobustProviderBuilder { subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, min_delay: DEFAULT_MIN_DELAY, + reconnect_interval: DEFAULT_RECONNECT_INTERVAL, _network: PhantomData, } } @@ -88,6 +92,17 @@ impl> RobustProviderBuilder { self } + /// Set the interval for attempting to reconnect to the primary provider. + /// + /// After a failover to a fallback provider, the subscription will periodically + /// attempt to reconnect to the primary provider at this interval. + /// Default is [`DEFAULT_RECONNECT_INTERVAL`]. + #[must_use] + pub fn reconnect_interval(mut self, reconnect_interval: Duration) -> Self { + self.reconnect_interval = reconnect_interval; + self + } + /// Build the `RobustProvider`. /// /// Final builder method: consumes the builder and returns the built [`RobustProvider`]. @@ -106,6 +121,7 @@ impl> RobustProviderBuilder { subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, min_delay: self.min_delay, + reconnect_interval: self.reconnect_interval, }) } } diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index 6548c483..db84aad0 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -12,7 +12,7 @@ use backon::{ExponentialBuilder, Retryable}; use tokio::time::timeout; use tracing::{error, info}; -use crate::robust_provider::{Error, RobustSubscription, subscription::DEFAULT_RECONNECT_INTERVAL}; +use crate::robust_provider::{Error, RobustSubscription}; /// Provider wrapper with built-in retry and timeout mechanisms. /// @@ -26,6 +26,7 @@ pub struct RobustProvider { pub(crate) subscription_timeout: Duration, pub(crate) max_retries: usize, pub(crate) min_delay: Duration, + pub(crate) reconnect_interval: Duration, } impl RobustProvider { @@ -164,7 +165,7 @@ impl RobustProvider { .await; match subscription { - Ok(sub) => Ok(RobustSubscription::new(sub, self.clone(), DEFAULT_RECONNECT_INTERVAL)), + Ok(sub) => Ok(RobustSubscription::new(sub, self.clone())), Err(e) => { error!(error = %e, "eth_subscribe failed"); Err(e) @@ -289,7 +290,10 @@ impl RobustProvider { #[cfg(test)] mod tests { use super::*; - use crate::robust_provider::{RobustProviderBuilder, builder::DEFAULT_SUBSCRIPTION_TIMEOUT}; + use crate::robust_provider::{ + RobustProviderBuilder, builder::DEFAULT_SUBSCRIPTION_TIMEOUT, + subscription::DEFAULT_RECONNECT_INTERVAL, + }; use alloy::{ consensus::BlockHeader, providers::{ProviderBuilder, WsConnect, ext::AnvilApi}, @@ -306,6 +310,7 @@ mod tests { subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, min_delay: Duration::from_millis(min_delay), + reconnect_interval: DEFAULT_RECONNECT_INTERVAL, } } @@ -518,7 +523,6 @@ mod tests { // Drop the primary provider to trigger failover drop(anvil_1); - // Wait a bit for the connection to be detected as closed sleep(Duration::from_millis(800)).await; ws_provider_2.anvil_mine(Some(1), None).await?; @@ -532,4 +536,60 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_subscription_reconnects_to_primary() -> anyhow::Result<()> { + let anvil_1 = Anvil::new().try_spawn()?; + let anvil_1_port = anvil_1.port(); + + let ws_provider_1 = + ProviderBuilder::new().connect(anvil_1.ws_endpoint_url().as_str()).await?; + + let anvil_2 = Anvil::new().try_spawn()?; + let ws_provider_2 = + ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; + + let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) + .fallback(ws_provider_2.clone()) + .subscription_timeout(Duration::from_millis(500)) + .reconnect_interval(Duration::from_secs(2)) + .build() + .await?; + + let subscription = robust.subscribe_blocks().await?; + + let mut stream = subscription.into_stream(); + + ws_provider_1.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + drop(anvil_1); + drop(ws_provider_1); + + // Wait a bit and receive from fallback (will timeout on primary and failover) + sleep(Duration::from_millis(800)).await; + ws_provider_2.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + // Spawn new anvil on the same port as primary (simulating primary coming back) + let anvil_3 = Anvil::new().port(anvil_1_port).try_spawn()?; + + let ws_provider_1 = + ProviderBuilder::new().connect(anvil_3.ws_endpoint_url().as_str()).await?; + + // Wait for reconnect interval to elapse (2 seconds) plus buffer + sleep(Duration::from_millis(2200)).await; + + ws_provider_1.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + ws_provider_1.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(2, block.number()); + + Ok(()) + } } diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs index bb10dde9..731bb399 100644 --- a/src/robust_provider/subscription.rs +++ b/src/robust_provider/subscription.rs @@ -34,7 +34,6 @@ pub const MAX_BUFFERED_BLOCKS: usize = 50000; pub struct RobustSubscription { subscription: Option>, robust_provider: RobustProvider, - reconnect_interval: Duration, last_reconnect_attempt: Option, consecutive_lags: usize, } @@ -44,12 +43,10 @@ impl RobustSubscription { pub(crate) fn new( subscription: Subscription, robust_provider: RobustProvider, - reconnect_interval: Duration, ) -> Self { Self { subscription: Some(subscription), robust_provider, - reconnect_interval, last_reconnect_attempt: None, consecutive_lags: 0, } @@ -128,7 +125,7 @@ impl RobustSubscription { // Only attempt reconnection if enough time has passed since last attempt match self.last_reconnect_attempt { None => false, - Some(last_attempt) => last_attempt.elapsed() >= self.reconnect_interval, + Some(last_attempt) => last_attempt.elapsed() >= self.robust_provider.reconnect_interval, } } From 1307368ccce94efeb1de61bb3ecd0c31b4efe940 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 18:12:07 +0900 Subject: [PATCH 45/50] test: more robust subscription scenarios --- src/robust_provider/provider.rs | 82 ++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 2 deletions(-) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index db84aad0..a7aa37bc 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -565,9 +565,7 @@ mod tests { assert_eq!(1, block.number()); drop(anvil_1); - drop(ws_provider_1); - // Wait a bit and receive from fallback (will timeout on primary and failover) sleep(Duration::from_millis(800)).await; ws_provider_2.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; @@ -592,4 +590,84 @@ mod tests { Ok(()) } + + #[tokio::test] + async fn test_subscription_cycles_through_multiple_fallbacks() -> anyhow::Result<()> { + let anvil_1 = Anvil::new().try_spawn()?; + let ws_provider_1 = + ProviderBuilder::new().connect(anvil_1.ws_endpoint_url().as_str()).await?; + + let anvil_2 = Anvil::new().try_spawn()?; + let ws_provider_2 = + ProviderBuilder::new().connect(anvil_2.ws_endpoint_url().as_str()).await?; + + let anvil_3 = Anvil::new().try_spawn()?; + let ws_provider_3 = + ProviderBuilder::new().connect(anvil_3.ws_endpoint_url().as_str()).await?; + + let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) + .fallback(ws_provider_2.clone()) + .fallback(ws_provider_3.clone()) + .subscription_timeout(Duration::from_millis(500)) + .build() + .await?; + + let subscription = robust.subscribe_blocks().await?; + + let mut stream = subscription.into_stream(); + + ws_provider_1.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + drop(anvil_1); + + sleep(Duration::from_millis(600)).await; + + ws_provider_2.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + drop(anvil_2); + + sleep(Duration::from_millis(600)).await; + + ws_provider_3.anvil_mine(Some(1), None).await?; + let block = stream.next().await.unwrap()?; + assert_eq!(1, block.number()); + + Ok(()) + } + + #[tokio::test] + async fn test_subscription_fails_with_no_fallbacks() -> anyhow::Result<()> { + let anvil = Anvil::new().try_spawn()?; + let ws_provider = ProviderBuilder::new().connect(anvil.ws_endpoint_url().as_str()).await?; + + let robust = RobustProviderBuilder::fragile(ws_provider.clone()) + .subscription_timeout(Duration::from_millis(500)) + .build() + .await?; + + let mut subscription = robust.subscribe_blocks().await?; + + // Receive initial block successfully + ws_provider.anvil_mine(Some(1), None).await?; + let _block = subscription.recv().await?; + + drop(anvil); + + sleep(Duration::from_millis(600)).await; + + let err = subscription.recv().await.unwrap_err(); + + match err { + Error::Timeout => {} + Error::RpcError(e) => { + assert!(matches!(e.as_ref(), RpcError::Transport(TransportErrorKind::BackendGone))); + } + Error::BlockNotFound(_) => panic!("Unexpected error type"), + } + Ok(()) + } } From cfa95cf6dfc388674083589add7e256f7d663fd6 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 21:55:21 +0900 Subject: [PATCH 46/50] feat: add primary and fallback fields seperately --- src/robust_provider/builder.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/robust_provider/builder.rs b/src/robust_provider/builder.rs index 3c41ffd0..093cd1b7 100644 --- a/src/robust_provider/builder.rs +++ b/src/robust_provider/builder.rs @@ -18,7 +18,8 @@ pub const DEFAULT_MIN_DELAY: Duration = Duration::from_secs(1); #[derive(Clone)] pub struct RobustProviderBuilder> { - providers: Vec

, + primary_provider: P, + fallback_providers: Vec

, call_timeout: Duration, subscription_timeout: Duration, max_retries: usize, @@ -34,7 +35,8 @@ impl> RobustProviderBuilder { #[must_use] pub fn new(provider: P) -> Self { Self { - providers: vec![provider], + primary_provider: provider, + fallback_providers: vec![], call_timeout: DEFAULT_CALL_TIMEOUT, subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries: DEFAULT_MAX_RETRIES, @@ -57,7 +59,7 @@ impl> RobustProviderBuilder { /// Fallback providers are used when the primary provider times out or fails. #[must_use] pub fn fallback(mut self, provider: P) -> Self { - self.providers.push(provider); + self.fallback_providers.push(provider); self } @@ -111,12 +113,16 @@ impl> RobustProviderBuilder { /// /// Returns an error if any of the providers fail to connect. pub async fn build(self) -> Result, Error> { - let mut providers = vec![]; - for p in self.providers { - providers.push(p.into_provider().await?.root().to_owned()); + let primary_provider = self.primary_provider.into_provider().await?.root().to_owned(); + + let mut fallback_providers = vec![]; + for p in self.fallback_providers { + fallback_providers.push(p.into_provider().await?.root().to_owned()); } + Ok(RobustProvider { - providers, + primary_provider, + fallback_providers, call_timeout: self.call_timeout, subscription_timeout: self.subscription_timeout, max_retries: self.max_retries, From 59d9c49bcacedc151f9fe6432eec98dcd42e5ce3 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 22:10:54 +0900 Subject: [PATCH 47/50] feat: add try fallback from logic --- src/robust_provider/provider.rs | 64 +++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 23 deletions(-) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index a7aa37bc..fb611676 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -18,10 +18,10 @@ use crate::robust_provider::{Error, RobustSubscription}; /// /// This wrapper around Alloy providers automatically handles retries, /// timeouts, and error logging for RPC calls. -/// The first provider in the vector is treated as the primary provider. #[derive(Clone, Debug)] pub struct RobustProvider { - pub(crate) providers: Vec>, + pub(crate) primary_provider: RootProvider, + pub(crate) fallback_providers: Vec>, pub(crate) call_timeout: Duration, pub(crate) subscription_timeout: Duration, pub(crate) max_retries: usize, @@ -30,15 +30,10 @@ pub struct RobustProvider { } impl RobustProvider { - /// Get a reference to the primary provider (the first provider in the list) - /// - /// # Panics - /// - /// If there are no providers set (this should never happen) + /// Get a reference to the primary provider #[must_use] pub fn primary(&self) -> &RootProvider { - // Safe to unwrap because we always have at least one provider - self.providers.first().expect("providers vector should never be empty") + &self.primary_provider } /// Fetch a block by number with retry and timeout. @@ -218,32 +213,48 @@ impl RobustProvider { &self, operation: F, require_pubsub: bool, - mut last_error: Error, + last_error: Error, ) -> Result where F: Fn(RootProvider) -> Fut, Fut: Future>>, { - let num_providers = self.providers.len(); - if num_providers > 1 { + self.try_fallback_providers_from(operation, require_pubsub, last_error, 0) + .await + .map(|(value, _idx)| value) + } + + pub(crate) async fn try_fallback_providers_from( + &self, + operation: F, + require_pubsub: bool, + mut last_error: Error, + start_index: usize, + ) -> Result<(T, usize), Error> + where + F: Fn(RootProvider) -> Fut, + Fut: Future>>, + { + let num_fallbacks = self.fallback_providers.len(); + if num_fallbacks > 0 && start_index == 0 { info!("Primary provider failed, trying fallback provider(s)"); } - let fallback_providers = self.providers.iter().skip(1); - for (idx, provider) in fallback_providers.enumerate() { - let fallback_num = idx + 1; + + let fallback_providers = self.fallback_providers.iter().enumerate().skip(start_index); + for (fallback_idx, provider) in fallback_providers { if require_pubsub && !Self::supports_pubsub(provider) { - info!("Fallback provider {} doesn't support pubsub, skipping", fallback_num); + info!("Fallback provider {} doesn't support pubsub, skipping", fallback_idx + 1); continue; } - info!("Attempting fallback provider {}/{}", fallback_num, num_providers - 1); + info!("Attempting fallback provider {}/{}", fallback_idx + 1, num_fallbacks); match self.try_provider_with_timeout(provider, &operation).await { Ok(value) => { - info!(provider_num = fallback_num, "Fallback provider succeeded"); - return Ok(value); + info!(provider_num = fallback_idx + 1, "Fallback provider succeeded"); + return Ok((value, fallback_idx)); } Err(e) => { - error!(provider_num = fallback_num, err = %e, "Fallback provider failed"); + error!(provider_num = fallback_idx + 1, err = %e, "Fallback provider failed"); last_error = e; } } @@ -305,7 +316,8 @@ mod tests { fn test_provider(timeout: u64, max_retries: usize, min_delay: u64) -> RobustProvider { RobustProvider { - providers: vec![RootProvider::new_http("http://localhost:8545".parse().unwrap())], + primary_provider: RootProvider::new_http("http://localhost:8545".parse().unwrap()), + fallback_providers: vec![], call_timeout: Duration::from_millis(timeout), subscription_timeout: DEFAULT_SUBSCRIPTION_TIMEOUT, max_retries, @@ -523,8 +535,11 @@ mod tests { // Drop the primary provider to trigger failover drop(anvil_1); + // Wait for subscription timeout to occur and switch to fallback + // The subscription will timeout after 500ms of inactivity, then switch to fallback sleep(Duration::from_millis(800)).await; + // Now mine blocks on fallback - the subscription should be connected to fallback now ws_provider_2.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; @@ -551,7 +566,7 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) - .subscription_timeout(Duration::from_millis(500)) + .subscription_timeout(Duration::from_secs(5)) .reconnect_interval(Duration::from_secs(2)) .build() .await?; @@ -566,7 +581,10 @@ mod tests { drop(anvil_1); - sleep(Duration::from_millis(800)).await; + // Wait for subscription to detect failure and switch to fallback + // (subscription_timeout is 5 seconds) + sleep(Duration::from_millis(5500)).await; + ws_provider_2.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; assert_eq!(1, block.number()); From f1f6369ef092cf79f3236b15c22020268fb984c3 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 22:17:29 +0900 Subject: [PATCH 48/50] feat: add fallback provider tracking --- src/robust_provider/subscription.rs | 77 ++++++++++++++++++----------- 1 file changed, 47 insertions(+), 30 deletions(-) diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs index 731bb399..35670ee3 100644 --- a/src/robust_provider/subscription.rs +++ b/src/robust_provider/subscription.rs @@ -36,6 +36,7 @@ pub struct RobustSubscription { robust_provider: RobustProvider, last_reconnect_attempt: Option, consecutive_lags: usize, + current_fallback_index: Option, } impl RobustSubscription { @@ -49,6 +50,7 @@ impl RobustSubscription { robust_provider, last_reconnect_attempt: None, consecutive_lags: 0, + current_fallback_index: None, } } @@ -64,7 +66,9 @@ impl RobustSubscription { /// /// Returns an error if all providers have been exhausted and failed. pub async fn recv(&mut self) -> Result { + let subscription_timeout = self.robust_provider.subscription_timeout; loop { + // Check if we should reconnect to primary before attempting to receive if self.should_reconnect_to_primary() { info!("Attempting to reconnect to primary provider"); if let Err(e) = self.try_reconnect_to_primary().await { @@ -75,36 +79,16 @@ impl RobustSubscription { } if let Some(subscription) = &mut self.subscription { - let subscription_timeout = self.robust_provider.subscription_timeout; - match timeout(subscription_timeout, subscription.recv()).await { + let recv_result = timeout(subscription_timeout, subscription.recv()).await; + match recv_result { Ok(recv_result) => match recv_result { Ok(header) => { self.consecutive_lags = 0; return Ok(header); } - Err(recv_error) => match recv_error { - RecvError::Closed => { - error!("Subscription channel closed, switching provider"); - let error = - RpcError::Transport(TransportErrorKind::BackendGone).into(); - self.switch_to_fallback(error).await?; - } - RecvError::Lagged(skipped) => { - self.consecutive_lags += 1; - warn!( - skipped = skipped, - consecutive_lags = self.consecutive_lags, - "Subscription lagged" - ); - - if self.consecutive_lags >= MAX_LAG_COUNT { - error!("Too many consecutive lags, switching provider"); - let error = - RpcError::Transport(TransportErrorKind::BackendGone).into(); - self.switch_to_fallback(error).await?; - } - } - }, + Err(recv_error) => { + self.process_recv_error(recv_error).await?; + } }, Err(e) => { error!( @@ -121,6 +105,32 @@ impl RobustSubscription { } } + /// Process subscription receive errors and handle failover + async fn process_recv_error(&mut self, recv_error: RecvError) -> Result<(), Error> { + match recv_error { + RecvError::Closed => { + error!("Subscription channel closed, switching provider"); + let error = RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + RecvError::Lagged(skipped) => { + self.consecutive_lags += 1; + warn!( + skipped = skipped, + consecutive_lags = self.consecutive_lags, + "Subscription lagged" + ); + + if self.consecutive_lags >= MAX_LAG_COUNT { + error!("Too many consecutive lags, switching provider"); + let error = RpcError::Transport(TransportErrorKind::BackendGone).into(); + self.switch_to_fallback(error).await?; + } + } + } + Ok(()) + } + fn should_reconnect_to_primary(&self) -> bool { // Only attempt reconnection if enough time has passed since last attempt match self.last_reconnect_attempt { @@ -142,6 +152,7 @@ impl RobustSubscription { match subscription { Ok(sub) => { self.subscription = Some(sub); + self.current_fallback_index = None; Ok(()) } Err(e) => { @@ -158,16 +169,23 @@ impl RobustSubscription { let operation = move |provider: RootProvider| async move { provider.subscribe_blocks().await }; - let subscription = - self.robust_provider.try_fallback_providers(&operation, true, last_error).await; + + // Start searching from the next provider after the current one + let start_index = self.current_fallback_index.map_or(0, |idx| idx + 1); + + let subscription = self + .robust_provider + .try_fallback_providers_from(&operation, true, last_error, start_index) + .await; match subscription { - Ok(sub) => { + Ok((sub, fallback_idx)) => { self.subscription = Some(sub); + self.current_fallback_index = Some(fallback_idx); Ok(()) } Err(e) => { - error!(error = %e, "eth_subscribe failed"); + error!(error = %e, "eth_subscribe failed - no fallbacks available"); Err(e) } } @@ -200,7 +218,6 @@ impl RobustSubscription { } } Err(e) => { - // Send the error and exit if let Err(err) = tx.send(Err(e)).await { warn!(error = %err, "Downstream channel closed, stopping stream"); } From d50aa758a04e219373ab4671b169cff0f0d5f238 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 14 Nov 2025 23:10:03 +0900 Subject: [PATCH 49/50] ref: refactor try connect primary to be simpler --- src/robust_provider/subscription.rs | 38 ++++++++++++++--------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs index 35670ee3..fc3a96b8 100644 --- a/src/robust_provider/subscription.rs +++ b/src/robust_provider/subscription.rs @@ -68,15 +68,7 @@ impl RobustSubscription { pub async fn recv(&mut self) -> Result { let subscription_timeout = self.robust_provider.subscription_timeout; loop { - // Check if we should reconnect to primary before attempting to receive - if self.should_reconnect_to_primary() { - info!("Attempting to reconnect to primary provider"); - if let Err(e) = self.try_reconnect_to_primary().await { - warn!(error = %e, "Failed to reconnect to primary provider"); - } else { - info!("Successfully reconnected to primary provider"); - } - } + self.try_reconnect_to_primary().await; if let Some(subscription) = &mut self.subscription { let recv_result = timeout(subscription_timeout, subscription.recv()).await; @@ -90,12 +82,13 @@ impl RobustSubscription { self.process_recv_error(recv_error).await?; } }, - Err(e) => { + Err(elapsed_err) => { error!( timeout_secs = subscription_timeout.as_secs(), "Subscription timeout - no block received, switching provider" ); - self.switch_to_fallback(e.into()).await?; + + self.switch_to_fallback(elapsed_err.into()).await?; } } } else { @@ -131,15 +124,21 @@ impl RobustSubscription { Ok(()) } - fn should_reconnect_to_primary(&self) -> bool { - // Only attempt reconnection if enough time has passed since last attempt - match self.last_reconnect_attempt { + /// Try to reconnect to the primary provider if enough time has elapsed. + /// Returns true if reconnection was successful, false if it's not time yet or if it failed. + async fn try_reconnect_to_primary(&mut self) -> bool { + // Check if we should attempt reconnection + let should_reconnect = match self.last_reconnect_attempt { None => false, Some(last_attempt) => last_attempt.elapsed() >= self.robust_provider.reconnect_interval, + }; + + if !should_reconnect { + return false; } - } - async fn try_reconnect_to_primary(&mut self) -> Result<(), Error> { + info!("Attempting to reconnect to primary provider"); + self.last_reconnect_attempt = Some(Instant::now()); let operation = @@ -151,13 +150,14 @@ impl RobustSubscription { match subscription { Ok(sub) => { + info!("Successfully reconnected to primary provider"); self.subscription = Some(sub); self.current_fallback_index = None; - Ok(()) + true } Err(e) => { - error!(error = %e, "eth_subscribe failed"); - Err(e) + warn!(error = %e, "Failed to reconnect to primary provider"); + false } } } From 4378c9ee0eb45176217eb9ac821a2b7c020f79dd Mon Sep 17 00:00:00 2001 From: Leo Date: Sat, 15 Nov 2025 00:08:05 +0900 Subject: [PATCH 50/50] ref: simplify test and add better timeout logic --- src/robust_provider/provider.rs | 59 ++++++++++------------------- src/robust_provider/subscription.rs | 9 +++++ 2 files changed, 29 insertions(+), 39 deletions(-) diff --git a/src/robust_provider/provider.rs b/src/robust_provider/provider.rs index fb611676..5d734818 100644 --- a/src/robust_provider/provider.rs +++ b/src/robust_provider/provider.rs @@ -482,9 +482,8 @@ mod tests { ws_provider.anvil_mine(Some(1), None).await?; assert_eq!(2, subscription.recv().await?.number()); - drop(anvil_1); - - sleep(Duration::from_millis(100)).await; + // simulate ws stream gone via no blocks mined > sub timeout + sleep(Duration::from_millis(600)).await; http_provider.anvil_mine(Some(1), None).await?; @@ -532,12 +531,8 @@ mod tests { let block = stream.next().await.unwrap()?; assert_eq!(2, block.number()); - // Drop the primary provider to trigger failover - drop(anvil_1); - - // Wait for subscription timeout to occur and switch to fallback - // The subscription will timeout after 500ms of inactivity, then switch to fallback - sleep(Duration::from_millis(800)).await; + // simulate ws stream gone via no blocks mined > sub timeout + sleep(Duration::from_millis(600)).await; // Now mine blocks on fallback - the subscription should be connected to fallback now ws_provider_2.anvil_mine(Some(1), None).await?; @@ -555,8 +550,6 @@ mod tests { #[tokio::test] async fn test_subscription_reconnects_to_primary() -> anyhow::Result<()> { let anvil_1 = Anvil::new().try_spawn()?; - let anvil_1_port = anvil_1.port(); - let ws_provider_1 = ProviderBuilder::new().connect(anvil_1.ws_endpoint_url().as_str()).await?; @@ -566,45 +559,36 @@ mod tests { let robust = RobustProviderBuilder::fragile(ws_provider_1.clone()) .fallback(ws_provider_2.clone()) - .subscription_timeout(Duration::from_secs(5)) + .subscription_timeout(Duration::from_millis(500)) .reconnect_interval(Duration::from_secs(2)) .build() .await?; let subscription = robust.subscribe_blocks().await?; - let mut stream = subscription.into_stream(); + // Verify primary works ws_provider_1.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; assert_eq!(1, block.number()); - drop(anvil_1); - - // Wait for subscription to detect failure and switch to fallback - // (subscription_timeout is 5 seconds) - sleep(Duration::from_millis(5500)).await; + sleep(Duration::from_millis(600)).await; + // Verify fallback works ws_provider_2.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; assert_eq!(1, block.number()); - // Spawn new anvil on the same port as primary (simulating primary coming back) - let anvil_3 = Anvil::new().port(anvil_1_port).try_spawn()?; - - let ws_provider_1 = - ProviderBuilder::new().connect(anvil_3.ws_endpoint_url().as_str()).await?; - - // Wait for reconnect interval to elapse (2 seconds) plus buffer - sleep(Duration::from_millis(2200)).await; + for _ in 0..30 { + ws_provider_2.anvil_mine(Some(10), None).await?; + let _ = stream.next().await.unwrap()?; + sleep(Duration::from_millis(100)).await; + // Mine on primary - should reconnect and receive from primary + ws_provider_1.anvil_mine(Some(1), None).await?; + } - ws_provider_1.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; - assert_eq!(1, block.number()); - - ws_provider_1.anvil_mine(Some(1), None).await?; - let block = stream.next().await.unwrap()?; - assert_eq!(2, block.number()); + assert_eq!(31 + 1, block.number()); Ok(()) } @@ -638,16 +622,14 @@ mod tests { let block = stream.next().await.unwrap()?; assert_eq!(1, block.number()); - drop(anvil_1); - + // simulate ws stream gone via no blocks mined > sub timeout sleep(Duration::from_millis(600)).await; ws_provider_2.anvil_mine(Some(1), None).await?; let block = stream.next().await.unwrap()?; assert_eq!(1, block.number()); - drop(anvil_2); - + // simulate ws stream gone via no blocks mined > sub timeout sleep(Duration::from_millis(600)).await; ws_provider_3.anvil_mine(Some(1), None).await?; @@ -669,12 +651,11 @@ mod tests { let mut subscription = robust.subscribe_blocks().await?; - // Receive initial block successfully + // simulate ws stream gone via no blocks mined > sub timeout ws_provider.anvil_mine(Some(1), None).await?; let _block = subscription.recv().await?; - drop(anvil); - + // simulate ws stream gone via no blocks mined > sub timeout sleep(Duration::from_millis(600)).await; let err = subscription.recv().await.unwrap_err(); diff --git a/src/robust_provider/subscription.rs b/src/robust_provider/subscription.rs index fc3a96b8..cf153b01 100644 --- a/src/robust_provider/subscription.rs +++ b/src/robust_provider/subscription.rs @@ -88,6 +88,14 @@ impl RobustSubscription { "Subscription timeout - no block received, switching provider" ); + // If we're on a fallback, try reconnecting to primary one more time + // before switching to the next fallback + if self.current_fallback_index.is_some() && + self.try_reconnect_to_primary().await + { + continue; + } + self.switch_to_fallback(elapsed_err.into()).await?; } } @@ -156,6 +164,7 @@ impl RobustSubscription { true } Err(e) => { + println!("what"); warn!(error = %e, "Failed to reconnect to primary provider"); false }