feat(sdk)!: detect stale nodes

packages/rs-dapi-client/src/dapi_client.rs

@@ -39,14 +39,14 @@ pub enum DapiClientError<TE: Mockable> {
 }
 
 impl<TE: CanRetry + Mockable> CanRetry for DapiClientError<TE> {
-    fn is_node_failure(&self) -> bool {
+    fn can_retry(&self) -> Option<bool> {
         use DapiClientError::*;
         match self {
-            NoAvailableAddresses => false,
-            Transport(transport_error, _) => transport_error.is_node_failure(),
-            AddressList(_) => false,
+            NoAvailableAddresses => Some(true),
+            Transport(transport_error, _) => transport_error.can_retry(),
+            AddressList(_) => Some(true),
             #[cfg(feature = "mocks")]
-            Mock(_) => false,
+            Mock(_) => None,
         }
     }
 }
@@ -233,7 +233,7 @@ impl DapiRequestExecutor for DapiClient {
                         tracing::trace!(?response, "received {} response", response_name);
                     }
                     Err(error) => {
-                        if error.is_node_failure() {
+                        if !error.can_retry().unwrap_or(false) {
                             if applied_settings.ban_failed_address {
                                 let mut address_list = self
                                     .address_list
@@ -264,13 +264,18 @@ impl DapiRequestExecutor for DapiClient {
                     duration.as_secs_f32()
                 )
             })
-            .when(|e| e.is_node_failure())
+            .when(|e| !e.can_retry().unwrap_or(false))
             .instrument(tracing::info_span!("request routine"))
             .await;
 
         if let Err(error) = &result {
-            if error.is_node_failure() {
-                tracing::error!(?error, "request failed");
+            match error.can_retry() {
+                Some(false) | None => {
+                    tracing::error!(?error, "request failed");
+                }
+                Some(true) => {
+                    tracing::warn!(?error, "request failed, retrying");
+                }
             }
         }
 

packages/rs-dapi-client/src/lib.rs

@@ -71,9 +71,19 @@ impl<T: transport::TransportRequest + Send> DapiRequest for T {
     }
 }
 
-/// Allows to flag the transport error variant how tolerant we are of it and whether we can
-/// try to do a request again.
+/// Returns true if the operation can be retried.
+/// `None` means unspecified - in this case, you should either inspect the error
+/// in more details, or assume `false`.
 pub trait CanRetry {
+    /// Returns true if the operation can be retried safely.
+    /// None value means it's unspecified
+    fn can_retry(&self) -> Option<bool>;
+
     /// Get boolean flag that indicates if the error is retryable.
-    fn is_node_failure(&self) -> bool;
+    ///
+    /// Depreacted in favor of [CanRetry::can_retry].
+    #[deprecated = "Use !can_retry() instead"]
+    fn is_node_failure(&self) -> bool {
+        !self.can_retry().unwrap_or(false)
+    }
 }

packages/rs-dapi-client/src/transport/grpc.rs

@@ -117,11 +117,12 @@ impl TransportClient for CoreGrpcClient {
 }
 
 impl CanRetry for dapi_grpc::tonic::Status {
-    fn is_node_failure(&self) -> bool {
+    fn can_retry(&self) -> Option<bool> {
         let code = self.code();
 
         use dapi_grpc::tonic::Code::*;
-        matches!(
+
+        let retry = !matches!(
             code,
             Ok | DataLoss
                 | Cancelled
@@ -131,7 +132,13 @@ impl CanRetry for dapi_grpc::tonic::Status {
                 | Aborted
                 | Internal
                 | Unavailable
-        )
+        );
+
+        if retry {
+            Some(true)
+        } else {
+            None
+        }
     }
 }
 

packages/rs-drive-proof-verifier/src/error.rs

@@ -82,6 +82,37 @@ pub enum Error {
     /// Context provider error
     #[error("context provider error: {0}")]
     ContextProviderError(#[from] ContextProviderError),
+
+    /// Proof is stale; try another server
+    #[error(transparent)]
+    StaleProof(#[from] StaleProofError),
+}
+
+/// Received proof is stale; try another server
+#[derive(Debug, thiserror::Error)]
+pub enum StaleProofError {
+    /// Stale proof height
+    #[error("stale proof height: expected height {expected_height}, received {received_height}, tolerance {tolerance_blocks}; try another server")]
+    Height {
+        /// Expected height - last block height seen by the Sdk
+        expected_height: u64,
+        /// Block height received from the server in the proof
+        received_height: u64,
+        /// Tolerance - how many blocks can be behind the expected height
+        tolerance_blocks: u64,
+    },
+    /// Proof time is stale
+    #[error(
+        "stale proof time: expected {expected_timestamp_ms}ms, received {received_timestamp_ms} ms, tolerance {tolerance_ms} ms; try another server"
+    )]
+    Time {
+        /// Expected time in milliseconds - is local time when the proof was received
+        expected_timestamp_ms: u64,
+        /// Time received from the server in the proof, in milliseconds
+        received_timestamp_ms: u64,
+        /// Tolerance in milliseconds
+        tolerance_ms: u64,
+    },
 }
 
 /// Errors returned by the context provider

packages/rs-sdk/Cargo.toml

@@ -6,6 +6,7 @@ edition = "2021"
 [dependencies]
 
 arc-swap = { version = "1.7.1" }
+chrono = { version = "0.4.38" }
 dpp = { path = "../rs-dpp", default-features = false, features = [
   "dash-sdk-features",
 ] }
@@ -52,7 +53,6 @@ data-contracts = { path = "../data-contracts" }
 tokio-test = { version = "0.4.4" }
 clap = { version = "4.5.4", features = ["derive"] }
 sanitize-filename = { version = "0.5.0" }
-chrono = { version = "0.4.38" }
 test-case = { version = "3.3.1" }
 
 [features]

packages/rs-sdk/src/error.rs

@@ -5,7 +5,7 @@ use std::time::Duration;
 use dapi_grpc::mock::Mockable;
 use dpp::version::PlatformVersionError;
 use dpp::ProtocolError;
-use rs_dapi_client::DapiClientError;
+use rs_dapi_client::{CanRetry, DapiClientError};
 
 pub use drive_proof_verifier::error::ContextProviderError;
 
@@ -80,3 +80,21 @@ impl From<PlatformVersionError> for Error {
         Self::Protocol(value.into())
     }
 }
+
+impl CanRetry for Error {
+    fn can_retry(&self) -> Option<bool> {
+        let retry = matches!(
+            self,
+            Error::Proof(drive_proof_verifier::Error::StaleProof(..))
+                | Error::DapiClientError(_)
+                | Error::CoreClientError(_)
+                | Error::TimeoutReached(_, _)
+        );
+
+        if retry {
+            Some(true)
+        } else {
+            None
+        }
+    }
+}