Switch from reqwest to bitreq

Cargo.toml

@@ -56,7 +56,7 @@ bdk_esplora = { version = "0.22.0", default-features = false, features = ["async
 bdk_electrum = { version = "0.23.0", default-features = false, features = ["use-rustls-ring"]}
 bdk_wallet = { version = "2.2.0", default-features = false, features = ["std", "keys-bip39"]}
 
-reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
+bitreq = { version = "0.3", default-features = false, features = ["async-https"] }
 rustls = { version = "0.23", default-features = false }
 rusqlite = { version = "0.31.0", features = ["bundled"] }
 bitcoin = "0.32.7"

src/config.rs

@@ -87,12 +87,20 @@ pub(crate) const FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS: u64 = 5;
 // The timeout after which we abort a transaction broadcast operation.
 pub(crate) const TX_BROADCAST_TIMEOUT_SECS: u64 = 5;
 
+// The maximum encoded size of an RGS snapshot we'll accept.
+// In practice the maximum we see is around 4MiB.
+pub(crate) const RGS_SNAPSHOT_MAX_SIZE: usize = 15 * 1024 * 1024;
+
 // The timeout after which we abort a RGS sync operation.
 pub(crate) const RGS_SYNC_TIMEOUT_SECS: u64 = 5;
 
 /// The length in bytes of our wallets' keys seed.
 pub const WALLET_KEYS_SEED_LEN: usize = 64;
 
+// The maximum encoded size of external scores we'll accept.
+// In practice we see scores files in the 5MiB range.
+pub(crate) const EXTERNAL_PATHFINDING_SCORES_MAX_SIZE: usize = 20 * 1024 * 1024;
+
 // The timeout after which we abort a external scores sync operation.
 pub(crate) const EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS: u64 = 5;
 

src/gossip.rs

@@ -7,13 +7,12 @@
 
 use std::sync::atomic::{AtomicU32, Ordering};
 use std::sync::Arc;
-use std::time::Duration;
 
 use lightning_block_sync::gossip::GossipVerifier;
 
 use crate::chain::ChainSource;
-use crate::config::RGS_SYNC_TIMEOUT_SECS;
-use crate::logger::{log_trace, LdkLogger, Logger};
+use crate::config::{RGS_SNAPSHOT_MAX_SIZE, RGS_SYNC_TIMEOUT_SECS};
+use crate::logger::{log_error, log_trace, LdkLogger, Logger};
 use crate::runtime::{Runtime, RuntimeSpawner};
 use crate::types::{GossipSync, Graph, P2PGossipSync, RapidGossipSync};
 use crate::Error;
@@ -70,29 +69,18 @@ impl GossipSource {
 				let query_timestamp = latest_sync_timestamp.load(Ordering::Acquire);
 				let query_url = format!("{}/{}", server_url, query_timestamp);
 
-				let response = tokio::time::timeout(
-					Duration::from_secs(RGS_SYNC_TIMEOUT_SECS),
-					reqwest::get(query_url),
-				)
-				.await
-				.map_err(|e| {
-					log_trace!(logger, "Retrieving RGS gossip update timed out: {}", e);
+				let query = bitreq::get(query_url)
+					.with_max_body_size(Some(RGS_SNAPSHOT_MAX_SIZE))
+					.with_timeout(RGS_SYNC_TIMEOUT_SECS);
+				let response = query.send_async().await.map_err(|e| {
+					log_error!(logger, "Failed to retrieve RGS gossip update: {e}");
 					Error::GossipUpdateTimeout
-				})?
-				.map_err(|e| {
-					log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e);
-					Error::GossipUpdateFailed
 				})?;
 
-				match response.error_for_status() {
-					Ok(res) => {
-						let update_data = res.bytes().await.map_err(|e| {
-							log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e);
-							Error::GossipUpdateFailed
-						})?;
-
+				match response.status_code {
+					200 => {
 						let new_latest_sync_timestamp =
-							gossip_sync.update_network_graph(&update_data).map_err(|e| {
+							gossip_sync.update_network_graph(response.as_bytes()).map_err(|e| {
 								log_trace!(
 									logger,
 									"Failed to update network graph with RGS data: {:?}",
@@ -103,8 +91,8 @@ impl GossipSource {
 						latest_sync_timestamp.store(new_latest_sync_timestamp, Ordering::Release);
 						Ok(new_latest_sync_timestamp)
 					},
-					Err(e) => {
-						log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e);
+					code => {
+						log_trace!(logger, "Failed to retrieve RGS gossip update: HTTP {}", code);
 						Err(Error::GossipUpdateFailed)
 					},
 				}

src/scoring.rs

@@ -1,12 +1,13 @@
 use std::sync::{Arc, Mutex, RwLock};
-use std::time::{Duration, SystemTime};
+use std::time::SystemTime;
 
 use lightning::routing::scoring::ChannelLiquidities;
 use lightning::util::ser::Readable;
 use lightning::{log_error, log_info, log_trace};
 
 use crate::config::{
-	EXTERNAL_PATHFINDING_SCORES_SYNC_INTERVAL, EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS,
+	EXTERNAL_PATHFINDING_SCORES_MAX_SIZE, EXTERNAL_PATHFINDING_SCORES_SYNC_INTERVAL,
+	EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS,
 };
 use crate::io::utils::write_external_pathfinding_scores_to_cache;
 use crate::logger::LdkLogger;
@@ -53,34 +54,27 @@ async fn sync_external_scores(
 	logger: &Logger, scorer: &Mutex<Scorer>, node_metrics: &RwLock<NodeMetrics>,
 	kv_store: Arc<DynStore>, url: &String,
 ) -> () {
-	let response = tokio::time::timeout(
-		Duration::from_secs(EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS),
-		reqwest::get(url),
-	)
-	.await;
+	let request = bitreq::get(url)
+		.with_timeout(EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS)
+		.with_max_body_size(Some(EXTERNAL_PATHFINDING_SCORES_MAX_SIZE));
 
-	let response = match response {
-		Ok(resp) => resp,
-		Err(e) => {
-			log_error!(logger, "Retrieving external scores timed out: {}", e);
-			return;
-		},
-	};
-	let response = match response {
+	let response = match request.send_async().await {
 		Ok(resp) => resp,
 		Err(e) => {
 			log_error!(logger, "Failed to retrieve external scores update: {}", e);
 			return;
 		},
 	};
-	let reader = match response.bytes().await {
-		Ok(bytes) => bytes,
-		Err(e) => {
-			log_error!(logger, "Failed to read external scores update: {}", e);
-			return;
-		},
-	};
-	match ChannelLiquidities::read(&mut &*reader) {
+	if response.status_code != 200 {
+		log_error!(
+			logger,
+			"Failed to retrieve external scores update: HTTP {}",
+			response.status_code
+		);
+		return;
+	}
+	let mut reader = response.as_bytes();
+	match ChannelLiquidities::read(&mut reader) {
 		Ok(liquidities) => {
 			if let Err(e) =
 				write_external_pathfinding_scores_to_cache(&*kv_store, &liquidities, logger).await