diff --git a/Cargo.toml b/Cargo.toml index e0ace123f..9881b5308 100755 --- a/Cargo.toml +++ b/Cargo.toml @@ -56,7 +56,7 @@ bdk_esplora = { version = "0.22.0", default-features = false, features = ["async bdk_electrum = { version = "0.23.0", default-features = false, features = ["use-rustls-ring"]} bdk_wallet = { version = "2.2.0", default-features = false, features = ["std", "keys-bip39"]} -reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } +bitreq = { version = "0.3", default-features = false, features = ["async-https"] } rustls = { version = "0.23", default-features = false } rusqlite = { version = "0.31.0", features = ["bundled"] } bitcoin = "0.32.7" diff --git a/src/config.rs b/src/config.rs index 1b71d0d4e..6c9d1640a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -87,12 +87,20 @@ pub(crate) const FEE_RATE_CACHE_UPDATE_TIMEOUT_SECS: u64 = 5; // The timeout after which we abort a transaction broadcast operation. pub(crate) const TX_BROADCAST_TIMEOUT_SECS: u64 = 5; +// The maximum encoded size of an RGS snapshot we'll accept. +// In practice the maximum we see is around 4MiB. +pub(crate) const RGS_SNAPSHOT_MAX_SIZE: usize = 15 * 1024 * 1024; + // The timeout after which we abort a RGS sync operation. pub(crate) const RGS_SYNC_TIMEOUT_SECS: u64 = 5; /// The length in bytes of our wallets' keys seed. pub const WALLET_KEYS_SEED_LEN: usize = 64; +// The maximum encoded size of external scores we'll accept. +// In practice we see scores files in the 5MiB range. +pub(crate) const EXTERNAL_PATHFINDING_SCORES_MAX_SIZE: usize = 20 * 1024 * 1024; + // The timeout after which we abort a external scores sync operation. pub(crate) const EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS: u64 = 5; diff --git a/src/gossip.rs b/src/gossip.rs index f42b4602c..4ef280273 100644 --- a/src/gossip.rs +++ b/src/gossip.rs @@ -7,13 +7,12 @@ use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; -use std::time::Duration; use lightning_block_sync::gossip::GossipVerifier; use crate::chain::ChainSource; -use crate::config::RGS_SYNC_TIMEOUT_SECS; -use crate::logger::{log_trace, LdkLogger, Logger}; +use crate::config::{RGS_SNAPSHOT_MAX_SIZE, RGS_SYNC_TIMEOUT_SECS}; +use crate::logger::{log_error, log_trace, LdkLogger, Logger}; use crate::runtime::{Runtime, RuntimeSpawner}; use crate::types::{GossipSync, Graph, P2PGossipSync, RapidGossipSync}; use crate::Error; @@ -70,29 +69,18 @@ impl GossipSource { let query_timestamp = latest_sync_timestamp.load(Ordering::Acquire); let query_url = format!("{}/{}", server_url, query_timestamp); - let response = tokio::time::timeout( - Duration::from_secs(RGS_SYNC_TIMEOUT_SECS), - reqwest::get(query_url), - ) - .await - .map_err(|e| { - log_trace!(logger, "Retrieving RGS gossip update timed out: {}", e); + let query = bitreq::get(query_url) + .with_max_body_size(Some(RGS_SNAPSHOT_MAX_SIZE)) + .with_timeout(RGS_SYNC_TIMEOUT_SECS); + let response = query.send_async().await.map_err(|e| { + log_error!(logger, "Failed to retrieve RGS gossip update: {e}"); Error::GossipUpdateTimeout - })? - .map_err(|e| { - log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e); - Error::GossipUpdateFailed })?; - match response.error_for_status() { - Ok(res) => { - let update_data = res.bytes().await.map_err(|e| { - log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e); - Error::GossipUpdateFailed - })?; - + match response.status_code { + 200 => { let new_latest_sync_timestamp = - gossip_sync.update_network_graph(&update_data).map_err(|e| { + gossip_sync.update_network_graph(response.as_bytes()).map_err(|e| { log_trace!( logger, "Failed to update network graph with RGS data: {:?}", @@ -103,8 +91,8 @@ impl GossipSource { latest_sync_timestamp.store(new_latest_sync_timestamp, Ordering::Release); Ok(new_latest_sync_timestamp) }, - Err(e) => { - log_trace!(logger, "Failed to retrieve RGS gossip update: {}", e); + code => { + log_trace!(logger, "Failed to retrieve RGS gossip update: HTTP {}", code); Err(Error::GossipUpdateFailed) }, } diff --git a/src/scoring.rs b/src/scoring.rs index 2e0d226ff..3ed7b9d1e 100644 --- a/src/scoring.rs +++ b/src/scoring.rs @@ -1,12 +1,13 @@ use std::sync::{Arc, Mutex, RwLock}; -use std::time::{Duration, SystemTime}; +use std::time::SystemTime; use lightning::routing::scoring::ChannelLiquidities; use lightning::util::ser::Readable; use lightning::{log_error, log_info, log_trace}; use crate::config::{ - EXTERNAL_PATHFINDING_SCORES_SYNC_INTERVAL, EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS, + EXTERNAL_PATHFINDING_SCORES_MAX_SIZE, EXTERNAL_PATHFINDING_SCORES_SYNC_INTERVAL, + EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS, }; use crate::io::utils::write_external_pathfinding_scores_to_cache; use crate::logger::LdkLogger; @@ -53,34 +54,27 @@ async fn sync_external_scores( logger: &Logger, scorer: &Mutex, node_metrics: &RwLock, kv_store: Arc, url: &String, ) -> () { - let response = tokio::time::timeout( - Duration::from_secs(EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS), - reqwest::get(url), - ) - .await; + let request = bitreq::get(url) + .with_timeout(EXTERNAL_PATHFINDING_SCORES_SYNC_TIMEOUT_SECS) + .with_max_body_size(Some(EXTERNAL_PATHFINDING_SCORES_MAX_SIZE)); - let response = match response { - Ok(resp) => resp, - Err(e) => { - log_error!(logger, "Retrieving external scores timed out: {}", e); - return; - }, - }; - let response = match response { + let response = match request.send_async().await { Ok(resp) => resp, Err(e) => { log_error!(logger, "Failed to retrieve external scores update: {}", e); return; }, }; - let reader = match response.bytes().await { - Ok(bytes) => bytes, - Err(e) => { - log_error!(logger, "Failed to read external scores update: {}", e); - return; - }, - }; - match ChannelLiquidities::read(&mut &*reader) { + if response.status_code != 200 { + log_error!( + logger, + "Failed to retrieve external scores update: HTTP {}", + response.status_code + ); + return; + } + let mut reader = response.as_bytes(); + match ChannelLiquidities::read(&mut reader) { Ok(liquidities) => { if let Err(e) = write_external_pathfinding_scores_to_cache(&*kv_store, &liquidities, logger).await