From c7258426cd73b4b8e48de3b0994c144b4a657a20 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sun, 24 Dec 2023 17:12:18 +0000 Subject: [PATCH 1/9] Disable fuzzing-reachable debug assertion Because of txid conflicts, a txid can match both a local and remote commitment transaction when fuzzing. Thus, we disable this assertion when fuzzing. --- lightning/src/chain/channelmonitor.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index e65b54f5701..55b7b503fdb 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -3679,6 +3679,7 @@ impl ChannelMonitorImpl { claimable_outpoints.append(&mut new_outpoints); if new_outpoints.is_empty() { if let Some((mut new_outpoints, new_outputs)) = self.check_spend_holder_transaction(&tx, height, &block_hash, &logger) { + #[cfg(not(fuzzing))] debug_assert!(commitment_tx_to_counterparty_output.is_none(), "A commitment transaction matched as both a counterparty and local commitment tx?"); if !new_outputs.1.is_empty() { From 248e2f5be5b415d794fe10123b871ec45ad348bb Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 25 Dec 2023 00:54:45 +0000 Subject: [PATCH 2/9] Disable fuzzing-reachable debug assertions in `ChannelMonitor`s --- lightning/src/chain/channelmonitor.rs | 6 +++++- lightning/src/chain/onchaintx.rs | 4 +++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index 55b7b503fdb..fb5c18eae97 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -3172,7 +3172,11 @@ impl ChannelMonitorImpl { (htlc, htlc_source.as_ref().map(|htlc_source| htlc_source.as_ref())) ), logger); } else { - debug_assert!(false, "We should have per-commitment option for any recognized old commitment txn"); + // Our fuzzers aren't contrained by pesky things like valid signatures, so can + // spend our funding output with a transaction which doesn't match our past + // commitment transactions. Thus, we can only debug-assert here when not + // fuzzing. + debug_assert!(cfg!(fuzzing), "We should have per-commitment option for any recognized old commitment txn"); fail_unbroadcast_htlcs!(self, "revoked counterparty", commitment_txid, tx, height, block_hash, [].iter().map(|reference| *reference), logger); } diff --git a/lightning/src/chain/onchaintx.rs b/lightning/src/chain/onchaintx.rs index 59c98f05ebc..108ff009355 100644 --- a/lightning/src/chain/onchaintx.rs +++ b/lightning/src/chain/onchaintx.rs @@ -806,7 +806,9 @@ impl OnchainTxHandler claim_id }, }; - debug_assert!(self.pending_claim_requests.get(&claim_id).is_none()); + // Because fuzzing can cause hash collisions, we can end up with conflicting claim + // ids here, so we only assert when not fuzzing. + debug_assert!(cfg!(fuzzing) || self.pending_claim_requests.get(&claim_id).is_none()); for k in req.outpoints() { log_info!(logger, "Registering claiming request for {}:{}", k.txid, k.vout); self.claimable_outpoints.insert(k.clone(), (claim_id, conf_height)); From f62a96ea93e241ad00859e508df988f5b5ab6f92 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Sat, 23 Dec 2023 19:13:34 +0000 Subject: [PATCH 3/9] Use arbitrary config object in full_stack_target fuzzer As we've added more and more configuration parameters which change our behavior, we're increasingly missing coverage in the general `full_stack_target` fuzzer. Sadly, a reachable `unwrap` slipped in uncaught that should have been stopped by the `full_stack_target`. Here, we update the `full_stack_target` fuzzer to read a full config object to substantially increase coverage. --- fuzz/src/full_stack.rs | 47 ++++++++++---------------- lightning/src/util/config.rs | 64 ++++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 30 deletions(-) diff --git a/fuzz/src/full_stack.rs b/fuzz/src/full_stack.rs index 2df63cf5453..c7afd69bb3b 100644 --- a/fuzz/src/full_stack.rs +++ b/fuzz/src/full_stack.rs @@ -48,11 +48,11 @@ use lightning::onion_message::messenger::{Destination, MessageRouter, OnionMessa use lightning::routing::gossip::{P2PGossipSync, NetworkGraph}; use lightning::routing::utxo::UtxoLookup; use lightning::routing::router::{InFlightHtlcs, PaymentParameters, Route, RouteParameters, Router}; -use lightning::util::config::{UserConfig, MaxDustHTLCExposure}; +use lightning::util::config::UserConfig; use lightning::util::errors::APIError; use lightning::util::test_channel_signer::{TestChannelSigner, EnforcementState}; use lightning::util::logger::Logger; -use lightning::util::ser::{ReadableArgs, Writeable}; +use lightning::util::ser::{Readable, ReadableArgs, Writeable}; use crate::utils::test_logger; use crate::utils::test_persister::TestPersister; @@ -438,7 +438,17 @@ impl SignerProvider for KeyProvider { } #[inline] -pub fn do_test(data: &[u8], logger: &Arc) { +pub fn do_test(mut data: &[u8], logger: &Arc) { + if data.len() < 32 { return; } + + let our_network_key = match SecretKey::from_slice(&data[..32]) { + Ok(key) => key, + Err(e) => return, + }; + data = &data[32..]; + + let config: UserConfig = if let Ok(config) = Readable::read(&mut data) { config } else { return; }; + let input = Arc::new(InputData { data: data.to_vec(), read_pos: AtomicUsize::new(0), @@ -466,10 +476,6 @@ pub fn do_test(data: &[u8], logger: &Arc) { } } - let our_network_key = match SecretKey::from_slice(get_slice!(32)) { - Ok(key) => key, - Err(_) => return, - }; let inbound_payment_key = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 42]; @@ -483,10 +489,6 @@ pub fn do_test(data: &[u8], logger: &Arc) { counter: AtomicU64::new(0), signer_state: RefCell::new(HashMap::new()) }); - let mut config = UserConfig::default(); - config.channel_config.forwarding_fee_proportional_millionths = slice_to_be32(get_slice!(4)); - config.channel_config.max_dust_htlc_exposure = MaxDustHTLCExposure::FeeRateMultiplier(5_000_000 / 253); - config.channel_handshake_config.announced_channel = get_slice!(1)[0] != 0; let network = Network::Bitcoin; let best_block_timestamp = genesis_block(network).header.time; let params = ChainParameters { @@ -782,8 +784,7 @@ mod tests { // Writing new code generating transactions and see a new failure ? Don't forget to add input for the FuzzEstimator ! // 0100000000000000000000000000000000000000000000000000000000000000 - our network key - // 00000000 - fee_proportional_millionths - // 01 - announce_channels_publicly + // 0000000000900000000000000000640001000000000001ffff0000000000000000ffffffffffffffffffffffffffffffff0000000000000000ffffffffffffffff000000ffffffff00ffff1a000400010000020400000000040200000a08ffffffffffffffff0001000000 - config // // 00 - new outbound connection with id 0 // 030000000000000000000000000000000000000000000000000000000000000002 - peer's pubkey @@ -793,7 +794,7 @@ mod tests { // 030012 - inbound read from peer id 0 of len 18 // 0010 03000000000000000000000000000000 - message header indicating message length 16 // 030020 - inbound read from peer id 0 of len 32 - // 0010 00021aaa 0008aaaaaaaaaaaa9aaa 03000000000000000000000000000000 - init message (type 16) with static_remotekey required and other bits optional and mac + // 0010 00021aaa 0008aaa20aaa2a0a9aaa 03000000000000000000000000000000 - init message (type 16) with static_remotekey required, no channel_type/anchors/taproot, and other bits optional and mac // // 030012 - inbound read from peer id 0 of len 18 // 0147 03000000000000000000000000000000 - message header indicating message length 327 @@ -841,7 +842,7 @@ mod tests { // 030112 - inbound read from peer id 1 of len 18 // 0010 01000000000000000000000000000000 - message header indicating message length 16 // 030120 - inbound read from peer id 1 of len 32 - // 0010 00021aaa 0008aaaaaaaaaaaa9aaa 01000000000000000000000000000000 - init message (type 16) with static_remotekey required and other bits optional and mac + // 0010 00021aaa 0008aaa20aaa2a0a9aaa 01000000000000000000000000000000 - init message (type 16) with static_remotekey required, no channel_type/anchors/taproot, and other bits optional and mac // // 05 01 030200000000000000000000000000000000000000000000000000000000000000 00c350 0003e8 - create outbound channel to peer 1 for 50k sat // 00fd - One feerate requests (all returning min feerate) (gonna be ingested by FuzzEstimator) @@ -855,8 +856,6 @@ mod tests { // // 0a - create the funding transaction (client should send funding_created now) // - // 00fd00fd - Two feerate requests (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // 030112 - inbound read from peer id 1 of len 18 // 0062 01000000000000000000000000000000 - message header indicating message length 98 // 030172 - inbound read from peer id 1 of len 114 @@ -885,8 +884,6 @@ mod tests { // 0300c1 - inbound read from peer id 0 of len 193 // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac // - // 00fd - One feerate request (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // 030012 - inbound read from peer id 0 of len 18 // 0064 03000000000000000000000000000000 - message header indicating message length 100 // 030074 - inbound read from peer id 0 of len 116 @@ -901,8 +898,6 @@ mod tests { // 07 - process the now-pending HTLC forward // - client now sends id 1 update_add_htlc and commitment_signed (CHECK 7: UpdateHTLCs event for node 03020000 with 1 HTLCs for channel 3f000000) // - // 00fd00fd - Two feerate requests (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // - we respond with commitment_signed then revoke_and_ack (a weird, but valid, order) // 030112 - inbound read from peer id 1 of len 18 // 0064 01000000000000000000000000000000 - message header indicating message length 100 @@ -946,8 +941,6 @@ mod tests { // 0300c1 - inbound read from peer id 0 of len 193 // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff ab00000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac // - // 00fd - One feerate request (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // - now respond to the update_fulfill_htlc+commitment_signed messages the client sent to peer 0 // 030012 - inbound read from peer id 0 of len 18 // 0063 03000000000000000000000000000000 - message header indicating message length 99 @@ -969,8 +962,6 @@ mod tests { // - client now sends id 1 update_add_htlc and commitment_signed (CHECK 7 duplicate) // - we respond with revoke_and_ack, then commitment_signed, then update_fail_htlc // - // 00fd00fd - Two feerate requests (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // 030112 - inbound read from peer id 1 of len 18 // 0064 01000000000000000000000000000000 - message header indicating message length 100 // 030174 - inbound read from peer id 1 of len 116 @@ -1026,8 +1017,6 @@ mod tests { // 0300c1 - inbound read from peer id 0 of len 193 // ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 5300000000000000000000000000000000000000000000000000000000000000 03000000000000000000000000000000 - end of update_add_htlc from 0 to 1 via client and mac // - // 00fd - One feerate request (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // 030012 - inbound read from peer id 0 of len 18 // 00a4 03000000000000000000000000000000 - message header indicating message length 164 // 0300b4 - inbound read from peer id 0 of len 180 @@ -1042,8 +1031,6 @@ mod tests { // 07 - process the now-pending HTLC forward // - client now sends id 1 update_add_htlc and commitment_signed (CHECK 7 duplicate) // - // 00fd00fd - Two feerate requests (calculating max dust exposure) (all returning min feerate) (gonna be ingested by FuzzEstimator) - // // 0c007d - connect a block with one transaction of len 125 // 02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c0000000000000160014280000000000000000000000000000000000000005000020 - the commitment transaction for channel 3f00000000000000000000000000000000000000000000000000000000000000 // @@ -1059,7 +1046,7 @@ mod tests { // - client now fails the HTLC backwards as it was unable to extract the payment preimage (CHECK 9 duplicate and CHECK 10) let logger = Arc::new(TrackingLogger { lines: Mutex::new(HashMap::new()) }); - super::do_test(&>::from_hex("01000000000000000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012001003000000000000000000000000000000030020001000021aaa0008aaaaaaaaaaaa9aaa030000000000000000000000000000000300120147030000000000000000000000000000000300fe00206fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030059030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010000010210000300000000000000000000000000000000fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112001001000000000000000000000000000000030120001000021aaa0008aaaaaaaaaaaa9aaa01000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120112010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f000050300000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000002000300000000000000000000000000000000000000000000000000000000000003000300000000000000000000000000000000000000000000000000000000000004000300000000000000000000000000000000000000000000000000000000000005000266000000000000000000000000000003012300000000000000000000000000000000000000010000000000000000000000000000000a00fd00fd03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000fd03001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000700fd00fd03011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000fd03001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000700fd00fd03011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005551202030927c00401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff53000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000000fd03001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000700fd00fd0c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc)); + super::do_test(&>::from_hex("01000000000000000000000000000000000000000000000000000000000000000000000000900000000000000000640001000000000001ffff0000000000000000ffffffffffffffffffffffffffffffff0000000000000000ffffffffffffffff000000ffffffff00ffff1a000400010000020400000000040200000a08ffffffffffffffff0001000000000300000000000000000000000000000000000000000000000000000000000000020300320003000000000000000000000000000000000000000000000000000000000000000203000000000000000000000000000000030012001003000000000000000000000000000000030020001000021aaa0008aaa20aaa2a0a9aaa030000000000000000000000000000000300120147030000000000000000000000000000000300fe00206fe28c0ab6f1b372c1a6a246ae63f74f931e8365e15a089c68d6190000000000ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679000000000000c35000000000000000000000000000000162ffffffffffffffff00000000000002220000000000000000000000fd000601e3030000000000000000000000000000000000000000000000000000000000000001030000000000000000000000000000000000000000000000000000000000000002030000000000000000000000000000000000000000000000000000000000000003030000000000000000000000000000000000000000000000000000000000000004030059030000000000000000000000000000000000000000000000000000000000000005020900000000000000000000000000000000000000000000000000000000000000010000010210000300000000000000000000000000000000fd0300120084030000000000000000000000000000000300940022ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb1819096793d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000210100000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000c005e020000000100000000000000000000000000000000000000000000000000000000000000000000000000ffffffff0150c3000000000000220020ae00000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c00000c000003001200430300000000000000000000000000000003005300243d0000000000000000000000000000000000000000000000000000000000000002080000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000010301320003000000000000000000000000000000000000000000000000000000000000000703000000000000000000000000000000030142000302000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003000000000000000000000000000000030112001001000000000000000000000000000000030120001000021aaa0008aaa20aaa2a0a9aaa01000000000000000000000000000000050103020000000000000000000000000000000000000000000000000000000000000000c3500003e800fd0301120112010000000000000000000000000000000301ff00210000000000000000000000000000000000000000000000000000000000000e05000000000000016200000000004c4b4000000000000003e800000000000003e80000000203f000050300000000000000000000000000000000000000000000000000000000000001000300000000000000000000000000000000000000000000000000000000000002000300000000000000000000000000000000000000000000000000000000000003000300000000000000000000000000000000000000000000000000000000000004000300000000000000000000000000000000000000000000000000000000000005000266000000000000000000000000000003012300000000000000000000000000000000000000010000000000000000000000000000000a03011200620100000000000000000000000000000003017200233a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007c0001000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000b03011200430100000000000000000000000000000003015300243a000000000000000000000000000000000000000000000000000000000000000267000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000030010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000900000000000000000000000000000000000000000000000000000000000000020b00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000006a000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000660000000000000000000000000000000000000000000000000000000000000002640000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112004a0100000000000000000000000000000003015a00823a000000000000000000000000000000000000000000000000000000000000000000000000000000ff008888888888888888888888888888888888888888888888888888888888880100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000010000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a0000000000000000000000000000000000000000000000000000000000000067000000000000000000000000000000000000000000000000000000000000000265000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d0000000000000000000000000000000000000000000000000000000000000000000000000000010000000000003e80ff00000000000000000000000000000000000000000000000000000000000000000003f00003000000000000000000000000000000000000000000000000000000000000055511020203e80401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffab000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000000000000000000000000000000020a000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000c3010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000b00000000000000000000000000000000000000000000000000000000000000020d00000000000000000000000000000000000000000000000000000000000000030000000000000000000000000000000703011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a00000000000000000000000000000000000000000000000000000000000000640000000000000000000000000000000000000000000000000000000000000002700000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000030112002c0100000000000000000000000000000003013c00833a00000000000000000000000000000000000000000000000000000000000000000000000000000100000100000000000000000000000000000003011200640100000000000000000000000000000003017400843a000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000039000100000000000000000000000000000000000000000000000000000000000000000100000000000000000000000000000003011200630100000000000000000000000000000003017300853a000000000000000000000000000000000000000000000000000000000000006500000000000000000000000000000000000000000000000000000000000000027100000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000703001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000020c000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200640300000000000000000000000000000003007400843d000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000032010000000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001205ac030000000000000000000000000000000300ff00803d00000000000000000000000000000000000000000000000000000000000000000000000000000200000000000b0838ff00000000000000000000000000000000000000000000000000000000000000000003f0000300000000000000000000000000000000000000000000000000000000000005551202030927c00401a0060800000e00000100000a00000000000000000000000000000000000000000000000000000000000000ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff0300c1ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff53000000000000000000000000000000000000000000000000000000000000000300000000000000000000000000000003001200a4030000000000000000000000000000000300b400843d00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007501000000000000000000000000000000000000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000006705000000000000000000000000000000000000000000000000000000000000060300000000000000000000000000000003001200630300000000000000000000000000000003007300853d000000000000000000000000000000000000000000000000000000000000000d00000000000000000000000000000000000000000000000000000000000000020f0000000000000000000000000000000000000000000000000000000000000003000000000000000000000000000000070c007d02000000013a000000000000000000000000000000000000000000000000000000000000000000000000000000800258020000000000002200204b0000000000000000000000000000000000000000000000000000000000000014c00000000000001600142800000000000000000000000000000000000000050000200c005e0200000001730000000000000000000000000000000000000000000000000000000000000000000000000000000001a701000000000000220020b200000000000000000000000000000000000000000000000000000000000000000000000c00000c00000c00000c00000c000007").unwrap(), &(Arc::clone(&logger) as Arc)); let log_entries = logger.lines.lock().unwrap(); assert_eq!(log_entries.get(&("lightning::ln::peer_handler".to_string(), "Handling SendAcceptChannel event in peer_handler for node 030000000000000000000000000000000000000000000000000000000000000002 for channel ff4f00f805273c1b203bb5ebf8436bfde57b3be8c2f5e95d9491dbb181909679".to_string())), Some(&1)); // 1 diff --git a/lightning/src/util/config.rs b/lightning/src/util/config.rs index 13284113d2c..2c8f03b93c8 100644 --- a/lightning/src/util/config.rs +++ b/lightning/src/util/config.rs @@ -13,6 +13,9 @@ use crate::ln::channel::MAX_FUNDING_SATOSHIS_NO_WUMBO; use crate::ln::channelmanager::{BREAKDOWN_TIMEOUT, MAX_LOCAL_BREAKDOWN_TIMEOUT}; +#[cfg(fuzzing)] +use crate::util::ser::Readable; + /// Configuration we set when applicable. /// /// Default::default() provides sane defaults. @@ -210,6 +213,27 @@ impl Default for ChannelHandshakeConfig { } } +// When fuzzing, we want to allow the fuzzer to pick any configuration parameters. Thus, we +// implement Readable here in a naive way (which is a bit easier for the fuzzer to handle). We +// don't really want to ever expose this to users (if we did we'd want to use TLVs). +#[cfg(fuzzing)] +impl Readable for ChannelHandshakeConfig { + fn read(reader: &mut R) -> Result { + Ok(Self { + minimum_depth: Readable::read(reader)?, + our_to_self_delay: Readable::read(reader)?, + our_htlc_minimum_msat: Readable::read(reader)?, + max_inbound_htlc_value_in_flight_percent_of_channel: Readable::read(reader)?, + negotiate_scid_privacy: Readable::read(reader)?, + announced_channel: Readable::read(reader)?, + commit_upfront_shutdown_pubkey: Readable::read(reader)?, + their_channel_reserve_proportional_millionths: Readable::read(reader)?, + negotiate_anchors_zero_fee_htlc_tx: Readable::read(reader)?, + our_max_accepted_htlcs: Readable::read(reader)?, + }) + } +} + /// Optional channel limits which are applied during channel creation. /// /// These limits are only applied to our counterparty's limits, not our own. @@ -315,6 +339,27 @@ impl Default for ChannelHandshakeLimits { } } +// When fuzzing, we want to allow the fuzzer to pick any configuration parameters. Thus, we +// implement Readable here in a naive way (which is a bit easier for the fuzzer to handle). We +// don't really want to ever expose this to users (if we did we'd want to use TLVs). +#[cfg(fuzzing)] +impl Readable for ChannelHandshakeLimits { + fn read(reader: &mut R) -> Result { + Ok(Self { + min_funding_satoshis: Readable::read(reader)?, + max_funding_satoshis: Readable::read(reader)?, + max_htlc_minimum_msat: Readable::read(reader)?, + min_max_htlc_value_in_flight_msat: Readable::read(reader)?, + max_channel_reserve_satoshis: Readable::read(reader)?, + min_max_accepted_htlcs: Readable::read(reader)?, + trust_own_funding_0conf: Readable::read(reader)?, + max_minimum_depth: Readable::read(reader)?, + force_announced_channel_preference: Readable::read(reader)?, + their_to_self_delay: Readable::read(reader)?, + }) + } +} + /// Options for how to set the max dust HTLC exposure allowed on a channel. See /// [`ChannelConfig::max_dust_htlc_exposure`] for details. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -781,3 +826,22 @@ impl Default for UserConfig { } } } + +// When fuzzing, we want to allow the fuzzer to pick any configuration parameters. Thus, we +// implement Readable here in a naive way (which is a bit easier for the fuzzer to handle). We +// don't really want to ever expose this to users (if we did we'd want to use TLVs). +#[cfg(fuzzing)] +impl Readable for UserConfig { + fn read(reader: &mut R) -> Result { + Ok(Self { + channel_handshake_config: Readable::read(reader)?, + channel_handshake_limits: Readable::read(reader)?, + channel_config: Readable::read(reader)?, + accept_forwards_to_priv_channels: Readable::read(reader)?, + accept_inbound_channels: Readable::read(reader)?, + manually_accept_inbound_channels: Readable::read(reader)?, + accept_intercept_htlcs: Readable::read(reader)?, + accept_mpp_keysend: Readable::read(reader)?, + }) + } +} From 76ed917ddc97d431359d745df7ecbd5ce9237a31 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 18 Jan 2024 20:34:56 +0000 Subject: [PATCH 4/9] Update fuzz CI job to use our MSRV rather than 1.58 --- .github/workflows/build.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ad5e1fc517b..cff105470d2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -176,7 +176,7 @@ jobs: fuzz: runs-on: ubuntu-latest env: - TOOLCHAIN: 1.58 + TOOLCHAIN: 1.63 steps: - name: Checkout source code uses: actions/checkout@v3 @@ -188,6 +188,10 @@ jobs: run: | sudo apt-get update sudo apt-get -y install build-essential binutils-dev libunwind-dev + - name: Pin the regex dependency + run: | + cd fuzz && cargo update -p regex --precise "1.9.6" --verbose && cd .. + cd lightning-invoice/fuzz && cargo update -p regex --precise "1.9.6" --verbose - name: Sanity check fuzz targets on Rust ${{ env.TOOLCHAIN }} run: cd fuzz && RUSTFLAGS="--cfg=fuzzing" cargo test --verbose --color always - name: Run fuzzers From 3e0d55bd2b96ea10ecc744d4cbd9bbfc1ad4ef50 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Mon, 1 Jan 2024 21:53:36 +0000 Subject: [PATCH 5/9] Use utility methods to construct `HashMap`s and `HashSet`s In the next commit we'll bump the `hashbrown` version, which no longer randomizes its hasher by default. Thus, we'll need to call a different constructor in no-std builds from std builds. Here we do a quick prefactor to use wrappers for constructors instead of calling the tables directly to make the version bump changeset smaller. --- lightning/src/chain/chainmonitor.rs | 2 +- lightning/src/chain/channelmonitor.rs | 30 +++++------ lightning/src/chain/onchaintx.rs | 12 ++--- lightning/src/events/bump_transaction.rs | 2 +- lightning/src/lib.rs | 34 +++++++++++- lightning/src/ln/channel.rs | 6 +-- lightning/src/ln/channelmanager.rs | 66 +++++++++++------------ lightning/src/ln/functional_test_utils.rs | 10 ++-- lightning/src/ln/functional_tests.rs | 8 +-- lightning/src/ln/monitor_tests.rs | 2 +- lightning/src/ln/outbound_payment.rs | 10 ++-- lightning/src/ln/peer_handler.rs | 6 +-- lightning/src/onion_message/messenger.rs | 4 +- lightning/src/routing/gossip.rs | 8 +-- lightning/src/routing/router.rs | 10 ++-- lightning/src/routing/scoring.rs | 10 ++-- lightning/src/routing/utxo.rs | 2 +- lightning/src/sign/mod.rs | 2 +- lightning/src/sync/debug_sync.rs | 8 +-- lightning/src/util/indexed_map.rs | 16 ++---- lightning/src/util/ser.rs | 4 +- lightning/src/util/test_utils.rs | 36 ++++++------- 22 files changed, 156 insertions(+), 132 deletions(-) diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index b71f10f58d4..4adafd2db8b 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -420,7 +420,7 @@ where C::Target: chain::Filter, /// transactions relevant to the watched channels. pub fn new(chain_source: Option, broadcaster: T, logger: L, feeest: F, persister: P) -> Self { Self { - monitors: RwLock::new(HashMap::new()), + monitors: RwLock::new(new_hash_map()), sync_persistence_id: AtomicCounter::new(), chain_source, broadcaster, diff --git a/lightning/src/chain/channelmonitor.rs b/lightning/src/chain/channelmonitor.rs index fb5c18eae97..a9250c0a787 100644 --- a/lightning/src/chain/channelmonitor.rs +++ b/lightning/src/chain/channelmonitor.rs @@ -1235,7 +1235,7 @@ impl ChannelMonitor { channel_parameters.clone(), initial_holder_commitment_tx, secp_ctx ); - let mut outputs_to_watch = HashMap::new(); + let mut outputs_to_watch = new_hash_map(); outputs_to_watch.insert(funding_info.0.txid, vec![(funding_info.0.index as u32, funding_info.1.clone())]); Self::from_impl(ChannelMonitorImpl { @@ -1262,17 +1262,17 @@ impl ChannelMonitor { on_holder_tx_csv: counterparty_channel_parameters.selected_contest_delay, commitment_secrets: CounterpartyCommitmentSecrets::new(), - counterparty_claimable_outpoints: HashMap::new(), - counterparty_commitment_txn_on_chain: HashMap::new(), - counterparty_hash_commitment_number: HashMap::new(), - counterparty_fulfilled_htlcs: HashMap::new(), + counterparty_claimable_outpoints: new_hash_map(), + counterparty_commitment_txn_on_chain: new_hash_map(), + counterparty_hash_commitment_number: new_hash_map(), + counterparty_fulfilled_htlcs: new_hash_map(), prev_holder_signed_commitment_tx: None, current_holder_commitment_tx: holder_commitment_tx, current_counterparty_commitment_number: 1 << 48, current_holder_commitment_number, - payment_preimages: HashMap::new(), + payment_preimages: new_hash_map(), pending_monitor_events: Vec::new(), pending_events: Vec::new(), is_processing_pending_events: false, @@ -2174,7 +2174,7 @@ impl ChannelMonitor { /// HTLCs which were resolved on-chain (i.e. where the final HTLC resolution was done by an /// event from this `ChannelMonitor`). pub(crate) fn get_all_current_outbound_htlcs(&self) -> HashMap)> { - let mut res = HashMap::new(); + let mut res = new_hash_map(); // Just examine the available counterparty commitment transactions. See docs on // `fail_unbroadcast_htlcs`, below, for justification. let us = self.inner.lock().unwrap(); @@ -2226,7 +2226,7 @@ impl ChannelMonitor { return self.get_all_current_outbound_htlcs(); } - let mut res = HashMap::new(); + let mut res = new_hash_map(); macro_rules! walk_htlcs { ($holder_commitment: expr, $htlc_iter: expr) => { for (htlc, source) in $htlc_iter { @@ -3940,7 +3940,7 @@ impl ChannelMonitorImpl { /// Filters a block's `txdata` for transactions spending watched outputs or for any child /// transactions thereof. fn filter_block<'a>(&self, txdata: &TransactionData<'a>) -> Vec<&'a Transaction> { - let mut matched_txn = HashSet::new(); + let mut matched_txn = new_hash_set(); txdata.iter().filter(|&&(_, tx)| { let mut matches = self.spends_watched_output(tx); for input in tx.input.iter() { @@ -4455,7 +4455,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP } let counterparty_claimable_outpoints_len: u64 = Readable::read(reader)?; - let mut counterparty_claimable_outpoints = HashMap::with_capacity(cmp::min(counterparty_claimable_outpoints_len as usize, MAX_ALLOC_SIZE / 64)); + let mut counterparty_claimable_outpoints = hash_map_with_capacity(cmp::min(counterparty_claimable_outpoints_len as usize, MAX_ALLOC_SIZE / 64)); for _ in 0..counterparty_claimable_outpoints_len { let txid: Txid = Readable::read(reader)?; let htlcs_count: u64 = Readable::read(reader)?; @@ -4469,7 +4469,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP } let counterparty_commitment_txn_on_chain_len: u64 = Readable::read(reader)?; - let mut counterparty_commitment_txn_on_chain = HashMap::with_capacity(cmp::min(counterparty_commitment_txn_on_chain_len as usize, MAX_ALLOC_SIZE / 32)); + let mut counterparty_commitment_txn_on_chain = hash_map_with_capacity(cmp::min(counterparty_commitment_txn_on_chain_len as usize, MAX_ALLOC_SIZE / 32)); for _ in 0..counterparty_commitment_txn_on_chain_len { let txid: Txid = Readable::read(reader)?; let commitment_number = ::read(reader)?.0; @@ -4479,7 +4479,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP } let counterparty_hash_commitment_number_len: u64 = Readable::read(reader)?; - let mut counterparty_hash_commitment_number = HashMap::with_capacity(cmp::min(counterparty_hash_commitment_number_len as usize, MAX_ALLOC_SIZE / 32)); + let mut counterparty_hash_commitment_number = hash_map_with_capacity(cmp::min(counterparty_hash_commitment_number_len as usize, MAX_ALLOC_SIZE / 32)); for _ in 0..counterparty_hash_commitment_number_len { let payment_hash: PaymentHash = Readable::read(reader)?; let commitment_number = ::read(reader)?.0; @@ -4502,7 +4502,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP let current_holder_commitment_number = ::read(reader)?.0; let payment_preimages_len: u64 = Readable::read(reader)?; - let mut payment_preimages = HashMap::with_capacity(cmp::min(payment_preimages_len as usize, MAX_ALLOC_SIZE / 32)); + let mut payment_preimages = hash_map_with_capacity(cmp::min(payment_preimages_len as usize, MAX_ALLOC_SIZE / 32)); for _ in 0..payment_preimages_len { let preimage: PaymentPreimage = Readable::read(reader)?; let hash = PaymentHash(Sha256::hash(&preimage.0[..]).to_byte_array()); @@ -4542,7 +4542,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP } let outputs_to_watch_len: u64 = Readable::read(reader)?; - let mut outputs_to_watch = HashMap::with_capacity(cmp::min(outputs_to_watch_len as usize, MAX_ALLOC_SIZE / (mem::size_of::() + mem::size_of::() + mem::size_of::>()))); + let mut outputs_to_watch = hash_map_with_capacity(cmp::min(outputs_to_watch_len as usize, MAX_ALLOC_SIZE / (mem::size_of::() + mem::size_of::() + mem::size_of::>()))); for _ in 0..outputs_to_watch_len { let txid = Readable::read(reader)?; let outputs_len: u64 = Readable::read(reader)?; @@ -4584,7 +4584,7 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP let mut counterparty_node_id = None; let mut confirmed_commitment_tx_counterparty_output = None; let mut spendable_txids_confirmed = Some(Vec::new()); - let mut counterparty_fulfilled_htlcs = Some(HashMap::new()); + let mut counterparty_fulfilled_htlcs = Some(new_hash_map()); let mut initial_counterparty_commitment_info = None; let mut channel_id = None; read_tlv_fields!(reader, { diff --git a/lightning/src/chain/onchaintx.rs b/lightning/src/chain/onchaintx.rs index 108ff009355..5fe4bd2c1d2 100644 --- a/lightning/src/chain/onchaintx.rs +++ b/lightning/src/chain/onchaintx.rs @@ -374,13 +374,13 @@ impl<'a, 'b, ES: EntropySource, SP: SignerProvider> ReadableArgs<(&'a ES, &'b SP signer.provide_channel_parameters(&channel_parameters); let pending_claim_requests_len: u64 = Readable::read(reader)?; - let mut pending_claim_requests = HashMap::with_capacity(cmp::min(pending_claim_requests_len as usize, MAX_ALLOC_SIZE / 128)); + let mut pending_claim_requests = hash_map_with_capacity(cmp::min(pending_claim_requests_len as usize, MAX_ALLOC_SIZE / 128)); for _ in 0..pending_claim_requests_len { pending_claim_requests.insert(Readable::read(reader)?, Readable::read(reader)?); } let claimable_outpoints_len: u64 = Readable::read(reader)?; - let mut claimable_outpoints = HashMap::with_capacity(cmp::min(pending_claim_requests_len as usize, MAX_ALLOC_SIZE / 128)); + let mut claimable_outpoints = hash_map_with_capacity(cmp::min(pending_claim_requests_len as usize, MAX_ALLOC_SIZE / 128)); for _ in 0..claimable_outpoints_len { let outpoint = Readable::read(reader)?; let ancestor_claim_txid = Readable::read(reader)?; @@ -445,8 +445,8 @@ impl OnchainTxHandler prev_holder_commitment: None, signer, channel_transaction_parameters: channel_parameters, - pending_claim_requests: HashMap::new(), - claimable_outpoints: HashMap::new(), + pending_claim_requests: new_hash_map(), + claimable_outpoints: new_hash_map(), locktimed_packages: BTreeMap::new(), onchain_events_awaiting_threshold_conf: Vec::new(), pending_claim_events: Vec::new(), @@ -834,7 +834,7 @@ impl OnchainTxHandler F::Target: FeeEstimator, { log_debug!(logger, "Updating claims view at height {} with {} matched transactions in block {}", cur_height, txn_matched.len(), conf_height); - let mut bump_candidates = HashMap::new(); + let mut bump_candidates = new_hash_map(); for tx in txn_matched { // Scan all input to verify is one of the outpoint spent is of interest for us let mut claimed_outputs_material = Vec::new(); @@ -1020,7 +1020,7 @@ impl OnchainTxHandler where B::Target: BroadcasterInterface, F::Target: FeeEstimator, { - let mut bump_candidates = HashMap::new(); + let mut bump_candidates = new_hash_map(); let onchain_events_awaiting_threshold_conf = self.onchain_events_awaiting_threshold_conf.drain(..).collect::>(); for entry in onchain_events_awaiting_threshold_conf { diff --git a/lightning/src/events/bump_transaction.rs b/lightning/src/events/bump_transaction.rs index 1b019ba349e..0676fef14b5 100644 --- a/lightning/src/events/bump_transaction.rs +++ b/lightning/src/events/bump_transaction.rs @@ -391,7 +391,7 @@ where /// Returns a new instance backed by the given [`WalletSource`] that serves as an implementation /// of [`CoinSelectionSource`]. pub fn new(source: W, logger: L) -> Self { - Self { source, logger, locked_utxos: Mutex::new(HashMap::new()) } + Self { source, logger, locked_utxos: Mutex::new(new_hash_map()) } } /// Performs coin selection on the set of UTXOs obtained from diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 160f632415b..33f596fd8b9 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -169,10 +169,40 @@ mod prelude { extern crate hashbrown; pub use alloc::{vec, vec::Vec, string::String, collections::VecDeque, boxed::Box}; + + #[cfg(not(feature = "hashbrown"))] + mod std_hashtables { + pub(crate) use std::collections::{HashMap, HashSet, hash_map}; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + std::collections::hash_map::OccupiedEntry<'a, K, V>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + std::collections::hash_map::VacantEntry<'a, K, V>; + } #[cfg(not(feature = "hashbrown"))] - pub use std::collections::{HashMap, HashSet, hash_map}; + pub(crate) use std_hashtables::*; + + #[cfg(feature = "hashbrown")] + mod hashbrown_tables { + pub(crate) use hashbrown::{HashMap, HashSet, hash_map}; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + hashbrown::hash_map::OccupiedEntry<'a, K, V, hash_map::DefaultHashBuilder>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + hashbrown::hash_map::VacantEntry<'a, K, V, hash_map::DefaultHashBuilder>; + } #[cfg(feature = "hashbrown")] - pub use self::hashbrown::{HashMap, HashSet, hash_map}; + pub(crate) use hashbrown_tables::*; + + pub(crate) fn new_hash_map() -> HashMap { HashMap::new() } + pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { + HashMap::with_capacity(cap) + } + + pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } + pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { + HashSet::with_capacity(cap) + } pub use alloc::borrow::ToOwned; pub use alloc::string::ToString; diff --git a/lightning/src/ln/channel.rs b/lightning/src/ln/channel.rs index e56fa978953..d2034ebc871 100644 --- a/lightning/src/ln/channel.rs +++ b/lightning/src/ln/channel.rs @@ -6431,7 +6431,7 @@ impl OutboundV1Channel where SP::Target: SignerProvider { channel_ready_event_emitted: false, #[cfg(any(test, fuzzing))] - historical_inbound_htlc_fulfills: HashSet::new(), + historical_inbound_htlc_fulfills: new_hash_set(), channel_type, channel_keys_id, @@ -7232,7 +7232,7 @@ impl InboundV1Channel where SP::Target: SignerProvider { channel_ready_event_emitted: false, #[cfg(any(test, fuzzing))] - historical_inbound_htlc_fulfills: HashSet::new(), + historical_inbound_htlc_fulfills: new_hash_set(), channel_type, channel_keys_id, @@ -8048,7 +8048,7 @@ impl<'a, 'b, 'c, ES: Deref, SP: Deref> ReadableArgs<(&'a ES, &'b SP, u32, &'c Ch let channel_update_status = Readable::read(reader)?; #[cfg(any(test, fuzzing))] - let mut historical_inbound_htlc_fulfills = HashSet::new(); + let mut historical_inbound_htlc_fulfills = new_hash_set(); #[cfg(any(test, fuzzing))] { let htlc_fulfills_len: u64 = Readable::read(reader)?; diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 976a7b567ae..a9ae6ee3cec 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -2432,14 +2432,14 @@ where best_block: RwLock::new(params.best_block), - outbound_scid_aliases: Mutex::new(HashSet::new()), - pending_inbound_payments: Mutex::new(HashMap::new()), + outbound_scid_aliases: Mutex::new(new_hash_set()), + pending_inbound_payments: Mutex::new(new_hash_map()), pending_outbound_payments: OutboundPayments::new(), - forward_htlcs: Mutex::new(HashMap::new()), - claimable_payments: Mutex::new(ClaimablePayments { claimable_payments: HashMap::new(), pending_claiming_payments: HashMap::new() }), - pending_intercepted_htlcs: Mutex::new(HashMap::new()), - outpoint_to_peer: Mutex::new(HashMap::new()), - short_to_chan_info: FairRwLock::new(HashMap::new()), + forward_htlcs: Mutex::new(new_hash_map()), + claimable_payments: Mutex::new(ClaimablePayments { claimable_payments: new_hash_map(), pending_claiming_payments: new_hash_map() }), + pending_intercepted_htlcs: Mutex::new(new_hash_map()), + outpoint_to_peer: Mutex::new(new_hash_map()), + short_to_chan_info: FairRwLock::new(new_hash_map()), our_network_pubkey: node_signer.get_node_id(Recipient::Node).unwrap(), secp_ctx, @@ -2451,7 +2451,7 @@ where highest_seen_timestamp: AtomicUsize::new(current_timestamp as usize), - per_peer_state: FairRwLock::new(HashMap::new()), + per_peer_state: FairRwLock::new(new_hash_map()), pending_events: Mutex::new(VecDeque::new()), pending_events_processor: AtomicBool::new(false), @@ -3685,7 +3685,7 @@ where ProbeSendFailure::RouteNotFound })?; - let mut used_liquidity_map = HashMap::with_capacity(first_hops.len()); + let mut used_liquidity_map = hash_map_with_capacity(first_hops.len()); let mut res = Vec::new(); @@ -4242,7 +4242,7 @@ where let mut failed_forwards = Vec::new(); let mut phantom_receives: Vec<(u64, OutPoint, ChannelId, u128, Vec<(PendingHTLCInfo, u64)>)> = Vec::new(); { - let mut forward_htlcs = HashMap::new(); + let mut forward_htlcs = new_hash_map(); mem::swap(&mut forward_htlcs, &mut self.forward_htlcs.lock().unwrap()); for (short_chan_id, mut pending_forwards) in forward_htlcs { @@ -9015,8 +9015,8 @@ where return NotifyOption::SkipPersistNoEvents; } e.insert(Mutex::new(PeerState { - channel_by_id: HashMap::new(), - inbound_channel_request_by_id: HashMap::new(), + channel_by_id: new_hash_map(), + inbound_channel_request_by_id: new_hash_map(), latest_features: init_msg.features.clone(), pending_msg_events: Vec::new(), in_flight_monitor_updates: BTreeMap::new(), @@ -10067,7 +10067,7 @@ where } // Encode without retry info for 0.0.101 compatibility. - let mut pending_outbound_payments_no_retry: HashMap> = HashMap::new(); + let mut pending_outbound_payments_no_retry: HashMap> = new_hash_map(); for (id, outbound) in pending_outbound_payments.iter() { match outbound { PendingOutboundPayment::Legacy { session_privs } | @@ -10095,7 +10095,7 @@ where for ((counterparty_id, _), peer_state) in per_peer_state.iter().zip(peer_states.iter()) { for (funding_outpoint, updates) in peer_state.in_flight_monitor_updates.iter() { if !updates.is_empty() { - if in_flight_monitor_updates.is_none() { in_flight_monitor_updates = Some(HashMap::new()); } + if in_flight_monitor_updates.is_none() { in_flight_monitor_updates = Some(new_hash_map()); } in_flight_monitor_updates.as_mut().unwrap().insert((counterparty_id, funding_outpoint), updates); } } @@ -10331,13 +10331,13 @@ where let mut failed_htlcs = Vec::new(); let channel_count: u64 = Readable::read(reader)?; - let mut funding_txo_set = HashSet::with_capacity(cmp::min(channel_count as usize, 128)); - let mut funded_peer_channels: HashMap>> = HashMap::with_capacity(cmp::min(channel_count as usize, 128)); - let mut outpoint_to_peer = HashMap::with_capacity(cmp::min(channel_count as usize, 128)); - let mut short_to_chan_info = HashMap::with_capacity(cmp::min(channel_count as usize, 128)); + let mut funding_txo_set = hash_set_with_capacity(cmp::min(channel_count as usize, 128)); + let mut funded_peer_channels: HashMap>> = hash_map_with_capacity(cmp::min(channel_count as usize, 128)); + let mut outpoint_to_peer = hash_map_with_capacity(cmp::min(channel_count as usize, 128)); + let mut short_to_chan_info = hash_map_with_capacity(cmp::min(channel_count as usize, 128)); let mut channel_closures = VecDeque::new(); let mut close_background_events = Vec::new(); - let mut funding_txo_to_channel_id = HashMap::with_capacity(channel_count as usize); + let mut funding_txo_to_channel_id = hash_map_with_capacity(channel_count as usize); for _ in 0..channel_count { let mut channel: Channel = Channel::read(reader, ( &args.entropy_source, &args.signer_provider, best_block_height, &provided_channel_type_features(&args.default_config) @@ -10423,7 +10423,7 @@ where by_id_map.insert(channel.context.channel_id(), ChannelPhase::Funded(channel)); }, hash_map::Entry::Vacant(entry) => { - let mut by_id_map = HashMap::new(); + let mut by_id_map = new_hash_map(); by_id_map.insert(channel.context.channel_id(), ChannelPhase::Funded(channel)); entry.insert(by_id_map); } @@ -10470,7 +10470,7 @@ where const MAX_ALLOC_SIZE: usize = 1024 * 64; let forward_htlcs_count: u64 = Readable::read(reader)?; - let mut forward_htlcs = HashMap::with_capacity(cmp::min(forward_htlcs_count as usize, 128)); + let mut forward_htlcs = hash_map_with_capacity(cmp::min(forward_htlcs_count as usize, 128)); for _ in 0..forward_htlcs_count { let short_channel_id = Readable::read(reader)?; let pending_forwards_count: u64 = Readable::read(reader)?; @@ -10496,7 +10496,7 @@ where let peer_state_from_chans = |channel_by_id| { PeerState { channel_by_id, - inbound_channel_request_by_id: HashMap::new(), + inbound_channel_request_by_id: new_hash_map(), latest_features: InitFeatures::empty(), pending_msg_events: Vec::new(), in_flight_monitor_updates: BTreeMap::new(), @@ -10507,10 +10507,10 @@ where }; let peer_count: u64 = Readable::read(reader)?; - let mut per_peer_state = HashMap::with_capacity(cmp::min(peer_count as usize, MAX_ALLOC_SIZE/mem::size_of::<(PublicKey, Mutex>)>())); + let mut per_peer_state = hash_map_with_capacity(cmp::min(peer_count as usize, MAX_ALLOC_SIZE/mem::size_of::<(PublicKey, Mutex>)>())); for _ in 0..peer_count { let peer_pubkey = Readable::read(reader)?; - let peer_chans = funded_peer_channels.remove(&peer_pubkey).unwrap_or(HashMap::new()); + let peer_chans = funded_peer_channels.remove(&peer_pubkey).unwrap_or(new_hash_map()); let mut peer_state = peer_state_from_chans(peer_chans); peer_state.latest_features = Readable::read(reader)?; per_peer_state.insert(peer_pubkey, Mutex::new(peer_state)); @@ -10544,7 +10544,7 @@ where let highest_seen_timestamp: u32 = Readable::read(reader)?; let pending_inbound_payment_count: u64 = Readable::read(reader)?; - let mut pending_inbound_payments: HashMap = HashMap::with_capacity(cmp::min(pending_inbound_payment_count as usize, MAX_ALLOC_SIZE/(3*32))); + let mut pending_inbound_payments: HashMap = hash_map_with_capacity(cmp::min(pending_inbound_payment_count as usize, MAX_ALLOC_SIZE/(3*32))); for _ in 0..pending_inbound_payment_count { if pending_inbound_payments.insert(Readable::read(reader)?, Readable::read(reader)?).is_some() { return Err(DecodeError::InvalidValue); @@ -10553,7 +10553,7 @@ where let pending_outbound_payments_count_compat: u64 = Readable::read(reader)?; let mut pending_outbound_payments_compat: HashMap = - HashMap::with_capacity(cmp::min(pending_outbound_payments_count_compat as usize, MAX_ALLOC_SIZE/32)); + hash_map_with_capacity(cmp::min(pending_outbound_payments_count_compat as usize, MAX_ALLOC_SIZE/32)); for _ in 0..pending_outbound_payments_count_compat { let session_priv = Readable::read(reader)?; let payment = PendingOutboundPayment::Legacy { @@ -10567,13 +10567,13 @@ where // pending_outbound_payments_no_retry is for compatibility with 0.0.101 clients. let mut pending_outbound_payments_no_retry: Option>> = None; let mut pending_outbound_payments = None; - let mut pending_intercepted_htlcs: Option> = Some(HashMap::new()); + let mut pending_intercepted_htlcs: Option> = Some(new_hash_map()); let mut received_network_pubkey: Option = None; let mut fake_scid_rand_bytes: Option<[u8; 32]> = None; let mut probing_cookie_secret: Option<[u8; 32]> = None; let mut claimable_htlc_purposes = None; let mut claimable_htlc_onion_fields = None; - let mut pending_claiming_payments = Some(HashMap::new()); + let mut pending_claiming_payments = Some(new_hash_map()); let mut monitor_update_blocked_actions_per_peer: Option>)>> = Some(Vec::new()); let mut events_override = None; let mut in_flight_monitor_updates: Option>> = None; @@ -10610,7 +10610,7 @@ where if pending_outbound_payments.is_none() && pending_outbound_payments_no_retry.is_none() { pending_outbound_payments = Some(pending_outbound_payments_compat); } else if pending_outbound_payments.is_none() { - let mut outbounds = HashMap::new(); + let mut outbounds = new_hash_map(); for (id, session_privs) in pending_outbound_payments_no_retry.unwrap().drain() { outbounds.insert(id, PendingOutboundPayment::Legacy { session_privs }); } @@ -10719,7 +10719,7 @@ where // still open, we need to replay any monitor updates that are for closed channels, // creating the neccessary peer_state entries as we go. let peer_state_mutex = per_peer_state.entry(counterparty_id).or_insert_with(|| { - Mutex::new(peer_state_from_chans(HashMap::new())) + Mutex::new(peer_state_from_chans(new_hash_map())) }); let mut peer_state = peer_state_mutex.lock().unwrap(); handle_in_flight_updates!(counterparty_id, chan_in_flight_updates, @@ -10905,7 +10905,7 @@ where let inbound_pmt_key_material = args.node_signer.get_inbound_payment_key_material(); let expanded_inbound_key = inbound_payment::ExpandedKey::new(&inbound_pmt_key_material); - let mut claimable_payments = HashMap::with_capacity(claimable_htlcs_list.len()); + let mut claimable_payments = hash_map_with_capacity(claimable_htlcs_list.len()); if let Some(purposes) = claimable_htlc_purposes { if purposes.len() != claimable_htlcs_list.len() { return Err(DecodeError::InvalidValue); @@ -10978,7 +10978,7 @@ where } } - let mut outbound_scid_aliases = HashSet::new(); + let mut outbound_scid_aliases = new_hash_set(); for (_peer_node_id, peer_state_mutex) in per_peer_state.iter_mut() { let mut peer_state_lock = peer_state_mutex.lock().unwrap(); let peer_state = &mut *peer_state_lock; @@ -12511,7 +12511,7 @@ mod tests { let (scid_1, scid_2) = (42, 43); - let mut forward_htlcs = HashMap::new(); + let mut forward_htlcs = new_hash_map(); forward_htlcs.insert(scid_1, dummy_htlcs_1.clone()); forward_htlcs.insert(scid_2, dummy_htlcs_2.clone()); diff --git a/lightning/src/ln/functional_test_utils.rs b/lightning/src/ln/functional_test_utils.rs index e53fd6b0c1e..26f64721180 100644 --- a/lightning/src/ln/functional_test_utils.rs +++ b/lightning/src/ln/functional_test_utils.rs @@ -618,7 +618,7 @@ impl<'a, 'b, 'c> Drop for Node<'a, 'b, 'c> { // Before using all the new monitors to check the watch outpoints, use the full set of // them to ensure we can write and reload our ChannelManager. { - let mut channel_monitors = HashMap::new(); + let mut channel_monitors = new_hash_map(); for monitor in deserialized_monitors.iter_mut() { channel_monitors.insert(monitor.get_funding_txo().0, monitor); } @@ -1049,7 +1049,7 @@ pub fn _reload_node<'a, 'b, 'c>(node: &'a Node<'a, 'b, 'c>, default_config: User let mut node_read = &chanman_encoded[..]; let (_, node_deserialized) = { - let mut channel_monitors = HashMap::new(); + let mut channel_monitors = new_hash_map(); for monitor in monitors_read.iter_mut() { assert!(channel_monitors.insert(monitor.get_funding_txo().0, monitor).is_none()); } @@ -3104,7 +3104,7 @@ pub enum HTLCType { NONE, TIMEOUT, SUCCESS } /// also fail. pub fn test_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, chan: &(msgs::ChannelUpdate, msgs::ChannelUpdate, ChannelId, Transaction), commitment_tx: Option, has_htlc_tx: HTLCType) -> Vec { let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap(); - let mut txn_seen = HashSet::new(); + let mut txn_seen = new_hash_set(); node_txn.retain(|tx| txn_seen.insert(tx.txid())); assert!(node_txn.len() >= if commitment_tx.is_some() { 0 } else { 1 } + if has_htlc_tx == HTLCType::NONE { 0 } else { 1 }); @@ -3169,7 +3169,7 @@ pub fn test_revoked_htlc_claim_txn_broadcast<'a, 'b, 'c>(node: &Node<'a, 'b, 'c> pub fn check_preimage_claim<'a, 'b, 'c>(node: &Node<'a, 'b, 'c>, prev_txn: &Vec) -> Vec { let mut node_txn = node.tx_broadcaster.txn_broadcasted.lock().unwrap(); - let mut txn_seen = HashSet::new(); + let mut txn_seen = new_hash_set(); node_txn.retain(|tx| txn_seen.insert(tx.txid())); let mut found_prev = false; @@ -3269,7 +3269,7 @@ macro_rules! get_channel_value_stat { macro_rules! get_chan_reestablish_msgs { ($src_node: expr, $dst_node: expr) => { { - let mut announcements = $crate::prelude::HashSet::new(); + let mut announcements = $crate::prelude::new_hash_set(); let mut res = Vec::with_capacity(1); for msg in $src_node.node.get_and_clear_pending_msg_events() { if let MessageSendEvent::SendChannelReestablish { ref node_id, ref msg } = msg { diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 5f989103170..821fd1f78bb 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -3323,7 +3323,7 @@ fn do_test_commitment_revoked_fail_backward_exhaustive(deliver_bs_raa: bool, use // block connection just like the !deliver_bs_raa case } - let mut failed_htlcs = HashSet::new(); + let mut failed_htlcs = new_hash_set(); assert!(nodes[1].node.get_and_clear_pending_events().is_empty()); mine_transaction(&nodes[1], &revoked_local_txn[0]); @@ -5402,7 +5402,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno let as_events = nodes[0].node.get_and_clear_pending_events(); assert_eq!(as_events.len(), if announce_latest { 10 } else { 6 }); - let mut as_failds = HashSet::new(); + let mut as_failds = new_hash_set(); let mut as_updates = 0; for event in as_events.iter() { if let &Event::PaymentPathFailed { ref payment_hash, ref payment_failed_permanently, ref failure, .. } = event { @@ -5428,7 +5428,7 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno let bs_events = nodes[1].node.get_and_clear_pending_events(); assert_eq!(bs_events.len(), if announce_latest { 8 } else { 6 }); - let mut bs_failds = HashSet::new(); + let mut bs_failds = new_hash_set(); let mut bs_updates = 0; for event in bs_events.iter() { if let &Event::PaymentPathFailed { ref payment_hash, ref payment_failed_permanently, ref failure, .. } = event { @@ -7340,7 +7340,7 @@ fn test_announce_disable_channels() { } let msg_events = nodes[0].node.get_and_clear_pending_msg_events(); assert_eq!(msg_events.len(), 3); - let mut chans_disabled = HashMap::new(); + let mut chans_disabled = new_hash_map(); for e in msg_events { match e { MessageSendEvent::BroadcastChannelUpdate { ref msg } => { diff --git a/lightning/src/ln/monitor_tests.rs b/lightning/src/ln/monitor_tests.rs index 6a56c3cf816..aaa4a0f66c8 100644 --- a/lightning/src/ln/monitor_tests.rs +++ b/lightning/src/ln/monitor_tests.rs @@ -2572,7 +2572,7 @@ fn test_anchors_aggregated_revoked_htlc_tx() { check_spends!(revoked_htlc_claim, htlc_tx); } - let mut revoked_claim_transaction_map = HashMap::new(); + let mut revoked_claim_transaction_map = new_hash_map(); for current_tx in txn.into_iter() { revoked_claim_transaction_map.insert(current_tx.txid(), current_tx); } diff --git a/lightning/src/ln/outbound_payment.rs b/lightning/src/ln/outbound_payment.rs index db8b43e3f67..c260bac856a 100644 --- a/lightning/src/ln/outbound_payment.rs +++ b/lightning/src/ln/outbound_payment.rs @@ -171,7 +171,7 @@ impl PendingOutboundPayment { } fn mark_fulfilled(&mut self) { - let mut session_privs = HashSet::new(); + let mut session_privs = new_hash_set(); core::mem::swap(&mut session_privs, match self { PendingOutboundPayment::Legacy { session_privs } | PendingOutboundPayment::Retryable { session_privs, .. } | @@ -186,7 +186,7 @@ impl PendingOutboundPayment { fn mark_abandoned(&mut self, reason: PaymentFailureReason) { if let PendingOutboundPayment::Retryable { session_privs, payment_hash, .. } = self { - let mut our_session_privs = HashSet::new(); + let mut our_session_privs = new_hash_set(); core::mem::swap(&mut our_session_privs, session_privs); *self = PendingOutboundPayment::Abandoned { session_privs: our_session_privs, @@ -195,7 +195,7 @@ impl PendingOutboundPayment { }; } else if let PendingOutboundPayment::InvoiceReceived { payment_hash, .. } = self { *self = PendingOutboundPayment::Abandoned { - session_privs: HashSet::new(), + session_privs: new_hash_set(), payment_hash: *payment_hash, reason: Some(reason) }; @@ -675,7 +675,7 @@ pub(super) struct OutboundPayments { impl OutboundPayments { pub(super) fn new() -> Self { Self { - pending_outbound_payments: Mutex::new(HashMap::new()), + pending_outbound_payments: Mutex::new(new_hash_map()), retry_lock: Mutex::new(()), } } @@ -1268,7 +1268,7 @@ impl OutboundPayments { retry_strategy, attempts: PaymentAttempts::new(), payment_params, - session_privs: HashSet::new(), + session_privs: new_hash_set(), pending_amt_msat: 0, pending_fee_msat: Some(0), payment_hash, diff --git a/lightning/src/ln/peer_handler.rs b/lightning/src/ln/peer_handler.rs index 3e07f50b203..b062a9a888c 100644 --- a/lightning/src/ln/peer_handler.rs +++ b/lightning/src/ln/peer_handler.rs @@ -944,8 +944,8 @@ impl HashMap> { let mut message_recipients = self.message_recipients.lock().unwrap(); - let mut msgs = HashMap::new(); + let mut msgs = new_hash_map(); // We don't want to disconnect the peers by removing them entirely from the original map, so we // release the pending message buffers individually. for (node_id, recipient) in &mut *message_recipients { diff --git a/lightning/src/routing/gossip.rs b/lightning/src/routing/gossip.rs index 9c8fd40af13..801f6bfa71a 100644 --- a/lightning/src/routing/gossip.rs +++ b/lightning/src/routing/gossip.rs @@ -1353,8 +1353,8 @@ impl ReadableArgs for NetworkGraph where L::Target: Logger { channels: RwLock::new(channels), nodes: RwLock::new(nodes), last_rapid_gossip_sync_timestamp: Mutex::new(last_rapid_gossip_sync_timestamp), - removed_nodes: Mutex::new(HashMap::new()), - removed_channels: Mutex::new(HashMap::new()), + removed_nodes: Mutex::new(new_hash_map()), + removed_channels: Mutex::new(new_hash_map()), pending_checks: utxo::PendingChecks::new(), }) } @@ -1398,8 +1398,8 @@ impl NetworkGraph where L::Target: Logger { channels: RwLock::new(IndexedMap::new()), nodes: RwLock::new(IndexedMap::new()), last_rapid_gossip_sync_timestamp: Mutex::new(None), - removed_channels: Mutex::new(HashMap::new()), - removed_nodes: Mutex::new(HashMap::new()), + removed_channels: Mutex::new(new_hash_map()), + removed_nodes: Mutex::new(new_hash_map()), pending_checks: utxo::PendingChecks::new(), } } diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 1bf90883ec5..4214a1821cb 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -277,7 +277,7 @@ pub struct InFlightHtlcs( impl InFlightHtlcs { /// Constructs an empty `InFlightHtlcs`. - pub fn new() -> Self { InFlightHtlcs(HashMap::new()) } + pub fn new() -> Self { InFlightHtlcs(new_hash_map()) } /// Takes in a path with payer's node id and adds the path's details to `InFlightHtlcs`. pub fn process_path(&mut self, path: &Path, payer_node_id: PublicKey) { @@ -1962,7 +1962,7 @@ where L::Target: Logger { // inserting first hops suggested by the caller as targets. // Our search will then attempt to reach them while traversing from the payee node. let mut first_hop_targets: HashMap<_, Vec<&ChannelDetails>> = - HashMap::with_capacity(if first_hops.is_some() { first_hops.as_ref().unwrap().len() } else { 0 }); + hash_map_with_capacity(if first_hops.is_some() { first_hops.as_ref().unwrap().len() } else { 0 }); if let Some(hops) = first_hops { for chan in hops { if chan.get_outbound_payment_scid().is_none() { @@ -1981,7 +1981,7 @@ where L::Target: Logger { } } - let mut private_hop_key_cache = HashMap::with_capacity( + let mut private_hop_key_cache = hash_map_with_capacity( payment_params.payee.unblinded_route_hints().iter().map(|path| path.0.len()).sum() ); @@ -2002,7 +2002,7 @@ where L::Target: Logger { // Map from node_id to information about the best current path to that node, including feerate // information. - let mut dist: HashMap = HashMap::with_capacity(network_nodes.len()); + let mut dist: HashMap = hash_map_with_capacity(network_nodes.len()); // During routing, if we ignore a path due to an htlc_minimum_msat limit, we set this, // indicating that we may wish to try again with a higher value, potentially paying to meet an @@ -2043,7 +2043,7 @@ where L::Target: Logger { // is used. Hence, liquidity used in one direction will not offset any used in the opposite // direction. let mut used_liquidities: HashMap = - HashMap::with_capacity(network_nodes.len()); + hash_map_with_capacity(network_nodes.len()); // Keeping track of how much value we already collected across other paths. Helps to decide // when we want to stop looking for new paths. diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index 8c36bd100e5..ec4921812e9 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -653,7 +653,7 @@ impl Default for ProbabilisticScoringFeeParameters { base_penalty_amount_multiplier_msat: 8192, liquidity_penalty_multiplier_msat: 30_000, liquidity_penalty_amount_multiplier_msat: 192, - manual_node_penalties: HashMap::new(), + manual_node_penalties: new_hash_map(), anti_probing_penalty_msat: 250, considered_impossible_penalty_msat: 1_0000_0000_000, historical_liquidity_penalty_multiplier_msat: 10_000, @@ -695,7 +695,7 @@ impl ProbabilisticScoringFeeParameters { /// Clears the list of manual penalties that are applied during path finding. pub fn clear_manual_penalties(&mut self) { - self.manual_node_penalties = HashMap::new(); + self.manual_node_penalties = new_hash_map(); } } @@ -709,7 +709,7 @@ impl ProbabilisticScoringFeeParameters { liquidity_penalty_amount_multiplier_msat: 0, historical_liquidity_penalty_multiplier_msat: 0, historical_liquidity_penalty_amount_multiplier_msat: 0, - manual_node_penalties: HashMap::new(), + manual_node_penalties: new_hash_map(), anti_probing_penalty_msat: 0, considered_impossible_penalty_msat: 0, linear_success_probability: true, @@ -819,7 +819,7 @@ impl>, L: Deref> ProbabilisticScorer whe decay_params, network_graph, logger, - channel_liquidities: HashMap::new(), + channel_liquidities: new_hash_map(), } } @@ -2073,7 +2073,7 @@ ReadableArgs<(ProbabilisticScoringDecayParameters, G, L)> for ProbabilisticScore r: &mut R, args: (ProbabilisticScoringDecayParameters, G, L) ) -> Result { let (decay_params, network_graph, logger) = args; - let mut channel_liquidities = HashMap::new(); + let mut channel_liquidities = new_hash_map(); read_tlv_fields!(r, { (0, channel_liquidities, required), }); diff --git a/lightning/src/routing/utxo.rs b/lightning/src/routing/utxo.rs index e190d4258ae..ada90345ee6 100644 --- a/lightning/src/routing/utxo.rs +++ b/lightning/src/routing/utxo.rs @@ -307,7 +307,7 @@ pub(super) struct PendingChecks { impl PendingChecks { pub(super) fn new() -> Self { PendingChecks { internal: Mutex::new(PendingChecksContext { - channels: HashMap::new(), nodes: HashMap::new(), + channels: new_hash_map(), nodes: new_hash_map(), }) } } diff --git a/lightning/src/sign/mod.rs b/lightning/src/sign/mod.rs index e2a89d8485f..a7237493be7 100644 --- a/lightning/src/sign/mod.rs +++ b/lightning/src/sign/mod.rs @@ -350,7 +350,7 @@ impl SpendableOutputDescriptor { let mut input = Vec::with_capacity(descriptors.len()); let mut input_value = 0; let mut witness_weight = 0; - let mut output_set = HashSet::with_capacity(descriptors.len()); + let mut output_set = hash_set_with_capacity(descriptors.len()); for outp in descriptors { match outp { SpendableOutputDescriptor::StaticPaymentOutput(descriptor) => { diff --git a/lightning/src/sync/debug_sync.rs b/lightning/src/sync/debug_sync.rs index b9f015af656..2b75e095380 100644 --- a/lightning/src/sync/debug_sync.rs +++ b/lightning/src/sync/debug_sync.rs @@ -14,7 +14,7 @@ use std::sync::Condvar as StdCondvar; pub use std::sync::WaitTimeoutResult; -use crate::prelude::HashMap; +use crate::prelude::*; use super::{LockTestExt, LockHeldState}; @@ -57,7 +57,7 @@ impl Condvar { thread_local! { /// We track the set of locks currently held by a reference to their `LockMetadata` - static LOCKS_HELD: RefCell>> = RefCell::new(HashMap::new()); + static LOCKS_HELD: RefCell>> = RefCell::new(new_hash_map()); } static LOCK_IDX: AtomicUsize = AtomicUsize::new(0); @@ -113,7 +113,7 @@ impl LockMetadata { let lock_idx = LOCK_IDX.fetch_add(1, Ordering::Relaxed) as u64; let res = Arc::new(LockMetadata { - locked_before: StdMutex::new(HashMap::new()), + locked_before: StdMutex::new(new_hash_map()), lock_idx, _lock_construction_bt: backtrace, }); @@ -122,7 +122,7 @@ impl LockMetadata { { let (lock_constr_location, lock_constr_colno) = locate_call_symbol(&res._lock_construction_bt); - LOCKS_INIT.call_once(|| { unsafe { LOCKS = Some(StdMutex::new(HashMap::new())); } }); + LOCKS_INIT.call_once(|| { unsafe { LOCKS = Some(StdMutex::new(new_hash_map())); } }); let mut locks = unsafe { LOCKS.as_ref() }.unwrap().lock().unwrap(); match locks.entry(lock_constr_location) { hash_map::Entry::Occupied(e) => { diff --git a/lightning/src/util/indexed_map.rs b/lightning/src/util/indexed_map.rs index 39565f048c0..d4c20f72215 100644 --- a/lightning/src/util/indexed_map.rs +++ b/lightning/src/util/indexed_map.rs @@ -1,6 +1,6 @@ //! This module has a map which can be iterated in a deterministic order. See the [`IndexedMap`]. -use crate::prelude::{HashMap, hash_map}; +use crate::prelude::*; use alloc::vec::Vec; use alloc::slice::Iter; use core::hash::Hash; @@ -34,7 +34,7 @@ impl IndexedMap { /// Constructs a new, empty map pub fn new() -> Self { Self { - map: HashMap::new(), + map: new_hash_map(), keys: Vec::new(), } } @@ -42,7 +42,7 @@ impl IndexedMap { /// Constructs a new, empty map with the given capacity pre-allocated pub fn with_capacity(capacity: usize) -> Self { Self { - map: HashMap::with_capacity(capacity), + map: hash_map_with_capacity(capacity), keys: Vec::with_capacity(capacity), } } @@ -176,10 +176,7 @@ impl<'a, K: Hash + Ord, V: 'a> Iterator for Range<'a, K, V> { /// /// This is not exported to bindings users as bindings provide alternate accessors rather than exposing maps directly. pub struct VacantEntry<'a, K: Hash + Ord, V> { - #[cfg(feature = "hashbrown")] - underlying_entry: hash_map::VacantEntry<'a, K, V, hash_map::DefaultHashBuilder>, - #[cfg(not(feature = "hashbrown"))] - underlying_entry: hash_map::VacantEntry<'a, K, V>, + underlying_entry: VacantHashMapEntry<'a, K, V>, key: K, keys: &'a mut Vec, } @@ -188,10 +185,7 @@ pub struct VacantEntry<'a, K: Hash + Ord, V> { /// /// This is not exported to bindings users as bindings provide alternate accessors rather than exposing maps directly. pub struct OccupiedEntry<'a, K: Hash + Ord, V> { - #[cfg(feature = "hashbrown")] - underlying_entry: hash_map::OccupiedEntry<'a, K, V, hash_map::DefaultHashBuilder>, - #[cfg(not(feature = "hashbrown"))] - underlying_entry: hash_map::OccupiedEntry<'a, K, V>, + underlying_entry: OccupiedHashMapEntry<'a, K, V>, keys: &'a mut Vec, } diff --git a/lightning/src/util/ser.rs b/lightning/src/util/ser.rs index 484d6034042..c24a99f8f97 100644 --- a/lightning/src/util/ser.rs +++ b/lightning/src/util/ser.rs @@ -749,7 +749,7 @@ macro_rules! impl_for_map { } impl_for_map!(BTreeMap, Ord, |_| BTreeMap::new()); -impl_for_map!(HashMap, Hash, |len| HashMap::with_capacity(len)); +impl_for_map!(HashMap, Hash, |len| hash_map_with_capacity(len)); // HashSet impl Writeable for HashSet @@ -771,7 +771,7 @@ where T: Readable + Eq + Hash #[inline] fn read(r: &mut R) -> Result { let len: CollectionLength = Readable::read(r)?; - let mut ret = HashSet::with_capacity(cmp::min(len.0 as usize, MAX_BUF_SIZE / core::mem::size_of::())); + let mut ret = hash_set_with_capacity(cmp::min(len.0 as usize, MAX_BUF_SIZE / core::mem::size_of::())); for _ in 0..len.0 { if !ret.insert(T::read(r)?) { return Err(DecodeError::InvalidValue) diff --git a/lightning/src/util/test_utils.rs b/lightning/src/util/test_utils.rs index a96711d144c..bd415e42b1f 100644 --- a/lightning/src/util/test_utils.rs +++ b/lightning/src/util/test_utils.rs @@ -318,8 +318,8 @@ impl<'a> TestChainMonitor<'a> { pub fn new(chain_source: Option<&'a TestChainSource>, broadcaster: &'a dyn chaininterface::BroadcasterInterface, logger: &'a TestLogger, fee_estimator: &'a TestFeeEstimator, persister: &'a dyn chainmonitor::Persist, keys_manager: &'a TestKeysInterface) -> Self { Self { added_monitors: Mutex::new(Vec::new()), - monitor_updates: Mutex::new(HashMap::new()), - latest_monitor_update_id: Mutex::new(HashMap::new()), + monitor_updates: Mutex::new(new_hash_map()), + latest_monitor_update_id: Mutex::new(new_hash_map()), chain_monitor: chainmonitor::ChainMonitor::new(chain_source, broadcaster, logger, fee_estimator, persister), keys_manager, expect_channel_force_closed: Mutex::new(None), @@ -414,8 +414,8 @@ impl WatchtowerPersister { pub(crate) fn new(destination_script: ScriptBuf) -> Self { WatchtowerPersister { persister: TestPersister::new(), - unsigned_justice_tx_data: Mutex::new(HashMap::new()), - watchtower_state: Mutex::new(HashMap::new()), + unsigned_justice_tx_data: Mutex::new(new_hash_map()), + watchtower_state: Mutex::new(new_hash_map()), destination_script, } } @@ -448,7 +448,7 @@ impl chainmonitor::Persist Self { Self { update_rets: Mutex::new(VecDeque::new()), - chain_sync_monitor_persistences: Mutex::new(HashMap::new()), - offchain_monitor_updates: Mutex::new(HashMap::new()), + chain_sync_monitor_persistences: Mutex::new(new_hash_map()), + offchain_monitor_updates: Mutex::new(new_hash_map()), } } @@ -534,9 +534,9 @@ impl chainmonitor::Persist Self { - let persisted_bytes = Mutex::new(HashMap::new()); + let persisted_bytes = Mutex::new(new_hash_map()); Self { persisted_bytes, read_only } } } @@ -589,7 +589,7 @@ impl KVStore for TestStore { } else { format!("{}/{}", primary_namespace, secondary_namespace) }; - let outer_e = persisted_lock.entry(prefixed).or_insert(HashMap::new()); + let outer_e = persisted_lock.entry(prefixed).or_insert(new_hash_map()); let mut bytes = Vec::new(); bytes.write_all(buf)?; outer_e.insert(key.to_string(), bytes); @@ -656,7 +656,7 @@ impl TestBroadcaster { pub fn unique_txn_broadcast(&self) -> Vec { let mut txn = self.txn_broadcasted.lock().unwrap().split_off(0); - let mut seen = HashSet::new(); + let mut seen = new_hash_set(); txn.retain(|tx| seen.insert(tx.txid())); txn } @@ -693,7 +693,7 @@ impl TestChannelMessageHandler { TestChannelMessageHandler { pending_events: Mutex::new(Vec::new()), expected_recv_msgs: Mutex::new(None), - connected_peers: Mutex::new(HashSet::new()), + connected_peers: Mutex::new(new_hash_set()), message_fetch_counter: AtomicUsize::new(0), chain_hash, } @@ -1045,8 +1045,8 @@ impl TestLogger { TestLogger { level: Level::Trace, id, - lines: Mutex::new(HashMap::new()), - context: Mutex::new(HashMap::new()), + lines: Mutex::new(new_hash_map()), + context: Mutex::new(new_hash_map()), } } pub fn enable(&mut self, level: Level) { @@ -1259,7 +1259,7 @@ impl TestKeysInterface { backing: sign::PhantomKeysManager::new(seed, now.as_secs(), now.subsec_nanos(), seed), override_random_bytes: Mutex::new(None), disable_revocation_policy_check: false, - enforcement_states: Mutex::new(HashMap::new()), + enforcement_states: Mutex::new(new_hash_map()), expectations: Mutex::new(None), } } @@ -1340,8 +1340,8 @@ impl TestChainSource { chain_hash: ChainHash::using_genesis_block(network), utxo_ret: Mutex::new(UtxoResult::Sync(Ok(TxOut { value: u64::max_value(), script_pubkey }))), get_utxo_call_count: AtomicUsize::new(0), - watched_txn: Mutex::new(HashSet::new()), - watched_outputs: Mutex::new(HashSet::new()), + watched_txn: Mutex::new(new_hash_set()), + watched_outputs: Mutex::new(new_hash_set()), } } } From efbaa41fae811bea2c004799f7e36c2882941985 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 18 Jan 2024 20:27:57 +0000 Subject: [PATCH 6/9] Stop relying on `Hash{Set,Map}::from_iter` directly In the next commit we'll bump the `hashbrown` version, which no longer randomizes its hasher by default. Thus, we'll need to call a different constructor in no-std builds from std builds. Here we do a quick prefactor to use wrappers for `FromIterator` constructors instead of calling the tables directly to make the version bump changeset smaller. --- lightning/src/chain/chainmonitor.rs | 7 +++---- lightning/src/lib.rs | 6 ++++++ lightning/src/ln/channelmanager.rs | 8 +++++--- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/lightning/src/chain/chainmonitor.rs b/lightning/src/chain/chainmonitor.rs index 4adafd2db8b..0e8a027897d 100644 --- a/lightning/src/chain/chainmonitor.rs +++ b/lightning/src/chain/chainmonitor.rs @@ -43,7 +43,6 @@ use crate::ln::channelmanager::ChannelDetails; use crate::prelude::*; use crate::sync::{RwLock, RwLockReadGuard, Mutex, MutexGuard}; -use core::iter::FromIterator; use core::ops::Deref; use core::sync::atomic::{AtomicUsize, Ordering}; use bitcoin::secp256k1::PublicKey; @@ -318,7 +317,7 @@ where C::Target: chain::Filter, FN: Fn(&ChannelMonitor, &TransactionData) -> Vec { let err_str = "ChannelMonitor[Update] persistence failed unrecoverably. This indicates we cannot continue normal operation and must shut down."; - let funding_outpoints: HashSet = HashSet::from_iter(self.monitors.read().unwrap().keys().cloned()); + let funding_outpoints = hash_set_from_iter(self.monitors.read().unwrap().keys().cloned()); for funding_outpoint in funding_outpoints.iter() { let monitor_lock = self.monitors.read().unwrap(); if let Some(monitor_state) = monitor_lock.get(funding_outpoint) { @@ -486,9 +485,9 @@ where C::Target: chain::Filter, #[cfg(not(c_bindings))] /// Lists the pending updates for each [`ChannelMonitor`] (by `OutPoint` being monitored). pub fn list_pending_monitor_updates(&self) -> HashMap> { - self.monitors.read().unwrap().iter().map(|(outpoint, holder)| { + hash_map_from_iter(self.monitors.read().unwrap().iter().map(|(outpoint, holder)| { (*outpoint, holder.pending_monitor_updates.lock().unwrap().clone()) - }).collect() + })) } #[cfg(c_bindings)] diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 33f596fd8b9..77b1f91cbc0 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -198,11 +198,17 @@ mod prelude { pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { HashMap::with_capacity(cap) } + pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { + HashMap::from_iter(iter) + } pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { HashSet::with_capacity(cap) } + pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { + HashSet::from_iter(iter) + } pub use alloc::borrow::ToOwned; pub use alloc::string::ToString; diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index a9ae6ee3cec..4d7be7d92b2 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -10284,7 +10284,9 @@ where mut channel_monitors: Vec<&'a mut ChannelMonitor<::EcdsaSigner>>) -> Self { Self { entropy_source, node_signer, signer_provider, fee_estimator, chain_monitor, tx_broadcaster, router, logger, default_config, - channel_monitors: channel_monitors.drain(..).map(|monitor| { (monitor.get_funding_txo().0, monitor) }).collect() + channel_monitors: hash_map_from_iter( + channel_monitors.drain(..).map(|monitor| { (monitor.get_funding_txo().0, monitor) }) + ), } } } @@ -10557,7 +10559,7 @@ where for _ in 0..pending_outbound_payments_count_compat { let session_priv = Readable::read(reader)?; let payment = PendingOutboundPayment::Legacy { - session_privs: [session_priv].iter().cloned().collect() + session_privs: hash_set_from_iter([session_priv]), }; if pending_outbound_payments_compat.insert(PaymentId(session_priv), payment).is_some() { return Err(DecodeError::InvalidValue) @@ -10780,7 +10782,7 @@ where retry_strategy: None, attempts: PaymentAttempts::new(), payment_params: None, - session_privs: [session_priv_bytes].iter().map(|a| *a).collect(), + session_privs: hash_set_from_iter([session_priv_bytes]), payment_hash: htlc.payment_hash, payment_secret: None, // only used for retries, and we'll never retry on startup payment_metadata: None, // only used for retries, and we'll never retry on startup From dedc8306f68e397dd51d0e9055eab2c04e373cdf Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 18 Jan 2024 22:39:26 +0000 Subject: [PATCH 7/9] Bump `hashbrown` dependency to 0.13 While this isn't expected to materially improve performance, it does get us ahash 0.8, which allows us to reduce fuzzing randomness, making our fuzzers much happier. Sadly, by default `ahash` no longer tries to autodetect a randomness source, so we cannot simply rely on `hashbrown` to do randomization for us, but rather have to also explicitly depend on `ahash`. --- .github/workflows/build.yml | 2 +- bench/Cargo.toml | 2 +- ci/check-cfg-flags.py | 2 + fuzz/Cargo.toml | 2 +- lightning-invoice/Cargo.toml | 2 +- lightning/Cargo.toml | 15 ++++- lightning/src/chain/onchaintx.rs | 2 +- lightning/src/lib.rs | 100 +++++++++++++++++++++++------ lightning/src/ln/channelmanager.rs | 6 +- lightning/src/routing/router.rs | 8 +-- lightning/src/routing/scoring.rs | 4 +- 11 files changed, 109 insertions(+), 36 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index cff105470d2..ab10b36d25a 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -110,7 +110,7 @@ jobs: run: | cd lightning RUSTFLAGS="--cfg=require_route_graph_test" cargo test - RUSTFLAGS="--cfg=require_route_graph_test" cargo test --features hashbrown + RUSTFLAGS="--cfg=require_route_graph_test" cargo test --features hashbrown,ahash cd .. - name: Run benchmarks on Rust ${{ matrix.toolchain }} run: | diff --git a/bench/Cargo.toml b/bench/Cargo.toml index 05354890c2a..ff254e1d9cc 100644 --- a/bench/Cargo.toml +++ b/bench/Cargo.toml @@ -9,7 +9,7 @@ name = "bench" harness = false [features] -hashbrown = ["lightning/hashbrown"] +hashbrown = ["lightning/hashbrown", "lightning/ahash"] [dependencies] lightning = { path = "../lightning", features = ["_test_utils", "criterion"] } diff --git a/ci/check-cfg-flags.py b/ci/check-cfg-flags.py index 277ae1077ba..d6e8e0a90fe 100755 --- a/ci/check-cfg-flags.py +++ b/ci/check-cfg-flags.py @@ -13,6 +13,8 @@ def check_feature(feature): pass elif feature == "no-std": pass + elif feature == "ahash": + pass elif feature == "hashbrown": pass elif feature == "backtrace": diff --git a/fuzz/Cargo.toml b/fuzz/Cargo.toml index 0c279a015c7..34c8704a7ed 100644 --- a/fuzz/Cargo.toml +++ b/fuzz/Cargo.toml @@ -22,7 +22,7 @@ lightning = { path = "../lightning", features = ["regex", "hashbrown", "_test_ut lightning-rapid-gossip-sync = { path = "../lightning-rapid-gossip-sync" } bitcoin = { version = "0.30.2", features = ["secp-lowmemory"] } hex = { package = "hex-conservative", version = "0.1.1", default-features = false } -hashbrown = "0.8" +hashbrown = "0.13" afl = { version = "0.12", optional = true } honggfuzz = { version = "0.5", optional = true, default-features = false } diff --git a/lightning-invoice/Cargo.toml b/lightning-invoice/Cargo.toml index e0d3560c10b..4376ac0dc70 100644 --- a/lightning-invoice/Cargo.toml +++ b/lightning-invoice/Cargo.toml @@ -24,7 +24,7 @@ bech32 = { version = "0.9.0", default-features = false } lightning = { version = "0.0.121", path = "../lightning", default-features = false } secp256k1 = { version = "0.27.0", default-features = false, features = ["recovery", "alloc"] } num-traits = { version = "0.2.8", default-features = false } -hashbrown = { version = "0.8", optional = true } +hashbrown = { version = "0.13", optional = true } serde = { version = "1.0.118", optional = true } bitcoin = { version = "0.30.2", default-features = false } diff --git a/lightning/Cargo.toml b/lightning/Cargo.toml index 1fe0f0dd119..8214193cc42 100644 --- a/lightning/Cargo.toml +++ b/lightning/Cargo.toml @@ -31,7 +31,7 @@ unsafe_revoked_tx_signing = [] # Override signing to not include randomness when generating signatures for test vectors. _test_vectors = [] -no-std = ["hashbrown", "bitcoin/no-std", "core2/alloc", "libm"] +no-std = ["hashbrown", "ahash", "bitcoin/no-std", "core2/alloc", "libm"] std = ["bitcoin/std"] # Generates low-r bitcoin signatures, which saves 1 byte in 50% of the cases @@ -42,7 +42,8 @@ default = ["std", "grind_signatures"] [dependencies] bitcoin = { version = "0.30.2", default-features = false, features = ["secp-recovery"] } -hashbrown = { version = "0.8", optional = true } +hashbrown = { version = "0.13", optional = true } +ahash = { version = "0.8", optional = true, default-features = false } hex = { package = "hex-conservative", version = "0.1.1", default-features = false } regex = { version = "1.5.6", optional = true } backtrace = { version = "0.3", optional = true } @@ -50,6 +51,16 @@ backtrace = { version = "0.3", optional = true } core2 = { version = "0.3.0", optional = true, default-features = false } libm = { version = "0.2", optional = true, default-features = false } +# Because ahash no longer (kinda poorly) does it for us, (roughly) list out the targets that +# getrandom supports and turn on ahash's `runtime-rng` feature for them. +[target.'cfg(not(any(target_os = "unknown", target_os = "none")))'.dependencies] +ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] } + +# Not sure what target_os gets set to for sgx, so to be safe always enable runtime-rng for x86_64 +# platforms (assuming LDK isn't being used on embedded x86-64 running directly on metal). +[target.'cfg(target_arch = "x86_64")'.dependencies] +ahash = { version = "0.8", optional = true, default-features = false, features = ["runtime-rng"] } + [dev-dependencies] regex = "1.5.6" diff --git a/lightning/src/chain/onchaintx.rs b/lightning/src/chain/onchaintx.rs index 5fe4bd2c1d2..05d59431fc3 100644 --- a/lightning/src/chain/onchaintx.rs +++ b/lightning/src/chain/onchaintx.rs @@ -686,7 +686,7 @@ impl OnchainTxHandler if let Some(claim_id) = claim_id { if let Some(claim) = self.pending_claim_requests.remove(&claim_id) { for outpoint in claim.outpoints() { - self.claimable_outpoints.remove(&outpoint); + self.claimable_outpoints.remove(outpoint); } } } else { diff --git a/lightning/src/lib.rs b/lightning/src/lib.rs index 77b1f91cbc0..9bc15832870 100644 --- a/lightning/src/lib.rs +++ b/lightning/src/lib.rs @@ -167,9 +167,19 @@ mod io_extras { mod prelude { #[cfg(feature = "hashbrown")] extern crate hashbrown; + #[cfg(feature = "ahash")] + extern crate ahash; pub use alloc::{vec, vec::Vec, string::String, collections::VecDeque, boxed::Box}; + pub use alloc::borrow::ToOwned; + pub use alloc::string::ToString; + + // For no-std builds, we need to use hashbrown, however, by default, it doesn't randomize the + // hashing and is vulnerable to HashDoS attacks. Thus, when not fuzzing, we use its default + // ahash hashing algorithm but randomize, opting to not randomize when fuzzing to avoid false + // positive branch coverage. + #[cfg(not(feature = "hashbrown"))] mod std_hashtables { pub(crate) use std::collections::{HashMap, HashSet, hash_map}; @@ -183,35 +193,85 @@ mod prelude { pub(crate) use std_hashtables::*; #[cfg(feature = "hashbrown")] - mod hashbrown_tables { - pub(crate) use hashbrown::{HashMap, HashSet, hash_map}; + pub(crate) use self::hashbrown::hash_map; + + #[cfg(all(feature = "hashbrown", fuzzing))] + mod nonrandomized_hashbrown { + pub(crate) use hashbrown::{HashMap, HashSet}; pub(crate) type OccupiedHashMapEntry<'a, K, V> = - hashbrown::hash_map::OccupiedEntry<'a, K, V, hash_map::DefaultHashBuilder>; + hashbrown::hash_map::OccupiedEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; pub(crate) type VacantHashMapEntry<'a, K, V> = - hashbrown::hash_map::VacantEntry<'a, K, V, hash_map::DefaultHashBuilder>; + hashbrown::hash_map::VacantEntry<'a, K, V, hashbrown::hash_map::DefaultHashBuilder>; } - #[cfg(feature = "hashbrown")] - pub(crate) use hashbrown_tables::*; + #[cfg(all(feature = "hashbrown", fuzzing))] + pub(crate) use nonrandomized_hashbrown::*; - pub(crate) fn new_hash_map() -> HashMap { HashMap::new() } - pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { - HashMap::with_capacity(cap) - } - pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { - HashMap::from_iter(iter) - } - pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } - pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { - HashSet::with_capacity(cap) + #[cfg(all(feature = "hashbrown", not(fuzzing)))] + mod randomized_hashtables { + use super::*; + use ahash::RandomState; + + pub(crate) type HashMap = hashbrown::HashMap; + pub(crate) type HashSet = hashbrown::HashSet; + + pub(crate) type OccupiedHashMapEntry<'a, K, V> = + hashbrown::hash_map::OccupiedEntry<'a, K, V, RandomState>; + pub(crate) type VacantHashMapEntry<'a, K, V> = + hashbrown::hash_map::VacantEntry<'a, K, V, RandomState>; + + pub(crate) fn new_hash_map() -> HashMap { + HashMap::with_hasher(RandomState::new()) + } + pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { + HashMap::with_capacity_and_hasher(cap, RandomState::new()) + } + pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + let mut res = HashMap::with_capacity_and_hasher(min_size, RandomState::new()); + res.extend(iter); + res + } + + pub(crate) fn new_hash_set() -> HashSet { + HashSet::with_hasher(RandomState::new()) + } + pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { + HashSet::with_capacity_and_hasher(cap, RandomState::new()) + } + pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { + let iter = iter.into_iter(); + let min_size = iter.size_hint().0; + let mut res = HashSet::with_capacity_and_hasher(min_size, RandomState::new()); + res.extend(iter); + res + } } - pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { - HashSet::from_iter(iter) + + #[cfg(any(not(feature = "hashbrown"), fuzzing))] + mod randomized_hashtables { + use super::*; + + pub(crate) fn new_hash_map() -> HashMap { HashMap::new() } + pub(crate) fn hash_map_with_capacity(cap: usize) -> HashMap { + HashMap::with_capacity(cap) + } + pub(crate) fn hash_map_from_iter>(iter: I) -> HashMap { + HashMap::from_iter(iter) + } + + pub(crate) fn new_hash_set() -> HashSet { HashSet::new() } + pub(crate) fn hash_set_with_capacity(cap: usize) -> HashSet { + HashSet::with_capacity(cap) + } + pub(crate) fn hash_set_from_iter>(iter: I) -> HashSet { + HashSet::from_iter(iter) + } } - pub use alloc::borrow::ToOwned; - pub use alloc::string::ToString; + pub(crate) use randomized_hashtables::*; } #[cfg(all(not(ldk_bench), feature = "backtrace", feature = "std", test))] diff --git a/lightning/src/ln/channelmanager.rs b/lightning/src/ln/channelmanager.rs index 4d7be7d92b2..1a4f6035aaa 100644 --- a/lightning/src/ln/channelmanager.rs +++ b/lightning/src/ln/channelmanager.rs @@ -5951,7 +5951,7 @@ where // TODO: Once we can rely on the counterparty_node_id from the // monitor event, this and the outpoint_to_peer map should be removed. let outpoint_to_peer = self.outpoint_to_peer.lock().unwrap(); - match outpoint_to_peer.get(&funding_txo) { + match outpoint_to_peer.get(funding_txo) { Some(cp_id) => cp_id.clone(), None => return, } @@ -5964,7 +5964,7 @@ where peer_state_lock = peer_state_mutex_opt.unwrap().lock().unwrap(); let peer_state = &mut *peer_state_lock; let channel = - if let Some(ChannelPhase::Funded(chan)) = peer_state.channel_by_id.get_mut(&channel_id) { + if let Some(ChannelPhase::Funded(chan)) = peer_state.channel_by_id.get_mut(channel_id) { chan } else { let update_actions = peer_state.monitor_update_blocked_actions @@ -11085,7 +11085,7 @@ where downstream_counterparty_and_funding_outpoint: Some((blocked_node_id, _blocked_channel_outpoint, blocked_channel_id, blocking_action)), .. } = action { - if let Some(blocked_peer_state) = per_peer_state.get(&blocked_node_id) { + if let Some(blocked_peer_state) = per_peer_state.get(blocked_node_id) { log_trace!(logger, "Holding the next revoke_and_ack from {} until the preimage is durably persisted in the inbound edge's ChannelMonitor", blocked_channel_id); diff --git a/lightning/src/routing/router.rs b/lightning/src/routing/router.rs index 4214a1821cb..a429bf1eba9 100644 --- a/lightning/src/routing/router.rs +++ b/lightning/src/routing/router.rs @@ -2566,9 +2566,9 @@ where L::Target: Logger { let mut aggregate_path_contribution_msat = path_value_msat; for (idx, (hop, prev_hop_id)) in hop_iter.zip(prev_hop_iter).enumerate() { - let target = private_hop_key_cache.get(&prev_hop_id).unwrap(); + let target = private_hop_key_cache.get(prev_hop_id).unwrap(); - if let Some(first_channels) = first_hop_targets.get(&target) { + if let Some(first_channels) = first_hop_targets.get(target) { if first_channels.iter().any(|d| d.outbound_scid_alias == Some(hop.short_channel_id)) { log_trace!(logger, "Ignoring route hint with SCID {} (and any previous) due to it being a direct channel of ours.", hop.short_channel_id); @@ -2578,7 +2578,7 @@ where L::Target: Logger { let candidate = network_channels .get(&hop.short_channel_id) - .and_then(|channel| channel.as_directed_to(&target)) + .and_then(|channel| channel.as_directed_to(target)) .map(|(info, _)| CandidateRouteHop::PublicHop(PublicHopCandidate { info, short_channel_id: hop.short_channel_id, @@ -2619,7 +2619,7 @@ where L::Target: Logger { .saturating_add(1); // Searching for a direct channel between last checked hop and first_hop_targets - if let Some(first_channels) = first_hop_targets.get_mut(&target) { + if let Some(first_channels) = first_hop_targets.get_mut(target) { sort_first_hop_channels(first_channels, &used_liquidities, recommended_value_msat, our_node_pubkey); for details in first_channels { diff --git a/lightning/src/routing/scoring.rs b/lightning/src/routing/scoring.rs index ec4921812e9..bb043164a0e 100644 --- a/lightning/src/routing/scoring.rs +++ b/lightning/src/routing/scoring.rs @@ -1330,7 +1330,7 @@ impl>, L: Deref> ScoreLookUp for Probabilistic _ => return 0, }; let source = candidate.source(); - if let Some(penalty) = score_params.manual_node_penalties.get(&target) { + if let Some(penalty) = score_params.manual_node_penalties.get(target) { return *penalty; } @@ -1360,7 +1360,7 @@ impl>, L: Deref> ScoreLookUp for Probabilistic let amount_msat = usage.amount_msat.saturating_add(usage.inflight_htlc_msat); let capacity_msat = usage.effective_capacity.as_msat(); self.channel_liquidities - .get(&scid) + .get(scid) .unwrap_or(&ChannelLiquidity::new(Duration::ZERO)) .as_directed(&source, &target, capacity_msat) .penalty_msat(amount_msat, score_params) From c9e4410a056df1087b1a86dd4f8bce64e81f426e Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Thu, 11 Jan 2024 21:33:53 +0000 Subject: [PATCH 8/9] Fix test variable typos --- lightning/src/ln/functional_tests.rs | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lightning/src/ln/functional_tests.rs b/lightning/src/ln/functional_tests.rs index 821fd1f78bb..4f917aefb97 100644 --- a/lightning/src/ln/functional_tests.rs +++ b/lightning/src/ln/functional_tests.rs @@ -5402,11 +5402,11 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno let as_events = nodes[0].node.get_and_clear_pending_events(); assert_eq!(as_events.len(), if announce_latest { 10 } else { 6 }); - let mut as_failds = new_hash_set(); + let mut as_faileds = new_hash_set(); let mut as_updates = 0; for event in as_events.iter() { if let &Event::PaymentPathFailed { ref payment_hash, ref payment_failed_permanently, ref failure, .. } = event { - assert!(as_failds.insert(*payment_hash)); + assert!(as_faileds.insert(*payment_hash)); if *payment_hash != payment_hash_2 { assert_eq!(*payment_failed_permanently, deliver_last_raa); } else { @@ -5418,21 +5418,21 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno } else if let &Event::PaymentFailed { .. } = event { } else { panic!("Unexpected event"); } } - assert!(as_failds.contains(&payment_hash_1)); - assert!(as_failds.contains(&payment_hash_2)); + assert!(as_faileds.contains(&payment_hash_1)); + assert!(as_faileds.contains(&payment_hash_2)); if announce_latest { - assert!(as_failds.contains(&payment_hash_3)); - assert!(as_failds.contains(&payment_hash_5)); + assert!(as_faileds.contains(&payment_hash_3)); + assert!(as_faileds.contains(&payment_hash_5)); } - assert!(as_failds.contains(&payment_hash_6)); + assert!(as_faileds.contains(&payment_hash_6)); let bs_events = nodes[1].node.get_and_clear_pending_events(); assert_eq!(bs_events.len(), if announce_latest { 8 } else { 6 }); - let mut bs_failds = new_hash_set(); + let mut bs_faileds = new_hash_set(); let mut bs_updates = 0; for event in bs_events.iter() { if let &Event::PaymentPathFailed { ref payment_hash, ref payment_failed_permanently, ref failure, .. } = event { - assert!(bs_failds.insert(*payment_hash)); + assert!(bs_faileds.insert(*payment_hash)); if *payment_hash != payment_hash_1 && *payment_hash != payment_hash_5 { assert_eq!(*payment_failed_permanently, deliver_last_raa); } else { @@ -5444,12 +5444,12 @@ fn do_test_fail_backwards_unrevoked_remote_announce(deliver_last_raa: bool, anno } else if let &Event::PaymentFailed { .. } = event { } else { panic!("Unexpected event"); } } - assert!(bs_failds.contains(&payment_hash_1)); - assert!(bs_failds.contains(&payment_hash_2)); + assert!(bs_faileds.contains(&payment_hash_1)); + assert!(bs_faileds.contains(&payment_hash_2)); if announce_latest { - assert!(bs_failds.contains(&payment_hash_4)); + assert!(bs_faileds.contains(&payment_hash_4)); } - assert!(bs_failds.contains(&payment_hash_5)); + assert!(bs_faileds.contains(&payment_hash_5)); // For each HTLC which was not failed-back by normal process (ie deliver_last_raa), we should // get a NetworkUpdate. A should have gotten 4 HTLCs which were failed-back due to From 7377cc99c7ae959a5d33fd170ff5b2b64c725fd2 Mon Sep 17 00:00:00 2001 From: Matt Corallo Date: Wed, 31 Jan 2024 19:48:44 +0000 Subject: [PATCH 9/9] [fuzz] Fix slice copy in `peer_crypt_target` This has apparently been broken for quite some time...I guess `peer_crypt_target` doesn't get much CPU anymore. --- fuzz/src/peer_crypt.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fuzz/src/peer_crypt.rs b/fuzz/src/peer_crypt.rs index 3acf4d664f6..41d8c093606 100644 --- a/fuzz/src/peer_crypt.rs +++ b/fuzz/src/peer_crypt.rs @@ -83,7 +83,7 @@ pub fn do_test(data: &[u8]) { Ok(len) => len, Err(_) => return, }; - buf.copy_from_slice(&get_slice!(len as usize + 16)); + buf[..len as usize + 16].copy_from_slice(&get_slice!(len as usize + 16)); match crypter.decrypt_message(&mut buf[..len as usize + 16]) { Ok(_) => {}, Err(_) => return,