Massive update, merge recent interactive tx branch (121)

.github/workflows/build.yml

@@ -18,19 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         platform: [ ubuntu-latest, windows-latest, macos-latest ]
-        toolchain: [ stable, beta ]
-        include:
-          - toolchain: stable
-            platform: ubuntu-latest
-          # 1.48.0 is the MSRV for all crates except lightning-transaction-sync and Win/Mac
-          - toolchain: 1.48.0
-            platform: ubuntu-latest
-          # Windows requires 1.49.0 because that's the MSRV for supported Tokio
-          - toolchain: 1.49.0
-            platform: windows-latest
-          # MacOS-latest requires 1.54.0 because that's what's required for linking to work properly
-          - toolchain: 1.54.0
-            platform: macos-latest
+        toolchain: [ stable, beta, 1.63.0 ] # 1.63.0 is the MSRV for all crates.
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Checkout source code
@@ -44,11 +32,16 @@ jobs:
         run: |
           rustup target add thumbv7m-none-eabi
           sudo apt-get -y install gcc-arm-none-eabi
+      - name: Check for unknown cfg tags
+        run: ci/check-cfg-flags.py
       - name: shellcheck the CI script
         if: "matrix.platform == 'ubuntu-latest'"
         run: |
           sudo apt-get -y install shellcheck
           shellcheck ci/ci-tests.sh
+      - name: Set RUSTFLAGS to deny warnings
+        if: "matrix.toolchain == '1.63.0'"
+        run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV"
       - name: Run CI script
         shell: bash # Default on Winblows is powershell
         run: CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh
@@ -168,13 +161,13 @@ jobs:
         run: |
           cargo check --release
           cargo check --no-default-features --features=no-std --release
-          cargo check --no-default-features --features=futures --release
+          cargo check --no-default-features --features=futures,std --release
           cargo doc --release
       - name: Run cargo check for Taproot build.
         run: |
           cargo check --release
           cargo check --no-default-features --features=no-std --release
-          cargo check --no-default-features --features=futures --release
+          cargo check --no-default-features --features=futures,std --release
           cargo doc --release
         env:
           RUSTFLAGS: '--cfg=taproot'
@@ -183,7 +176,7 @@ jobs:
   fuzz:
     runs-on: ubuntu-latest
     env:
-      TOOLCHAIN: 1.58
+      TOOLCHAIN: 1.63
     steps:
       - name: Checkout source code
         uses: actions/checkout@v3
@@ -195,6 +188,10 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get -y install build-essential binutils-dev libunwind-dev
+      - name: Pin the regex dependency
+        run: |
+          cd fuzz && cargo update -p regex --precise "1.9.6" --verbose && cd ..
+          cd lightning-invoice/fuzz && cargo update -p regex --precise "1.9.6" --verbose
       - name: Sanity check fuzz targets on Rust ${{ env.TOOLCHAIN }}
         run: cd fuzz && RUSTFLAGS="--cfg=fuzzing" cargo test --verbose --color always
       - name: Run fuzzers
@@ -219,3 +216,20 @@ jobs:
       - name: Run default clippy linting
         run: |
           cargo clippy -- -Aclippy::erasing_op -Aclippy::never_loop -Aclippy::if_same_then_else -Dclippy::try_err
+
+  rustfmt:
+    runs-on: ubuntu-latest
+    env:
+      TOOLCHAIN: 1.63.0
+    steps:
+      - name: Checkout source code
+        uses: actions/checkout@v3
+      - name: Install Rust ${{ env.TOOLCHAIN }} toolchain
+        run: |
+          curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }}
+          rustup override set ${{ env.TOOLCHAIN }}
+      - name: Install rustfmt
+        run: |
+          rustup component add rustfmt
+      - name: Run rustfmt checks
+        run: ci/rustfmt.sh

.gitignore

@@ -13,3 +13,4 @@ lightning-rapid-gossip-sync/res/full_graph.lngossip
 lightning-custom-message/target
 lightning-transaction-sync/target
 no-std-check/target
+msrv-no-dev-deps-check/target

CHANGELOG.md

@@ -1,3 +1,189 @@
+# 0.0.121 - Jan 22, 2024 - "Unwraps are Bad"
+
+## Bug Fixes
+ * Fix a deadlock when calling `batch_funding_transaction_generated` with
+   invalid input (#2841).
+
+## Security
+0.0.121 fixes a denial-of-service vulnerability which is reachable from
+untrusted input from peers in rare cases if we have a public channel or in
+common cases if `P2PGossipSync` is used.
+ * A peer that failed to complete its handshake would cause a reachable
+   `unwrap` in LDK since 0.0.119 when LDK attempts to broadcast gossip to all
+   peers (#2842).
+
+In total, this release features 4 files changed, 52 insertions, 10
+deletions in 4 commits from 2 authors, in alphabetical order:
+ * Jeffrey Czyz
+ * Matt Corallo
+
+# 0.0.120 - Jan 17, 2024 - "Unblinded Fuzzers"
+
+## API Updates
+ * The `PeerManager` bound on `UtxoLookup` was removed entirely. This enables
+   use of `UtxoLookup` in cases broken in 0.0.119 by #2773 (#2822).
+ * LDK now exposes and fully implements the route blinding feature (#2812).
+ * The `lightning-transaction-sync` crate no longer relies on system time
+   without the `time` feature (#2799, #2817).
+ * `lightning::onion_message`'s module layout has changed (#2821).
+ * `Event::ChannelClosed` now includes the `channel_funding_txo` (#2800).
+ * `CandidateRouteHop` variants were destructured into individual structs,
+   hiding some fields which were not generally consumable (#2802).
+
+## Bug Fixes
+ * Fixed a rare issue where `lightning-net-tokio` may not fully flush its send
+   buffer, leading to connection hangs (#2832).
+ * Fixed a panic which may occur when connecting to a peer if we opened a second
+   channel with that peer while they were disconnected (#2808).
+ * Retries for a payment which previously failed in a blinded path will now
+   always use an alternative blinded path (#2818).
+ * `Feature`'s `Eq` and `Hash` implementation now ignore dummy bytes (#2808).
+ * Some missing `DiscardFunding` or `ChannelClosed` events are now generated in
+   rare funding-related failures (#2809).
+ * Fixed a privacy issue in blinded path generation where the real
+   `cltv_expiry_delta` would be exposed to senders (#2831).
+
+## Security
+0.0.120 fixes a denial-of-service vulnerability which is reachable from
+untrusted input from peers if the `UserConfig::manually_accept_inbound_channels`
+option is enabled.
+ * A peer that sent an `open_channel` message with the `channel_type` field
+   unfilled would trigger a reachable `unwrap` since LDK 0.0.117 (#2808).
+ * In protocols where a funding output is shared with our counterparty before
+   it is given to LDK, a malicious peer could have caused a reachable panic
+   by reusing the same funding info in (#2809).
+
+In total, this release features 67 files changed, 3016 insertions, 2473
+deletions in 79 commits from 9 authors, in alphabetical order:
+ * Elias Rohrer
+ * Jeffrey Czyz
+ * José A.P
+ * Matt Corallo
+ * Tibo-lg
+ * Valentine Wallace
+ * benthecarman
+ * optout
+ * shuoer86
+
+# 0.0.119 - Dec 15, 2023 - "Spring Cleaning for Christmas"
+
+## API Updates
+ * The LDK crate ecosystem MSRV has been increased to 1.63 (#2681).
+ * The `bitcoin` dependency has been updated to version 0.30 (#2740).
+ * `lightning-invoice::payment::*` have been replaced with parameter generation
+   via `payment_parameters_from[_zero_amount]_invoice` (#2727).
+ * `{CoinSelection,Wallet}Source::sign_tx` are now `sign_psbt`, providing more
+   information, incl spent outputs, about the transaction being signed (#2775).
+ * Logger `Record`s now include `channel_id` and `peer_id` fields. These are
+   opportunistically filled in when a log record is specific to a given channel
+   and/or peer, and may occasionally be spuriously empty (#2314).
+ * When handling send or reply onion messages (e.g. for BOLT12 payments), a new
+   `Event::ConnectionNeeded` may be raised, indicating a direct connection
+   should be made to a payee or an introduction point. This event is expected to
+   be removed once onion message forwarding is widespread in the network (#2723)
+ * Scoring data decay now happens via `ScoreUpDate::time_passed`, called from
+   `lightning-background-processor`. `process_events_async` now takes a new
+   time-fetch function, and `ScoreUpDate` methods now take the current time as a
+   `Duration` argument. This avoids fetching time during pathfinding (#2656).
+ * Receiving payments to multi-hop blinded paths is now supported (#2688).
+ * `MessageRouter` and `Router` now feature methods to generate blinded paths to
+   the local node for incoming messages and payments. `Router` now extends
+   `MessageRouter`, and both are used in `ChannelManager` when processing or
+   creating BOLT12 structures to generate multi-hop blinded paths (#1781).
+ * `lightning-transaction-sync` now supports Electrum-based sync (#2685).
+ * `Confirm::get_relevant_txids` now returns the height at which a transaction
+   was confirmed. This can be used to assist in reorg detection (#2685).
+ * `ConfirmationTarget::MaxAllowedNonAnchorChannelRemoteFee` has been removed.
+   Non-anchor channel feerates are bounded indirectly through
+   `ChannelConfig::max_dust_htlc_exposure` (#2696).
+ * `lightning-invoice` `Description`s now rely on `UntrustedString` for
+   sanitization (#2730).
+ * `ScoreLookUp::channel_penalty_msat` now uses `CandidateRouteHop` (#2551).
+ * The `EcdsaChannelSigner` trait was moved to `lightning::sign::ecdsa` (#2512).
+ * `SignerProvider::get_destination_script` now takes `channel_keys_id` (#2744)
+ * `SpendableOutputDescriptor::StaticOutput` now has `channel_keys_id` (#2749).
+ * `EcdsaChannelSigner::sign_counterparty_commitment` now takes HTLC preimages
+   for both inbound and outbound HTLCs (#2753).
+ * `ClaimedHTLC` now includes a `counterparty_skimmed_fee_msat` field (#2715).
+ * `peel_payment_onion` was added to decode an encrypted onion for a payment
+   without receiving an HTLC. This allows for stateless verification of if a
+   theoretical payment would be accepted prior to receipt (#2700).
+ * `create_payment_onion` was added to construct an encrypted onion for a
+   payment path without sending an HTLC immediately (#2677).
+ * Various keys used in channels are now wrapped to provide type-safety for
+   specific usages of the keys (#2675).
+ * `TaggedHash` now includes the raw `tag` and `merkle_root` (#2687).
+ * `Offer::is_expired_no_std` was added (#2689).
+ * `PaymentPurpose::preimage()` was added (#2768).
+ * `temporary_channel_id` can now be specified in `create_channel` (#2699).
+ * Wire definitions for splicing messages were added (#2544).
+ * Various `lightning-invoice` structs now impl `Display`, now have pub fields,
+   or impl `From` (#2730).
+ * The `Hash` trait is now implemented for more structs, incl P2P msgs (#2716).
+
+## Performance Improvements
+ * Memory allocations (though not memory usage) have been substantially reduced,
+   meaning less overhead and hopefully less memory fragmentation (#2708, #2779).
+
+## Bug Fixes
+ * Since 0.0.117, calling `close_channel*` on a channel which has not yet been
+   funded would previously result in an infinite loop and hang (#2760).
+ * Since 0.0.116, sending payments requiring data in the onion for the recipient
+   which was too large for the onion may have caused corruption which resulted
+   in payment failure (#2752).
+ * Cooperative channel closure on channels with remaining output HTLCs may have
+   spuriously force-closed (#2529).
+ * In LDK versions 0.0.116 through 0.0.118, in rare cases where skimmed fees are
+   present on shutdown the `ChannelManager` may fail to deserialize (#2735).
+ * `ChannelConfig::max_dust_exposure` values which, converted to absolute fees,
+   exceeded 2^63 - 1 would result in an overflow and could lead to spurious
+   payment failures or channel closures (#2722).
+ * In cases where LDK is operating with provably-stale state, it panics to
+   avoid funds loss. This may not have happened in cases where LDK was behind
+   only exactly one state, leading instead to a revoked broadcast and funds
+   loss (#2721).
+ * Fixed a bug where decoding `Txid`s from Bitcoin Core JSON-RPC responses using
+   `lightning-block-sync` would not properly byte-swap the hash. Note that LDK
+   does not use this API internally (#2796).
+
+## Backwards Compatibility
+ * `ChannelManager`s written with LDK 0.0.119 are no longer readable by versions
+   of LDK prior to 0.0.113. Users wishing to downgrade to LDK 0.0.112 or before
+   can read an 0.0.119-serialized `ChannelManager` with a version of LDK from
+   0.0.113 to 0.0.118, re-serialize it, and then downgrade (#2708).
+ * Nodes that upgrade to 0.0.119 and subsequently downgrade after receiving a
+   payment to a blinded path may leak recipient information if one or more of
+   those HTLCs later fails (#2688).
+ * Similarly, forwarding a blinded HTLC and subsequently downgrading to an LDK
+   version prior to 0.0.119 may result in leaking the path information to the
+   payment sender (#2540).
+
+In total, this release features 148 files changed, 13780 insertions, 6279
+deletions in 280 commits from 22 authors, in alphabetical order:
+ * Arik Sosman
+ * Chris Waterson
+ * Elias Rohrer
+ * Evan Feenstra
+ * Gursharan Singh
+ * Jeffrey Czyz
+ * John Cantrell
+ * Lalitmohansharma1
+ * Matt Corallo
+ * Matthew Rheaume
+ * Orbital
+ * Rachel Malonson
+ * Valentine Wallace
+ * Willem Van Lint
+ * Wilmer Paulino
+ * alexanderwiederin
+ * benthecarman
+ * henghonglee
+ * jbesraa
+ * olegkubrakov
+ * optout
+ * shaavan
+
+
 # 0.0.118 - Oct 23, 2023 - "Just the Twelve Sinks"
 
 ## API Updates

CONTRIBUTING.md

@@ -88,7 +88,7 @@ be covered by functional tests.
 When refactoring, structure your PR to make it easy to review and don't
 hesitate to split it into multiple small, focused PRs.
 
-The Minimum Supported Rust Version (MSRV) currently is 1.48.0 (enforced by
+The Minimum Supported Rust Version (MSRV) currently is 1.63.0 (enforced by
 our GitHub Actions). We support reading serialized LDK objects written by any
 version of LDK 0.0.99 and above. We support LDK versions 0.0.113 and above
 reading serialized LDK objects written by modern LDK. Any expected issues with
@@ -120,7 +120,15 @@ Coding Conventions
 ------------------
 
 Use tabs. If you want to align lines, use spaces. Any desired alignment should
-display fine at any tab-length display setting.
+display fine at any tab-length display setting. We use `rustfmt` to establish
+uniform coding standards throughout the codebase. Please run
+
+```bash
+./ci/rustfmt.sh
+```
+
+before committing and pushing any changes, as compliance will also be checked
+and enforced by our CI scripts.
 
 Our CI enforces [clippy's](https://github.com/rust-lang/rust-clippy) default
 linting
@@ -148,7 +156,7 @@ Security
 --------
 
 Security is the primary focus of `rust-lightning`; disclosure of security
-vulnerabilites helps prevent user loss of funds. If you believe a vulnerability
+vulnerabilities helps prevent user loss of funds. If you believe a vulnerability
 may affect other Lightning implementations, please inform them.
 
 You can find further information on submitting (possible) vulnerabilities in the

Cargo.toml

@@ -1,4 +1,5 @@
 [workspace]
+resolver = "2"
 
 members = [
     "lightning",
@@ -7,12 +8,13 @@ members = [
     "lightning-net-tokio",
     "lightning-persister",
     "lightning-background-processor",
-    "lightning-rapid-gossip-sync"
+    "lightning-rapid-gossip-sync",
+    "lightning-custom-message",
+    "lightning-transaction-sync",
+    "possiblyrandom",
 ]
 
 exclude = [
-    "lightning-custom-message",
-    "lightning-transaction-sync",
     "no-std-check",
     "msrv-no-dev-deps-check",
     "bench",
@@ -37,3 +39,6 @@ lto = "off"
 opt-level = 3
 lto = true
 panic = "abort"
+
+[patch.crates-io.possiblyrandom]
+path = "possiblyrandom"

README.md

@@ -4,8 +4,9 @@ Rust-Lightning
 [![Crate](https://img.shields.io/crates/v/lightning.svg?logo=rust)](https://crates.io/crates/lightning)
 [![Documentation](https://img.shields.io/static/v1?logo=read-the-docs&label=docs.rs&message=lightning&color=informational)](https://docs.rs/lightning/)
 [![Safety Dance](https://img.shields.io/badge/unsafe-forbidden-success.svg)](https://github.com/rust-secure-code/safety-dance/)
+[![Security Audit](https://github.com/lightningdevkit/rust-lightning/actions/workflows/audit.yml/badge.svg)](https://github.com/lightningdevkit/rust-lightning/actions/workflows/audit.yml)
 
-[LDK](https://lightningdevkit.org)/`rust-lightning` is a highly performant and flexible 
+[LDK](https://lightningdevkit.org)/`rust-lightning` is a highly performant and flexible
 implementation of the Lightning Network protocol.
 
 The primary crate, `lightning`, is runtime-agnostic. Data persistence, chain interactions,

bench/Cargo.toml

@@ -2,7 +2,7 @@
 name = "lightning-bench"
 version = "0.0.1"
 authors = ["Matt Corallo"]
-edition = "2018"
+edition = "2021"
 
 [[bench]]
 name = "bench"

bench/benches/bench.rs

@@ -21,5 +21,6 @@ criterion_group!(benches,
 	lightning_persister::fs_store::bench::bench_sends,
 	lightning_rapid_gossip_sync::bench::bench_reading_full_graph_from_file,
 	lightning::routing::gossip::benches::read_network_graph,
-	lightning::routing::gossip::benches::write_network_graph);
+	lightning::routing::gossip::benches::write_network_graph,
+	lightning::routing::scoring::benches::decay_100k_channel_bounds);
 criterion_main!(benches);