From 203c47fe971bc9182d7b9ee1d8128a8d7b984150 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Mon, 31 Mar 2025 15:12:01 +1030
Subject: [PATCH 01/12] trace: minimal fix to avoid crash when > 128 traces
 active.

chanbackup with many peers can do more than 128 concurrent rpc commands.
autoclean is the other plugin which can do many requests at once, so I
expect a similar issue there.

I tested this by rebuilding with `MAX_ACTIVE_SPANS` 1, which autoclean
tests triggered immediately.

The real fix is probably to use a hash table with a large initial size.

```
Mar 24 06:30:45 mlbb2 sh[28000]: chanbackup: common/trace.c:190: trace_span_slot: Assertion `s' failed.
Mar 24 06:30:45 mlbb2 sh[28000]: chanbackup: FATAL SIGNAL 6 (version v25.02)
Mar 24 06:30:45 mlbb2 sh[28000]: 0x5575232bac4f send_backtrace
Mar 24 06:30:45 mlbb2 sh[28000]:         common/daemon.c:33
Mar 24 06:30:45 mlbb2 sh[28000]: 0x5575232baceb crashdump
Mar 24 06:30:45 mlbb2 sh[28000]:         common/daemon.c:78
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958cd851f ???
Mar 24 06:30:45 mlbb2 sh[28000]:         ./signal/../sysdeps/unix/sysv/linux/x86_64/libc_sigaction.c:0
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958d2c9fc __pthread_kill_implementation
Mar 24 06:30:45 mlbb2 sh[28000]:         ./nptl/pthread_kill.c:44
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958d2c9fc __pthread_kill_internal
Mar 24 06:30:45 mlbb2 sh[28000]:         ./nptl/pthread_kill.c:78
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958d2c9fc __GI___pthread_kill
Mar 24 06:30:45 mlbb2 sh[28000]:         ./nptl/pthread_kill.c:89
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958cd8475 __GI_raise
Mar 24 06:30:45 mlbb2 sh[28000]:         ../sysdeps/posix/raise.c:26
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958cbe7f2 __GI_abort
Mar 24 06:30:45 mlbb2 sh[28000]:         ./stdlib/abort.c:79
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958cbe71a __assert_fail_base
Mar 24 06:30:45 mlbb2 sh[28000]:         ./assert/assert.c:94
Mar 24 06:30:45 mlbb2 sh[28000]: 0x7f2958ccfe95 __GI___assert_fail
Mar 24 06:30:45 mlbb2 sh[28000]:         ./assert/assert.c:103
Mar 24 06:30:45 mlbb2 sh[28000]: 0x5575232ab7fa trace_span_slot
Mar 24 06:30:45 mlbb2 sh[28000]:         common/trace.c:190
Mar 24 06:30:45 mlbb2 sh[28000]: 0x5575232abc9f trace_span_start
Mar 24 06:30:45 mlbb2 sh[28000]:         common/trace.c:267
Mar 24 06:30:45 mlbb2 sh[28000]: 0x5575232a7c34 send_outreq
Mar 24 06:30:45 mlbb2 sh[28000]:         plugins/libplugin.c:1112
```

Changelog-Fixed: autoclean/chanbackup: fixed tracepoint crash on large number of requests.
Fixes: https://github.com/ElementsProject/lightning/issues/8177
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 common/trace.c | 31 +++++++++++++++++++++++++++++--
 1 file changed, 29 insertions(+), 2 deletions(-)

diff --git a/common/trace.c b/common/trace.c
index 9f2ce2b11dbe..9e7a01c863d3 100644
--- a/common/trace.c
+++ b/common/trace.c
@@ -10,9 +10,10 @@
 #include <common/trace.h>
 #include <sodium/randombytes.h>
 #include <stdio.h>
+#include <unistd.h>
 
 #if HAVE_USDT
-#include <sys/sdt.h>
+  #include <sys/sdt.h>
 
 #define MAX_ACTIVE_SPANS 128
 
@@ -40,6 +41,7 @@
 #endif
 
 const char *trace_service_name = "lightningd";
+static bool disable_trace = false;
 
 struct span_tag {
 	char *name, *value;
@@ -176,6 +178,9 @@ static struct span *trace_span_find(size_t key)
 	return NULL;
 }
 
+/* FIXME: Forward declaration for minimal patch size */
+static void trace_span_clear(struct span *s);
+
 /**
  * Find an empty slot for a new span.
  */
@@ -187,7 +192,13 @@ static struct span *trace_span_slot(void)
 
 	/* Might end up here if we have more than MAX_ACTIVE_SPANS
 	 * concurrent spans. */
-	assert(s);
+	if (!s) {
+		fprintf(stderr, "%u: out of spans, disabling tracing\n", getpid());
+		for (size_t i = 0; i < MAX_ACTIVE_SPANS; i++)
+			trace_span_clear(&active_spans[i]);
+		disable_trace = true;
+		return NULL;
+	}
 	assert(s->parent == NULL);
 
 	/* Be extra careful not to create cycles. If we return the
@@ -260,11 +271,15 @@ void trace_span_start(const char *name, const void *key)
 	size_t numkey = trace_key(key);
 	struct timeabs now = time_now();
 
+	if (disable_trace)
+		return;
 	trace_init();
 	trace_check_tree();
 
 	assert(trace_span_find(numkey) == NULL);
 	struct span *s = trace_span_slot();
+	if (!s)
+		return;
 	s->key = numkey;
 	randombytes_buf(s->id, SPAN_ID_SIZE);
 	s->start_time = (now.ts.tv_sec * 1000000) + now.ts.tv_nsec / 1000;
@@ -293,6 +308,9 @@ void trace_span_remote(u8 trace_id[TRACE_ID_SIZE], u8 span_id[SPAN_ID_SIZE])
 
 void trace_span_end(const void *key)
 {
+	if (disable_trace)
+		return;
+
 	size_t numkey = trace_key(key);
 	struct span *s = trace_span_find(numkey);
 	assert(s && "Span to end not found");
@@ -323,6 +341,9 @@ void trace_span_end(const void *key)
 
 void trace_span_tag(const void *key, const char *name, const char *value)
 {
+	if (disable_trace)
+		return;
+
 	size_t numkey = trace_key(key);
 	struct span *span = trace_span_find(numkey);
 	assert(span);
@@ -341,6 +362,9 @@ void trace_span_tag(const void *key, const char *name, const char *value)
 
 void trace_span_suspend_(const void *key, const char *lbl)
 {
+	if (disable_trace)
+		return;
+
 	size_t numkey = trace_key(key);
 	struct span *span = trace_span_find(numkey);
 	TRACE_DBG("Suspending span %s (%zu)\n", current->name, current->key);
@@ -351,6 +375,9 @@ void trace_span_suspend_(const void *key, const char *lbl)
 
 void trace_span_resume_(const void *key, const char *lbl)
 {
+	if (disable_trace)
+		return;
+
 	size_t numkey = trace_key(key);
 	current = trace_span_find(numkey);
 	TRACE_DBG("Resuming span %s (%zu)\n", current->name, current->key);

From 2289b0fa705f94a0ca94c630449101a7924c5d75 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Thu, 27 Mar 2025 12:05:38 +1030
Subject: [PATCH 02/12] lightningd: fix anchorspend HTLC deadline logic.

It's not the *outgoing* HTLC which sets the deadline, it's the incoming.

Reported-by: @whitslack
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-Fixed: Protocol: Egregious anchor fee paid for unilateral close txs due to HTLC timeouts; it's not as urgent as our code made out!
---
 lightningd/anchorspend.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/lightningd/anchorspend.c b/lightningd/anchorspend.c
index aa64c4316e3f..9531c61d7d84 100644
--- a/lightningd/anchorspend.c
+++ b/lightningd/anchorspend.c
@@ -204,7 +204,12 @@ struct anchor_details *create_anchor_details(const tal_t *ctx,
 			continue;
 
 		v.msat = hout->msat;
-		v.block = hout->cltv_expiry;
+		/* Our real deadline here is the INCOMING htlc.  If it's us, use the default so we don't leak
+		 * too much information about it. */
+		if (hout->in)
+			v.block = hout->in->cltv_expiry;
+		else
+			v.block = hout->cltv_expiry + ld->config.cltv_expiry_delta;
 		v.important = true;
 		tal_arr_expand(&adet->vals, v);
 	}

From ef355b47a8af25733dc6d24d3fed96b0596bb32d Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Mon, 31 Mar 2025 12:06:55 +1030
Subject: [PATCH 03/12] lightningd: allow bitcoind_getrawblockbyheight callback
 to free call struct.

Use the indirect-free trick, otherwise this can happen:

```
2025-03-28T10:46:16.437Z BROKEN lightningd: FATAL SIGNAL 6 (version v25.02)
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: common/daemon.c:41 (send_backtrace) 0x6447525af68c
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: common/daemon.c:78 (crashdump) 0x6447525af6db
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ./signal/../sysdeps/unix/sysv/linux/x86_64/libc_sigaction.c:0 ((null)) 0x7783e2c4532f
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ./nptl/pthread_kill.c:44 (__pthread_kill_implementation) 0x7783e2c9eb2c
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ./nptl/pthread_kill.c:78 (__pthread_kill_internal) 0x7783e2c9eb2c
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ./nptl/pthread_kill.c:89 (__GI___pthread_kill) 0x7783e2c9eb2c
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ../sysdeps/posix/raise.c:26 (__GI_raise) 0x7783e2c4527d
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ./stdlib/abort.c:79 (__GI_abort) 0x7783e2c288fe
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ccan/ccan/tal/tal.c:95 (call_error) 0x644752675535
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ccan/ccan/tal/tal.c:169 (check_bounds) 0x6447526755de
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ccan/ccan/tal/tal.c:180 (to_tal_hdr) 0x644752675618
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: ccan/ccan/tal/tal.c:525 (tal_free) 0x644752676001
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: lightningd/bitcoind.c:509 (getrawblockbyheight_callback) 0x64475252c01b
2025-03-28T10:46:16.437Z BROKEN lightningd: backtrace: lightningd/plugin.c:661 (plugin_response_handle) 0x64475257be0a
```

Changelog-Fixed: lightningd: occasional crash on bitcoind callback.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 lightningd/bitcoind.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lightningd/bitcoind.c b/lightningd/bitcoind.c
index 8bae8d722171..bab129cbdd39 100644
--- a/lightningd/bitcoind.c
+++ b/lightningd/bitcoind.c
@@ -477,9 +477,15 @@ getrawblockbyheight_callback(const char *buf, const jsmntok_t *toks,
 	const char *block_str, *err;
 	struct bitcoin_blkid blkid;
 	struct bitcoin_block *blk;
+	const tal_t *ctx;
 	trace_span_resume(call);
 	trace_span_end(call);
 
+	/* Callback may free parent of call, so steal onto context to
+	 * free if it doesn't */
+	ctx = tal(NULL, char);
+	tal_steal(ctx, call);
+
 	/* If block hash is `null`, this means not found! Call the callback
 	 * with NULL values. */
 	err = json_scan(tmpctx, buf, toks, "{result:{blockhash:null}}");
@@ -506,7 +512,7 @@ getrawblockbyheight_callback(const char *buf, const jsmntok_t *toks,
 	call->cb(call->bitcoind, call->height, &blkid, blk, call->cb_arg);
 
 clean:
-	tal_free(call);
+	tal_free(ctx);
 }
 
 void bitcoind_getrawblockbyheight_(const tal_t *ctx,

From 31693fb900b984933916afcddff4738ba8648ae9 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Mon, 31 Mar 2025 11:25:19 +1030
Subject: [PATCH 04/12] lightningd: make sure we register all addresses at
 opening if peer doesn't support OPT_SHUTDOWN_ANYSEGWIT.

We select the close key index at opening time, but the non-DF code didn't correctly register the
address as possibly used for P2WPKH for older nodes.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-Fixed: wallet: we could miss our own returned outputs on mutual closes if peer doesn't support option_shutdown_anysegwit (you will still need to rescan after update, if this happened to you!)
Reported-by: Grubles
---
 lightningd/opening_control.c | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/lightningd/opening_control.c b/lightningd/opening_control.c
index 9b574beda882..5444e05d0236 100644
--- a/lightningd/opening_control.c
+++ b/lightningd/opening_control.c
@@ -122,8 +122,14 @@ wallet_commit_channel(struct lightningd *ld,
 	/* FIXME: P2TR for elements! */
 	if (chainparams->is_elements)
 		addrtype = ADDR_BECH32;
-	else
+	else if (feature_negotiated(ld->our_features,
+				    uc->peer->their_features,
+				    OPT_SHUTDOWN_ANYSEGWIT))
 		addrtype = ADDR_P2TR;
+	else
+		/* They *may* update to OPT_SHUTDOWN_ANYSEGWIT by the
+		 * time we close, so be prepared for both. */
+		addrtype = ADDR_ALL;
 
 	/* Get a key to use for closing outputs from this tx */
 	final_key_idx = wallet_get_newindex(ld, addrtype);

From c5b5080667f5830713c477a2dfbbf677be6aae93 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Tue, 1 Apr 2025 14:01:11 +1030
Subject: [PATCH 05/12] lightningd: don't spam logs on dangling outgoing HTLCs.

We pre-close incoming under some circumstances, so this does happen (it
didn't when this code was written).  Don't walk all the HTLCs complaining
about them in this case, and don't freak out.

Changelog-Fixed: lightningd: incorrect spamming of log and potential crash on testnet case of duplicate HTLCs and slow closing.
Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Fixes: https://github.com/ElementsProject/lightning/issues/8176
---
 lightningd/peer_htlcs.c | 36 ------------------------------------
 1 file changed, 36 deletions(-)

diff --git a/lightningd/peer_htlcs.c b/lightningd/peer_htlcs.c
index 8bfb9afa42f1..8d06d891d13f 100644
--- a/lightningd/peer_htlcs.c
+++ b/lightningd/peer_htlcs.c
@@ -1704,41 +1704,6 @@ static void check_already_failed(const struct channel *channel, struct htlc_out
 	}
 }
 
-/* This case searches harder to see if there are any incoming HTLCs */
-static void fail_dangling_htlc_in(struct lightningd *ld,
-				  const struct sha256 *payment_hash)
-{
-	struct htlc_in *hin;
-	struct htlc_in_map_iter ini;
-
-	for (hin = htlc_in_map_first(ld->htlcs_in, &ini);
-	     hin;
-	     hin = htlc_in_map_next(ld->htlcs_in, &ini)) {
-		if (!sha256_eq(&hin->payment_hash, payment_hash))
-			continue;
-		if (hin->badonion) {
-			log_broken(hin->key.channel->log,
-				   "htlc %"PRIu64" already failed with badonion",
-				   hin->key.id);
-		} else if (hin->preimage) {
-			log_broken(hin->key.channel->log,
-				   "htlc %"PRIu64" already succeeded with preimage",
-				   hin->key.id);
-		} else if (hin->failonion) {
-			log_broken(hin->key.channel->log,
-				   "htlc %"PRIu64" already failed with failonion %s",
-				   hin->key.id,
-				   tal_hex(tmpctx, hin->failonion->contents));
-		} else {
-			log_broken(hin->key.channel->log,
-				   "htlc %"PRIu64" has matching hash: failing",
-				   hin->key.id);
-			local_fail_in_htlc(hin,
-					   take(towire_permanent_channel_failure(NULL)));
-		}
-	}
-}
-
 void onchain_failed_our_htlc(const struct channel *channel,
 			     const struct htlc_stub *htlc,
 			     const char *why,
@@ -1841,7 +1806,6 @@ void onchain_failed_our_htlc(const struct channel *channel,
 		/* Immediate corruption sanity check if this happens */
 		htable_check(&ld->htlcs_out->raw, "onchain_failed_our_htlc out");
 		htable_check(&ld->htlcs_in->raw, "onchain_failed_our_htlc in");
-		fail_dangling_htlc_in(ld, &hout->payment_hash);
 	}
 }
 

From 4e78d196a15c2a0aabab377c526d33a180dbb810 Mon Sep 17 00:00:00 2001
From: ShahanaFarooqui <shahana.farooqui@gmail.com>
Date: Mon, 6 Jan 2025 13:51:21 -0800
Subject: [PATCH 06/12] ci: Lock Ubuntu version on 22.04

These workflows are failing due to `ubuntu-latest` being updated from version `22.04` to `24.04`.

Reference: https://github.com/actions/runner-images/issues/10636

Changelog-None.
---
 .github/workflows/bsd.yml             | 2 +-
 .github/workflows/crate-io.yml        | 2 +-
 .github/workflows/docker-release.yml  | 2 +-
 .github/workflows/rdme-docs-sync.yml  | 2 +-
 .github/workflows/readme-rpc-sync.yml | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/bsd.yml b/.github/workflows/bsd.yml
index 206acd11f384..ed8411896f79 100644
--- a/.github/workflows/bsd.yml
+++ b/.github/workflows/bsd.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   testfreebsd:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     name: Build and test on FreeBSD
     timeout-minutes: 120
     strategy:
diff --git a/.github/workflows/crate-io.yml b/.github/workflows/crate-io.yml
index b7552698de72..47e90cb482ce 100644
--- a/.github/workflows/crate-io.yml
+++ b/.github/workflows/crate-io.yml
@@ -8,7 +8,7 @@ on:
 
 jobs:
   release_rust:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - uses: actions/checkout@v4
       - uses: dtolnay/rust-toolchain@stable
diff --git a/.github/workflows/docker-release.yml b/.github/workflows/docker-release.yml
index 227a4a2c0636..9d696256ac4c 100644
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@@ -29,7 +29,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     steps:
     - name: Checkout repository
diff --git a/.github/workflows/rdme-docs-sync.yml b/.github/workflows/rdme-docs-sync.yml
index 69fff4d8ea8b..bf796122900e 100644
--- a/.github/workflows/rdme-docs-sync.yml
+++ b/.github/workflows/rdme-docs-sync.yml
@@ -10,7 +10,7 @@ on:
 
 jobs:
   rdme-docs-sync:  
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Check out repo 📚
         uses: actions/checkout@v4
diff --git a/.github/workflows/readme-rpc-sync.yml b/.github/workflows/readme-rpc-sync.yml
index a955566f5b81..f98055f9b7e8 100644
--- a/.github/workflows/readme-rpc-sync.yml
+++ b/.github/workflows/readme-rpc-sync.yml
@@ -15,7 +15,7 @@ on:
 
 jobs:
   rdme-rpc-sync:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
     - name: Checkout repository
       uses: actions/checkout@v4

From d23d1254d81a4b2fab70bddbccb8fb7e2d38c17f Mon Sep 17 00:00:00 2001
From: ShahanaFarooqui <shahana.farooqui@gmail.com>
Date: Tue, 7 Jan 2025 15:05:25 -0800
Subject: [PATCH 07/12] ci: Lock MacOS and poetry versions

---
 .github/workflows/macos.yaml | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/.github/workflows/macos.yaml b/.github/workflows/macos.yaml
index 95561ec57393..37543b064a6e 100644
--- a/.github/workflows/macos.yaml
+++ b/.github/workflows/macos.yaml
@@ -5,7 +5,7 @@ on:
 jobs:
   smoke-test:
     name: Smoke Test macOS
-    runs-on: macos-latest
+    runs-on: macos-14
     timeout-minutes: 120
     strategy:
       fail-fast: true
@@ -27,17 +27,13 @@ jobs:
 
       - name: Install dependencies
         run: |
-          export PATH="/usr/local/opt:/Users/runner/.local/bin:/Users/runner/Library/Python/3.10/bin:$PATH"
+          export PATH="/usr/local/opt:/Users/runner/.local/bin:/opt/homebrew/bin/python3.10/bin:$PATH"
 
-          brew install wget autoconf automake libtool python@3.10 gnu-sed gettext libsodium protobuf
-
-          python3.10 -m pip install -U --user poetry wheel pip
+          brew install gnu-sed python@3.10 autoconf automake libtool protobuf
+          python3.10 -m pip install -U --user poetry==1.8.0 wheel pip mako
           python3.10 -m poetry install
-          python3.10 -m pip install -U --user mako
-
-          sudo ln -s /usr/local/Cellar/gettext/0.20.1/bin/xgettext /usr/local/opt
 
-      - name: Build and install
+      - name: Build and install CLN
         run: |
           export CPATH=/opt/homebrew/include
           export LIBRARY_PATH=/opt/homebrew/lib
@@ -45,8 +41,6 @@ jobs:
           python3.10 -m poetry run ./configure --disable-valgrind --disable-compat
           python3.10 -m poetry run make
 
-          # sudo PATH="/usr/local/opt:$PATH" LIBRARY_PATH=/opt/homebrew/lib CPATH=/opt/homebrew/include make install
-
       - name: Start bitcoind in regtest mode
         run: |
           bitcoind -regtest -daemon
@@ -58,11 +52,11 @@ jobs:
           bitcoin-cli -regtest generatetoaddress 1 $(bitcoin-cli -regtest getnewaddress)
           sleep 2
 
-      - name: Start lightningd in regtest mode
+      - name: Start CLN in regtest mode
         run: |
           lightningd/lightningd --network=regtest --log-file=/tmp/l1.log --daemon
           sleep 5
 
-      - name: Verify lightningd is running
+      - name: Verify CLN is running
         run: |
           cli/lightning-cli --regtest getinfo

From 4229faead34a85f6f2fb2d0e7e33d1b820b68641 Mon Sep 17 00:00:00 2001
From: ShahanaFarooqui <shahana.farooqui@gmail.com>
Date: Tue, 7 Jan 2025 15:10:15 -0800
Subject: [PATCH 08/12] ci: Install poetry-plugin-export for poetry export
 command

Poetry will no longer include the `poetry-plugin-export` plugin by default, which is essential for exporting dependencies. So, we now need to install it explicitly.
---
 .github/workflows/ci.yaml                | 4 ++++
 Dockerfile                               | 2 ++
 contrib/docker/Dockerfile.builder.fedora | 1 +
 contrib/reprobuild/Dockerfile.focal      | 3 ++-
 contrib/reprobuild/Dockerfile.jammy      | 3 ++-
 contrib/reprobuild/Dockerfile.noble      | 3 ++-
 6 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
index 7c87cf34324c..abe3de9761e0 100644
--- a/.github/workflows/ci.yaml
+++ b/.github/workflows/ci.yaml
@@ -77,6 +77,7 @@ jobs:
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -145,6 +146,7 @@ jobs:
         run: |
           set -e
           pip3 install --user pip wheel poetry
+          poetry self add poetry-plugin-export
           poetry export -o requirements.txt --with dev --without-hashes
           python3 -m pip install -r requirements.txt
           ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
@@ -193,6 +195,7 @@ jobs:
           sudo apt-get update -qq
           sudo apt-get install -y -qq lowdown
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
@@ -227,6 +230,7 @@ jobs:
         run: |
           bash -x .github/scripts/setup.sh
           pip install -U pip wheel poetry
+          poetry self add poetry-plugin-export
           # Export and then use pip to install into the current env
           poetry export -o /tmp/requirements.txt --without-hashes --with dev
           pip install -r /tmp/requirements.txt
diff --git a/Dockerfile b/Dockerfile
index 8c3acd8bcbf4..735069842a75 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -92,6 +92,7 @@ ENV PYTHON_VERSION=3
 RUN curl -sSL https://install.python-poetry.org | python3 -
 RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.9 1
 RUN pip3 install --upgrade pip setuptools wheel
+RUN poetry self add poetry-plugin-export
 
 RUN wget -q https://zlib.net/fossils/zlib-1.2.13.tar.gz -O zlib.tar.gz && \
     wget -q https://www.sqlite.org/2019/sqlite-src-3290000.zip -O sqlite.zip
@@ -198,6 +199,7 @@ RUN ( ! [ "${target_host}" = "arm-linux-gnueabihf" ] ) || \
 # Ensure that the desired grpcio-tools & protobuf versions are installed
 # https://github.com/ElementsProject/lightning/pull/7376#issuecomment-2161102381
 RUN poetry lock --no-update && poetry install
+RUN poetry self add poetry-plugin-export
 
 # Ensure that git differences are removed before making bineries, to avoid `-modded` suffix
 # poetry.lock changed due to pyln-client, pyln-proto and pyln-testing version updates
diff --git a/contrib/docker/Dockerfile.builder.fedora b/contrib/docker/Dockerfile.builder.fedora
index e6b2637f8494..b49775f0a14c 100644
--- a/contrib/docker/Dockerfile.builder.fedora
+++ b/contrib/docker/Dockerfile.builder.fedora
@@ -37,3 +37,4 @@ ENV PATH=/opt/venv/bin:${PATH}
 RUN python3 -m pip install pip wheel && \
     python3 -m virtualenv /opt/venv && \
     /opt/venv/bin/python3 -m pip install --force-reinstall -U pip poetry wheel
+RUN poetry self add poetry-plugin-export
diff --git a/contrib/reprobuild/Dockerfile.focal b/contrib/reprobuild/Dockerfile.focal
index 5e951a88a726..8a860f0c254c 100644
--- a/contrib/reprobuild/Dockerfile.focal
+++ b/contrib/reprobuild/Dockerfile.focal
@@ -49,7 +49,8 @@ RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv && \
 
 RUN wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py \
-    && pip install poetry mako grpcio-tools==1.62.2
+    && pip install poetry mako grpcio-tools==1.62.2 && \
+    poetry self add poetry-plugin-export
 
 RUN wget https://sh.rustup.rs -O rustup-install.sh && \
     bash rustup-install.sh --default-toolchain none --quiet -y && \
diff --git a/contrib/reprobuild/Dockerfile.jammy b/contrib/reprobuild/Dockerfile.jammy
index 4f38b480b655..5cd808ce855f 100644
--- a/contrib/reprobuild/Dockerfile.jammy
+++ b/contrib/reprobuild/Dockerfile.jammy
@@ -47,7 +47,8 @@ RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv && \
 
 RUN wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py \
-    && pip install poetry mako grpcio-tools
+    && pip install poetry mako grpcio-tools && \
+    poetry self add poetry-plugin-export
 
 RUN wget https://sh.rustup.rs -O rustup-install.sh && \
     bash rustup-install.sh --default-toolchain none --quiet -y && \
diff --git a/contrib/reprobuild/Dockerfile.noble b/contrib/reprobuild/Dockerfile.noble
index 2636aef4890a..686b7f3f0a9f 100644
--- a/contrib/reprobuild/Dockerfile.noble
+++ b/contrib/reprobuild/Dockerfile.noble
@@ -45,7 +45,8 @@ RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv && \
 
 RUN wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py \
-    && pip install poetry mako grpcio-tools
+    && pip install poetry mako grpcio-tools && \
+    poetry self add poetry-plugin-export
 
 RUN wget https://sh.rustup.rs -O rustup-install.sh && \
     bash rustup-install.sh --default-toolchain none --quiet -y && \

From e5c387d43846e59372ea665c488557bec923a685 Mon Sep 17 00:00:00 2001
From: ShahanaFarooqui <shahana.farooqui@gmail.com>
Date: Tue, 7 Jan 2025 15:11:19 -0800
Subject: [PATCH 09/12] tests: Reckless test fix for project must contain name

Fix for `The Poetry configuration is invalid:  - project must contain ['name'] properties`
---
 .../data/recklessrepo/lightningd/testplugpyproj/pyproject.toml | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tests/data/recklessrepo/lightningd/testplugpyproj/pyproject.toml b/tests/data/recklessrepo/lightningd/testplugpyproj/pyproject.toml
index 738c943c0878..467c791beaf7 100644
--- a/tests/data/recklessrepo/lightningd/testplugpyproj/pyproject.toml
+++ b/tests/data/recklessrepo/lightningd/testplugpyproj/pyproject.toml
@@ -1,6 +1,3 @@
-[project]
-dependencies = ["pyln-client"]
-
 [tool.poetry]
 name = "testplugpyproj"
 version = "0.1.0"

From e0692521649c00a73ea77b329c5091cf7c92dc20 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Thu, 3 Apr 2025 07:03:59 +1030
Subject: [PATCH 10/12] trace: handle key being freed while suspended.

This happens with autoclean, which does a datastore request then frees
the parent command without waiting for a response (see clean_finished).

This leaks a trace, and causes a crash if the pointer is later reused.

My solution is to create a trace variant which declares the trace key
to be a tal ptr and then we can clean up in the destructor if this happens.
This fixes the issue for me.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
Changelog-Fixed: autoclean: fixed occasional crash when tracepoints compiled in.
---
 common/trace.c      | 23 +++++++++++++++++++++++
 common/trace.h      |  2 ++
 plugins/libplugin.c |  2 +-
 3 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/common/trace.c b/common/trace.c
index 9e7a01c863d3..5bd5fa5647fa 100644
--- a/common/trace.c
+++ b/common/trace.c
@@ -373,6 +373,28 @@ void trace_span_suspend_(const void *key, const char *lbl)
 	DTRACE_PROBE1(lightningd, span_suspend, span->id);
 }
 
+static void destroy_trace_span(const void *key)
+{
+	size_t numkey = trace_key(key);
+	struct span *span = trace_span_find(numkey);
+
+	/* It's usually ended normally. */
+	if (!span)
+		return;
+
+	/* Otherwise resume so we can terminate it */
+	trace_span_resume(key);
+	trace_span_end(key);
+}
+
+void trace_span_suspend_may_free_(const void *key, const char *lbl)
+{
+	if (disable_trace)
+		return;
+	trace_span_suspend_(key, lbl);
+	tal_add_destructor(key, destroy_trace_span);
+}
+
 void trace_span_resume_(const void *key, const char *lbl)
 {
 	if (disable_trace)
@@ -395,6 +417,7 @@ void trace_cleanup(void)
 void trace_span_start(const char *name, const void *key) {}
 void trace_span_end(const void *key) {}
 void trace_span_suspend_(const void *key, const char *lbl) {}
+void trace_span_suspend_may_free_(const void *key, const char *lbl) {}
 void trace_span_resume_(const void *key, const char *lbl) {}
 void trace_span_tag(const void *key, const char *name, const char *value) {}
 void trace_cleanup(void) {}
diff --git a/common/trace.h b/common/trace.h
index 4fc8a4149e8a..d8996684b9d7 100644
--- a/common/trace.h
+++ b/common/trace.h
@@ -15,8 +15,10 @@ void trace_span_remote(u8 trace_id[TRACE_ID_SIZE], u8 span_id[SPAN_ID_SIZE]);
 
 #define TRACE_LBL __FILE__ ":" stringify(__LINE__)
 void trace_span_suspend_(const void *key, const char *lbl);
+void trace_span_suspend_may_free_(const void *key, const char *lbl);
 void trace_span_resume_(const void *key, const char *lbl);
 #define trace_span_suspend(key) trace_span_suspend_(key, TRACE_LBL)
+#define trace_span_suspend_may_free(key) trace_span_suspend_may_free_(key, TRACE_LBL)
 #define trace_span_resume(key) trace_span_resume_(key, TRACE_LBL)
 
 #endif /* LIGHTNING_COMMON_TRACE_H */
diff --git a/plugins/libplugin.c b/plugins/libplugin.c
index abec6103831f..75271a4f6fd2 100644
--- a/plugins/libplugin.c
+++ b/plugins/libplugin.c
@@ -1111,7 +1111,7 @@ send_outreq(const struct out_req *req)
 	 * callback. */
 	trace_span_start("jsonrpc", req);
 	trace_span_tag(req, "id", req->id);
-	trace_span_suspend(req);
+	trace_span_suspend_may_free(req);
 
 	ld_rpc_send(req->cmd->plugin, req->js);
 	notleak_with_children(req->cmd);

From 2cd38297dfd5c61a83d9fff69beb8ed1b174690b Mon Sep 17 00:00:00 2001
From: Alex Myers <alex@endothermic.dev>
Date: Fri, 4 Apr 2025 11:29:52 -0500
Subject: [PATCH 11/12] meta: update versions for 24.11.2 release

---
 .version                                      |  2 +-
 contrib/msggen/msggen/schema.json             |  2 +-
 contrib/pyln-client/pyln/client/__init__.py   |  2 +-
 contrib/pyln-client/pyproject.toml            |  2 +-
 contrib/pyln-proto/pyln/proto/__init__.py     |  2 +-
 contrib/pyln-proto/pyproject.toml             |  2 +-
 contrib/pyln-testing/pyln/testing/__init__.py |  2 +-
 contrib/pyln-testing/pyproject.toml           |  2 +-
 doc/schemas/lightning-getinfo.json            |  2 +-
 plugins/clnrest/pyproject.toml                |  2 +-
 plugins/wss-proxy/pyproject.toml              |  2 +-
 poetry.lock                                   | 10 +++++-----
 12 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.version b/.version
index 9e2934aa3435..a8985996194f 100644
--- a/.version
+++ b/.version
@@ -1 +1 @@
-24.11.1
+24.11.2
diff --git a/contrib/msggen/msggen/schema.json b/contrib/msggen/msggen/schema.json
index dd7c025df80b..1a50d90f514b 100644
--- a/contrib/msggen/msggen/schema.json
+++ b/contrib/msggen/msggen/schema.json
@@ -15032,7 +15032,7 @@
                 "port": 19735
               }
             ],
-            "version": "v24.11.1",
+            "version": "v24.11.2",
             "blockheight": 110,
             "network": "regtest",
             "fees_collected_msat": 0,
diff --git a/contrib/pyln-client/pyln/client/__init__.py b/contrib/pyln-client/pyln/client/__init__.py
index a708971030a5..e2fe9b92c87b 100644
--- a/contrib/pyln-client/pyln/client/__init__.py
+++ b/contrib/pyln-client/pyln/client/__init__.py
@@ -3,7 +3,7 @@
 from .gossmap import Gossmap, GossmapNode, GossmapChannel, GossmapHalfchannel, GossmapNodeId, LnFeatureBits
 from .gossmapstats import GossmapStats
 
-__version__ = "24.11.1"
+__version__ = "24.11.2"
 
 __all__ = [
     "LightningRpc",
diff --git a/contrib/pyln-client/pyproject.toml b/contrib/pyln-client/pyproject.toml
index e7f2dbff7476..0ea50c5e97d5 100644
--- a/contrib/pyln-client/pyproject.toml
+++ b/contrib/pyln-client/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyln-client"
-version = "24.11.1"
+version = "24.11.2"
 description = "Client library and plugin library for Core Lightning"
 authors = ["Christian Decker <decker.christian@gmail.com>"]
 license = "BSD-MIT"
diff --git a/contrib/pyln-proto/pyln/proto/__init__.py b/contrib/pyln-proto/pyln/proto/__init__.py
index ebb78d6a8209..e31580d891c2 100644
--- a/contrib/pyln-proto/pyln/proto/__init__.py
+++ b/contrib/pyln-proto/pyln/proto/__init__.py
@@ -4,7 +4,7 @@
 from .onion import OnionPayload, TlvPayload, LegacyOnionPayload
 from .wire import LightningConnection, LightningServerSocket
 
-__version__ = "24.11.1"
+__version__ = "24.11.2"
 
 __all__ = [
     "Invoice",
diff --git a/contrib/pyln-proto/pyproject.toml b/contrib/pyln-proto/pyproject.toml
index 08f598a1a5dd..cd2241825e6a 100644
--- a/contrib/pyln-proto/pyproject.toml
+++ b/contrib/pyln-proto/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyln-proto"
-version = "24.11.1"
+version = "24.11.2"
 description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)."
 authors = ["Christian Decker <decker.christian@gmail.com>"]
 license = "BSD-MIT"
diff --git a/contrib/pyln-testing/pyln/testing/__init__.py b/contrib/pyln-testing/pyln/testing/__init__.py
index 3e7d70ca58d6..2d52458f871f 100644
--- a/contrib/pyln-testing/pyln/testing/__init__.py
+++ b/contrib/pyln-testing/pyln/testing/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "24.11.1"
+__version__ = "24.11.2"
 
 __all__ = [
     "__version__",
diff --git a/contrib/pyln-testing/pyproject.toml b/contrib/pyln-testing/pyproject.toml
index eb36d9515526..7ab2890f9056 100644
--- a/contrib/pyln-testing/pyproject.toml
+++ b/contrib/pyln-testing/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pyln-testing"
-version = "24.11.1"
+version = "24.11.2"
 description = "Test your Core Lightning integration, plugins or whatever you want"
 authors = ["Christian Decker <decker.christian@gmail.com>"]
 license = "BSD-MIT"
diff --git a/doc/schemas/lightning-getinfo.json b/doc/schemas/lightning-getinfo.json
index 7d250ab230c5..2cd418497ef1 100644
--- a/doc/schemas/lightning-getinfo.json
+++ b/doc/schemas/lightning-getinfo.json
@@ -410,7 +410,7 @@
             "port": 19735
           }
         ],
-        "version": "v24.11.1",
+        "version": "v24.11.2",
         "blockheight": 110,
         "network": "regtest",
         "fees_collected_msat": 0,
diff --git a/plugins/clnrest/pyproject.toml b/plugins/clnrest/pyproject.toml
index 8314f36072eb..f3c67106ca4e 100644
--- a/plugins/clnrest/pyproject.toml
+++ b/plugins/clnrest/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "clnrest"
-version = "24.11.1"
+version = "24.11.2"
 description = "Transforms RPC calls into REST APIs"
 authors = ["ShahanaFarooqui <sfarooqui@blockstream.com>"]
 
diff --git a/plugins/wss-proxy/pyproject.toml b/plugins/wss-proxy/pyproject.toml
index 5674e28a18dc..4c681b1d43b5 100644
--- a/plugins/wss-proxy/pyproject.toml
+++ b/plugins/wss-proxy/pyproject.toml
@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "wss-proxy"
-version = "24.11.1"
+version = "24.11.2"
 description = "Web secure socket proxy"
 authors = ["ShahanaFarooqui <sfarooqui@blockstream.com>"]
 
diff --git a/poetry.lock b/poetry.lock
index bab69e4ee89e..02fe25025920 100644
--- a/poetry.lock
+++ b/poetry.lock
@@ -433,7 +433,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""}
 
 [[package]]
 name = "clnrest"
-version = "24.11.1"
+version = "24.11.2"
 description = "Transforms RPC calls into REST APIs"
 optional = false
 python-versions = "^3.8"
@@ -1690,7 +1690,7 @@ files = [
 
 [[package]]
 name = "pyln-client"
-version = "24.11.1"
+version = "24.11.2"
 description = "Client library and plugin library for Core Lightning"
 optional = false
 python-versions = "^3.8"
@@ -1724,7 +1724,7 @@ url = "contrib/pyln-grpc-proto"
 
 [[package]]
 name = "pyln-proto"
-version = "24.11.1"
+version = "24.11.2"
 description = "This package implements some of the Lightning Network protocol in pure python. It is intended for protocol testing and some minor tooling only. It is not deemed secure enough to handle any amount of real funds (you have been warned!)."
 optional = false
 python-versions = "^3.8"
@@ -1744,7 +1744,7 @@ url = "contrib/pyln-proto"
 
 [[package]]
 name = "pyln-testing"
-version = "24.11.1"
+version = "24.11.2"
 description = "Test your Core Lightning integration, plugins or whatever you want"
 optional = false
 python-versions = "^3.8"
@@ -2289,7 +2289,7 @@ h11 = ">=0.9.0,<1"
 
 [[package]]
 name = "wss-proxy"
-version = "24.11.1"
+version = "24.11.2"
 description = "Web secure socket proxy"
 optional = false
 python-versions = "^3.8"

From 9345f415bc0f0ca8ac116778f397592238bea468 Mon Sep 17 00:00:00 2001
From: Rusty Russell <rusty@rustcorp.com.au>
Date: Wed, 2 Apr 2025 11:33:44 +1030
Subject: [PATCH 12/12] v24.11.2: CHANGELOG update.

Note that we didn't backport the db changes, since that would complicate upgrades to v25.02.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
---
 CHANGELOG.md | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d314a5bfa464..9f39adf02d41 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -3,6 +3,29 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
+## [24.11.2] - 2025-04-03: "The lightning-dev Mailing List III"
+
+Important fixes for bugs in v24.11.
+
+### Fixed
+
+ - wallet: we could miss our own returned outputs on mutual closes if peer doesn't support option_shutdown_anysegwit.  Any missing outputs can be found by rescan after upgrade, or upgrading to v25.02.1 ([#8175])
+ - lightningd: incorrect spamming of log and potential crash on testnet case of duplicate HTLCs and slow closing. ([#8192])
+ - Protocol: Egregious anchor fee paid for unilateral close txs due to HTLC timeouts; it's not as urgent as our code made out! ([#8190])
+ - lightningd: occasional crash on bitcoind callback. ([#8186])
+ - autoclean/chanbackup: fixed tracepoint crash on large number of requests. ([#8188])
+ - autoclean: fixed occasional crash when tracepoints compiled in. ([#8198])
+ - `topology` crash on invoice creation if a peer had a really high feerate. ([#8187])
+
+[#8186]: https://github.com/ElementsProject/lightning/pull/8186
+[#8192]: https://github.com/ElementsProject/lightning/pull/8192
+[#8187]: https://github.com/ElementsProject/lightning/pull/8187
+[#8190]: https://github.com/ElementsProject/lightning/pull/8190
+[#8175]: https://github.com/ElementsProject/lightning/pull/8175
+[#8188]: https://github.com/ElementsProject/lightning/pull/8188
+[#8198]: https://github.com/ElementsProject/lightning/pull/8198
+[25.11.2]: https://github.com/ElementsProject/lightning/releases/tag/v24.11.2
+
 ## [24.11.1] - 2024-12-16: "The lightning-dev Mailing List II"
 
 Minor fixes, particularly for xpay users.