lightningd: rescan for missing p2wkph for closed channels.

This can happen with 24.11 and later.  We scan back to exposed channel
opens, or that release.

The BROKEN log messages cause some tests to fail, so we fix those.

Fixes: #8169
Changelog-Fixed: wallet: rescan for missing close outputs (can happen if peer doesn't support option_shutdown_anysegwit)

Author: rustyrussell

Diff for: lightningd/lightningd.c

@@ -1471,6 +1471,12 @@ int main(int argc, char *argv[])
 	if (ld->daemon_parent_fd != -1)
 		complete_daemonize(ld);
 
+	/*~ At one stage, we didn't catch mutual closes with old node to p2wkph
+	 * so now we start a scan if the db doesn't say we completed it. */
+	db_begin_transaction(ld->wallet->db);
+	wallet_begin_old_close_rescan(ld);
+	db_commit_transaction(ld->wallet->db);
+
 	/*~ Setting this (global) activates the crash log: we don't usually need
 	 * a backtrace if we fail during startup. */
 	crashlog = ld->log;

Diff for: lightningd/test/run-find_my_abspath.c

@@ -286,6 +286,9 @@ const char *version(void)
 void waitblockheight_notify_new_block(struct lightningd *ld UNNEEDED,
 				      u32 block_height UNNEEDED)
 { fprintf(stderr, "waitblockheight_notify_new_block called!\n"); abort(); }
+/* Generated stub for wallet_begin_old_close_rescan */
+void wallet_begin_old_close_rescan(struct lightningd *ld UNNEEDED)
+{ fprintf(stderr, "wallet_begin_old_close_rescan called!\n"); abort(); }
 /* Generated stub for wallet_new */
 struct wallet *wallet_new(struct lightningd *ld UNNEEDED, struct timers *timers UNNEEDED)
 { fprintf(stderr, "wallet_new called!\n"); abort(); }

Diff for: wallet/db.c

@@ -1033,6 +1033,8 @@ static struct migration dbmigrations[] = {
     {SQL("ALTER TABLE peers ADD last_known_address BLOB DEFAULT NULL;"), NULL},
     {SQL("ALTER TABLE channels ADD close_attempt_height INTEGER DEFAULT 0;"), NULL},
     {NULL, migrate_convert_old_channel_keyidx},
+    {SQL("INSERT INTO vars(name, intval)"
+	 "  VALUES('needs_p2wpkh_close_rescan', 1)"), NULL},
 };
 
 /**

Diff for: wallet/test/run-db.c

@@ -25,6 +25,17 @@ static void db_log_(struct logger *log UNUSED, enum log_level level UNUSED, cons
 /* Generated stub for bip32_pubkey */
 void bip32_pubkey(struct lightningd *ld UNNEEDED, struct pubkey *pubkey UNNEEDED, u32 index UNNEEDED)
 { fprintf(stderr, "bip32_pubkey called!\n"); abort(); }
+/* Generated stub for bitcoind_getrawblockbyheight_ */
+void bitcoind_getrawblockbyheight_(const tal_t *ctx UNNEEDED,
+				   struct bitcoind *bitcoind UNNEEDED,
+				   u32 height UNNEEDED,
+				   void (*cb)(struct bitcoind *bitcoind_ UNNEEDED,
+					      u32 height_ UNNEEDED,
+					      struct bitcoin_blkid *blkid UNNEEDED,
+					      struct bitcoin_block *blk UNNEEDED,
+					      void *arg) UNNEEDED,
+				   void *arg UNNEEDED)
+{ fprintf(stderr, "bitcoind_getrawblockbyheight_ called!\n"); abort(); }
 /* Generated stub for channel_gossip_get_remote_update */
 const struct peer_update *channel_gossip_get_remote_update(const struct channel *channel UNNEEDED)
 { fprintf(stderr, "channel_gossip_get_remote_update called!\n"); abort(); }
@@ -296,9 +307,6 @@ struct htlc_in *remove_htlc_in_by_dbid(struct htlc_in_map *remaining_htlcs_in UN
 /* Generated stub for rune_unique_id */
 u64 rune_unique_id(const struct rune *rune UNNEEDED)
 { fprintf(stderr, "rune_unique_id called!\n"); abort(); }
-/* Generated stub for scriptpubkey_hash */
-size_t scriptpubkey_hash(const u8 *out UNNEEDED)
-{ fprintf(stderr, "scriptpubkey_hash called!\n"); abort(); }
 /* Generated stub for sendpay_index_created */
 u64 sendpay_index_created(struct lightningd *ld UNNEEDED,
 			  const struct sha256 *payment_hash UNNEEDED,

Diff for: wallet/test/run-wallet.c

@@ -49,6 +49,17 @@ void add_node_announcement_sig(u8 *nannounce UNNEEDED,
 /* Generated stub for bigsize_put */
 size_t bigsize_put(u8 buf[BIGSIZE_MAX_LEN] UNNEEDED, bigsize_t v UNNEEDED)
 { fprintf(stderr, "bigsize_put called!\n"); abort(); }
+/* Generated stub for bitcoind_getrawblockbyheight_ */
+void bitcoind_getrawblockbyheight_(const tal_t *ctx UNNEEDED,
+				   struct bitcoind *bitcoind UNNEEDED,
+				   u32 height UNNEEDED,
+				   void (*cb)(struct bitcoind *bitcoind_ UNNEEDED,
+					      u32 height_ UNNEEDED,
+					      struct bitcoin_blkid *blkid UNNEEDED,
+					      struct bitcoin_block *blk UNNEEDED,
+					      void *arg) UNNEEDED,
+				   void *arg UNNEEDED)
+{ fprintf(stderr, "bitcoind_getrawblockbyheight_ called!\n"); abort(); }
 /* Generated stub for bitcoind_getutxout_ */
 void bitcoind_getutxout_(const tal_t *ctx UNNEEDED,
 			 struct bitcoind *bitcoind UNNEEDED,
@@ -967,9 +978,6 @@ void resolve_close_command(struct lightningd *ld UNNEEDED, struct channel *chann
 /* Generated stub for rune_unique_id */
 u64 rune_unique_id(const struct rune *rune UNNEEDED)
 { fprintf(stderr, "rune_unique_id called!\n"); abort(); }
-/* Generated stub for scriptpubkey_hash */
-size_t scriptpubkey_hash(const u8 *out UNNEEDED)
-{ fprintf(stderr, "scriptpubkey_hash called!\n"); abort(); }
 /* Generated stub for serialize_onionpacket */
 u8 *serialize_onionpacket(
 	const tal_t *ctx UNNEEDED,

Diff for: wallet/wallet.c

@@ -2917,7 +2917,8 @@ static void got_utxo(struct wallet *w,
 		     const struct wally_tx *wtx,
 		     size_t outnum,
 		     bool is_coinbase,
-		     const u32 *blockheight)
+		     const u32 *blockheight,
+		     struct bitcoin_outpoint *outpoint)
 {
 	struct utxo *utxo = tal(tmpctx, struct utxo);
 	const struct wally_tx_output *txout = &wtx->outputs[outnum];
@@ -2972,6 +2973,8 @@ static void got_utxo(struct wallet *w,
 	outpointfilter_add(w->owned_outpoints, &utxo->outpoint);
 
 	wallet_annotate_txout(w, &utxo->outpoint, TX_WALLET_DEPOSIT, 0);
+	if (outpoint)
+		*outpoint = utxo->outpoint;
 }
 
 int wallet_extract_owned_outputs(struct wallet *w, const struct wally_tx *wtx,
@@ -2991,7 +2994,7 @@ int wallet_extract_owned_outputs(struct wallet *w, const struct wally_tx *wtx,
 		if (!wallet_can_spend(w, txout->script, txout->script_len, &keyindex))
 			continue;
 
-		got_utxo(w, keyindex, wtx, i, is_coinbase, blockheight);
+		got_utxo(w, keyindex, wtx, i, is_coinbase, blockheight, NULL);
 		num_utxos++;
 	}
 	return num_utxos;
@@ -6512,3 +6515,170 @@ struct issued_address_type *wallet_list_addresses(const tal_t *ctx, struct walle
 	tal_free(stmt);
 	return addresseslist;
 }
+
+struct missing {
+	size_t num_found;
+	struct missing_addr *addrs;
+};
+
+struct missing_addr {
+	u64 keyidx;
+	const u8 *scriptpubkey;
+};
+
+static void mutual_close_p2pkh_catch(struct bitcoind *bitcoind,
+				     u32 height,
+				     struct bitcoin_blkid *blkid,
+				     struct bitcoin_block *blk,
+				     struct missing *missing)
+{
+	struct wallet *w = bitcoind->ld->wallet;
+
+	/* Are we finished? */
+	if (!blkid) {
+		log_broken(bitcoind->ld->log,
+			   "Rescan finished! %zu outputs recovered.  Let's never do that again.",
+			   missing->num_found);
+		db_set_intvar(w->db, "needs_p2wpkh_close_rescan", 0);
+		/* Call is allocated off of this, so we can't free just yet */
+		tal_steal(tmpctx, missing);
+		return;
+	}
+
+	log_debug(bitcoind->ld->log, "Mutual close p2wpkh recovery block %u", height);
+	for (size_t i = 0; i < tal_count(blk->tx); i++) {
+		const struct wally_tx *wtx = blk->tx[i]->wtx;
+		for (size_t outnum = 0; outnum < wtx->num_outputs; outnum++) {
+			const struct wally_tx_output *txout = &wtx->outputs[outnum];
+			for (size_t n = 0; n < tal_count(missing->addrs); n++) {
+				struct bitcoin_outpoint outp;
+				log_debug(bitcoind->ld->log, "%zu out %zu: %s (seeking %s)",
+					  i, outnum,
+					  tal_hexstr(tmpctx, txout->script, txout->script_len),
+					  tal_hex(tmpctx, missing->addrs[n].scriptpubkey));
+				if (!memeq(txout->script, txout->script_len,
+					   missing->addrs[n].scriptpubkey,
+					   tal_bytelen(missing->addrs[n].scriptpubkey)))
+					continue;
+				got_utxo(w, missing->addrs[n].keyidx,
+					 wtx, outnum, i == 0, &height, &outp);
+				log_broken(bitcoind->ld->log, "Rescan found %s!",
+					   fmt_bitcoin_outpoint(tmpctx, &outp));
+				missing->num_found++;
+			}
+		}
+	}
+
+	/* Next block! */
+	bitcoind_getrawblockbyheight(missing, bitcoind, height + 1,
+				     mutual_close_p2pkh_catch, missing);
+}
+
+void wallet_begin_old_close_rescan(struct lightningd *ld)
+{
+	struct db_stmt *stmt;
+	u32 earliest_block = UINT32_MAX;
+	struct missing *missing;
+	int v = db_get_intvar(ld->wallet->db, "needs_p2wpkh_close_rescan", -1);
+	if (v == 0)
+		return;
+	assert(v == 1);
+
+	/* Channels where we might have already seen a mutual close,
+	 * which said their key was only taproot, but we didn't see an
+	 * output, may actually have closed to the p2wpkh address.
+	 * Some debugging help with ChatGPT here!
+	 */
+	stmt = db_prepare_v2(ld->wallet->db,
+			     SQL("SELECT "
+				 " full_channel_id"
+				 ", state"
+				 ", scid"
+				 ", shutdown_keyidx_local"
+				 " FROM channels"
+				 " JOIN addresses"
+				 "  ON channels.shutdown_keyidx_local = addresses.keyidx"
+				 " WHERE addresses.addrtype = ?"
+				 " AND NOT EXISTS ("
+				 "    SELECT 1"
+				 "    FROM outputs"
+				 "    WHERE outputs.keyindex = addresses.keyidx"
+				 ");"));
+	db_bind_int(stmt, wallet_addrtype_in_db(ADDR_P2TR));
+	db_query_prepared(stmt);
+
+	missing = tal(tmpctx, struct missing);
+	missing->num_found = 0;
+	missing->addrs = tal_arr(missing, struct missing_addr, 0);
+	while (db_step(stmt)) {
+		enum channel_state state;
+		struct short_channel_id scid;
+		struct channel_id cid;
+		struct missing_addr maddr;
+		struct ext_key ext;
+
+		state = channel_state_in_db(db_col_int(stmt, "state"));
+		switch (state) {
+		case DUALOPEND_OPEN_INIT:
+		case CHANNELD_AWAITING_LOCKIN:
+		case CHANNELD_NORMAL:
+		case CHANNELD_SHUTTING_DOWN:
+		case CLOSINGD_SIGEXCHANGE:
+		case DUALOPEND_OPEN_COMMITTED:
+		case DUALOPEND_AWAITING_LOCKIN:
+		case CHANNELD_AWAITING_SPLICE:
+		case DUALOPEND_OPEN_COMMIT_READY:
+			db_col_ignore(stmt, "full_channel_id");
+			db_col_ignore(stmt, "scid");
+			db_col_ignore(stmt, "shutdown_keyidx_local");
+			continue;
+		/* States where we may have already seen close */
+		case CLOSINGD_COMPLETE:
+		case AWAITING_UNILATERAL:
+		case FUNDING_SPEND_SEEN:
+		case ONCHAIN:
+		case CLOSED:
+			db_col_channel_id(stmt, "full_channel_id", &cid);
+			maddr.keyidx = db_col_u64(stmt, "shutdown_keyidx_local");
+
+			/* This can happen with zeroconf, but it's unusual.  In that case
+			 * we haven't even seen the open, let alone the close.*/
+			if (db_col_is_null(stmt, "scid"))
+				continue;
+
+			/* Don't search for close before open */
+			scid = db_col_short_channel_id(stmt, "scid");
+			if (short_channel_id_blocknum(scid) < earliest_block)
+				earliest_block = short_channel_id_blocknum(scid);
+
+			if (bip32_key_from_parent(ld->bip32_base, maddr.keyidx,
+						  BIP32_FLAG_KEY_PUBLIC | BIP32_FLAG_SKIP_HASH,
+						  &ext) != WALLY_OK) {
+				abort();
+			}
+			maddr.scriptpubkey = scriptpubkey_p2wpkh_derkey(missing, ext.pub_key);
+			tal_arr_expand(&missing->addrs, maddr);
+		}
+	}
+	tal_free(stmt);
+
+	/* No results?  We didn't have a problem, never test again. */
+	if (tal_count(missing->addrs) == 0) {
+		db_set_intvar(ld->wallet->db, "needs_p2wpkh_close_rescan", 0);
+		return;
+	}
+
+	/* We only released 24.11 in December, so don't go back before
+	* block 870000 which was mid-November */
+	if (streq(chainparams->network_name, "bitcoin") && earliest_block < 870000)
+		earliest_block = 870000;
+
+	log_broken(ld->log,
+		   "Potentially missing %zu outputs from previous closes: scanning from block %u",
+		   tal_count(missing->addrs), earliest_block);
+
+	/* This is not a leak, though it may take a while! */
+	tal_steal(ld, notleak(missing));
+	bitcoind_getrawblockbyheight(missing, ld->topology->bitcoind, earliest_block,
+				     mutual_close_p2pkh_catch, missing);
+}

Diff for: wallet/wallet.h

@@ -1825,6 +1825,13 @@ struct issued_address_type *wallet_list_addresses(const tal_t *ctx, struct walle
 					 u64 liststart, const u32 *listlimit);
 
 
+/**
+ * wallet_begin_old_close_rescan: rescan for missing mutual close p2wpkh outputs.
+ *
+ * Once complete, we set a db var so we never do this again.
+ */
+void wallet_begin_old_close_rescan(struct lightningd *ld);
+
 /**
  * wallet_memleak_scan - Check for memleaks in wallet.
  */