DB changes to shrink unneeded HTLCs, only pass minimal set to onchaind.

.github/scripts/build.sh

@@ -19,6 +19,7 @@ export TEST_NETWORK=${NETWORK:-"regtest"}
 export TIMEOUT=900
 export VALGRIND=${VALGRIND:-0}
 export FUZZING=${FUZZING:-0}
+export LIGHTNINGD_POSTGRES_NO_VACUUM=1
 
 pip3 install --user -U \
      -r requirements.lock

bitcoin/psbt.c

@@ -126,8 +126,8 @@ struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
 }
 
 struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
-					   const struct bitcoin_txid *txid,
-					   u32 outnum, u32 sequence,
+					   const struct bitcoin_outpoint *outpoint,
+					   u32 sequence,
 					   const u8 *scriptSig,
 					   const u8 *input_wscript,
 					   const u8 *redeemscript)
@@ -139,9 +139,10 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
 
 	tal_wally_start();
 	if (chainparams->is_elements) {
-		if (wally_tx_elements_input_init_alloc(txid->shad.sha.u.u8,
-						       sizeof(txid->shad.sha.u.u8),
-						       outnum, sequence, NULL, 0,
+		if (wally_tx_elements_input_init_alloc(outpoint->txid.shad.sha.u.u8,
+						       sizeof(outpoint->txid.shad.sha.u.u8),
+						       outpoint->n,
+						       sequence, NULL, 0,
 						       NULL,
 						       NULL, 0,
 						       NULL, 0, NULL, 0,
@@ -150,9 +151,10 @@ struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
 						       &tx_in) != WALLY_OK)
 			abort();
 	} else {
-		if (wally_tx_input_init_alloc(txid->shad.sha.u.u8,
-					      sizeof(txid->shad.sha.u.u8),
-					      outnum, sequence, NULL, 0, NULL,
+		if (wally_tx_input_init_alloc(outpoint->txid.shad.sha.u.u8,
+					      sizeof(outpoint->txid.shad.sha.u.u8),
+					      outpoint->n,
+					      sequence, NULL, 0, NULL,
 					      &tx_in) != WALLY_OK)
 			abort();
 	}
@@ -412,18 +414,17 @@ void psbt_elements_normalize_fees(struct wally_psbt *psbt)
 }
 
 bool psbt_has_input(const struct wally_psbt *psbt,
-		    const struct bitcoin_txid *txid,
-		    u32 outnum)
+		    const struct bitcoin_outpoint *outpoint)
 {
 	for (size_t i = 0; i < psbt->num_inputs; i++) {
 		struct bitcoin_txid in_txid;
 		struct wally_tx_input *in = &psbt->tx->inputs[i];
 
-		if (outnum != in->index)
+		if (outpoint->n != in->index)
 			continue;
 
 		wally_tx_input_get_txid(in, &in_txid);
-		if (bitcoin_txid_eq(txid, &in_txid))
+		if (bitcoin_txid_eq(&outpoint->txid, &in_txid))
 			return true;
 	}
 	return false;

bitcoin/psbt.h

@@ -12,6 +12,7 @@ struct wally_tx_output;
 struct wally_map;
 struct amount_asset;
 struct amount_sat;
+struct bitcoin_outpoint;
 struct bitcoin_signature;
 struct bitcoin_txid;
 struct pubkey;
@@ -119,8 +120,8 @@ struct wally_psbt_input *psbt_add_input(struct wally_psbt *psbt,
 
 /* One stop shop for adding an input + metadata to a PSBT */
 struct wally_psbt_input *psbt_append_input(struct wally_psbt *psbt,
-					   const struct bitcoin_txid *txid,
-					   u32 outnum, u32 sequence,
+					   const struct bitcoin_outpoint *outpoint,
+					   u32 sequence,
 					   const u8 *scriptSig,
 					   const u8 *input_wscript,
 					   const u8 *redeemscript);
@@ -253,12 +254,10 @@ struct amount_sat psbt_compute_fee(const struct wally_psbt *psbt);
 /* psbt_has_input - Is this input present on this psbt
  *
  * @psbt - psbt
- * @txid - txid of input
- * @outnum - output index of input
+ * @outpoint - txid/index spent by input
  */
 bool psbt_has_input(const struct wally_psbt *psbt,
-		    const struct bitcoin_txid *txid,
-		    u32 outnum);
+		    const struct bitcoin_outpoint *outpoint);
 
 struct wally_psbt *psbt_from_b64(const tal_t *ctx,
 				 const char *b64,

bitcoin/script.c

@@ -12,8 +12,6 @@
 /* To push 0-75 bytes onto stack. */
 #define OP_PUSHBYTES(val) (val)
 
-#define max(a, b) ((a) > (b) ? (a) : (b))
-
 /* Bitcoin's OP_HASH160 is RIPEMD(SHA256()) */
 static void hash160(struct ripemd160 *redeemhash, const void *mem, size_t len)
 {
@@ -556,7 +554,7 @@ u8 *bitcoin_wscript_to_local(const tal_t *ctx, u16 to_self_delay,
 	add_op(&script, OP_IF);
 	add_push_key(&script, revocation_pubkey);
 	add_op(&script, OP_ELSE);
-	add_number(&script, max(lease_remaining, to_self_delay));
+	add_number(&script, max_unsigned(lease_remaining, to_self_delay));
 	add_op(&script, OP_CHECKSEQUENCEVERIFY);
 	add_op(&script, OP_DROP);
 	add_push_key(&script, local_delayedkey);

bitcoin/tx.c

@@ -4,6 +4,7 @@
 #include <bitcoin/psbt.h>
 #include <bitcoin/script.h>
 #include <ccan/str/hex/hex.h>
+#include <ccan/tal/str/str.h>
 #include <common/type_to_string.h>
 #include <wally_psbt.h>
 #include <wire/wire.h>
@@ -195,15 +196,17 @@ void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime)
 	tx->psbt->tx->locktime = locktime;
 }
 
-int bitcoin_tx_add_input(struct bitcoin_tx *tx, const struct bitcoin_txid *txid,
-			 u32 outnum, u32 sequence, const u8 *scriptSig,
+int bitcoin_tx_add_input(struct bitcoin_tx *tx,
+			 const struct bitcoin_outpoint *outpoint,
+			 u32 sequence, const u8 *scriptSig,
 			 struct amount_sat amount, const u8 *scriptPubkey,
 			 const u8 *input_wscript)
 {
 	int wally_err;
 	int input_num = tx->wtx->num_inputs;
 
-	psbt_append_input(tx->psbt, txid, outnum, sequence, scriptSig,
+	psbt_append_input(tx->psbt, outpoint,
+			  sequence, scriptSig,
 			  input_wscript, NULL);
 
 	if (input_wscript) {
@@ -387,33 +390,50 @@ const u8 *bitcoin_tx_input_get_witness(const tal_t *ctx,
 	return witness_item;
 }
 
+/* FIXME: remove */
 void bitcoin_tx_input_get_txid(const struct bitcoin_tx *tx, int innum,
 			       struct bitcoin_txid *out)
 {
 	assert(innum < tx->wtx->num_inputs);
 	wally_tx_input_get_txid(&tx->wtx->inputs[innum], out);
 }
 
-void bitcoin_tx_input_set_txid(struct bitcoin_tx *tx, int innum,
-			       const struct bitcoin_txid *txid,
-			       u32 index)
+void bitcoin_tx_input_get_outpoint(const struct bitcoin_tx *tx,
+				   int innum,
+				   struct bitcoin_outpoint *outpoint)
+{
+	assert(innum < tx->wtx->num_inputs);
+	wally_tx_input_get_outpoint(&tx->wtx->inputs[innum], outpoint);
+}
+
+void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
+				   const struct bitcoin_outpoint *outpoint)
 {
 	struct wally_tx_input *in;
 	assert(innum < tx->wtx->num_inputs);
 
 	in = &tx->wtx->inputs[innum];
 	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
-	memcpy(in->txhash, txid, sizeof(struct bitcoin_txid));
-	in->index = index;
+	memcpy(in->txhash, &outpoint->txid, sizeof(struct bitcoin_txid));
+	in->index = outpoint->n;
 }
 
+/* FIXME: remove */
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid)
 {
 	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
 	memcpy(txid, in->txhash, sizeof(struct bitcoin_txid));
 }
 
+void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
+				 struct bitcoin_outpoint *outpoint)
+{
+	BUILD_ASSERT(sizeof(struct bitcoin_txid) == sizeof(in->txhash));
+	memcpy(&outpoint->txid, in->txhash, sizeof(struct bitcoin_txid));
+	outpoint->n = in->index;
+}
+
 /* BIP144:
  * If the witness is empty, the old serialization format should be used. */
 static bool uses_witness(const struct wally_tx *wtx)
@@ -677,6 +697,14 @@ static char *fmt_bitcoin_txid(const tal_t *ctx, const struct bitcoin_txid *txid)
 	return hexstr;
 }
 
+static char *fmt_bitcoin_outpoint(const tal_t *ctx,
+				  const struct bitcoin_outpoint *outpoint)
+{
+	return tal_fmt(ctx, "%s:%u",
+		       fmt_bitcoin_txid(tmpctx, &outpoint->txid),
+		       outpoint->n);
+}
+
 static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 {
 	u8 *lin = linearize_wtx(ctx, tx);
@@ -687,6 +715,7 @@ static char *fmt_wally_tx(const tal_t *ctx, const struct wally_tx *tx)
 
 REGISTER_TYPE_TO_STRING(bitcoin_tx, fmt_bitcoin_tx);
 REGISTER_TYPE_TO_STRING(bitcoin_txid, fmt_bitcoin_txid);
+REGISTER_TYPE_TO_STRING(bitcoin_outpoint, fmt_bitcoin_outpoint);
 REGISTER_TYPE_TO_STRING(wally_tx, fmt_wally_tx);
 
 void fromwire_bitcoin_txid(const u8 **cursor, size_t *max,
@@ -781,16 +810,15 @@ void towire_bitcoin_tx_output(u8 **pptr, const struct bitcoin_tx_output *output)
 }
 
 bool wally_tx_input_spends(const struct wally_tx_input *input,
-			   const struct bitcoin_txid *txid,
-			   int outnum)
+			   const struct bitcoin_outpoint *outpoint)
 {
 	/* Useful, as tx_part can have some NULL inputs */
 	if (!input)
 		return false;
-	BUILD_ASSERT(sizeof(*txid) == sizeof(input->txhash));
-	if (memcmp(txid, input->txhash, sizeof(*txid)) != 0)
+	BUILD_ASSERT(sizeof(outpoint->txid) == sizeof(input->txhash));
+	if (memcmp(&outpoint->txid, input->txhash, sizeof(outpoint->txid)) != 0)
 		return false;
-	return input->index == outnum;
+	return input->index == outpoint->n;
 }
 
 /* FIXME(cdecker) Make the caller pass in a reference to amount_asset, and

bitcoin/tx.h

@@ -27,6 +27,9 @@ struct bitcoin_outpoint {
 /* Define bitcoin_txid_eq */
 STRUCTEQ_DEF(bitcoin_txid, 0, shad.sha.u);
 
+/* Define bitcoin_outpoint_eq */
+STRUCTEQ_DEF(bitcoin_outpoint, 0, txid.shad.sha.u, n);
+
 struct bitcoin_tx {
 	struct wally_tx *wtx;
 
@@ -109,15 +112,15 @@ void bitcoin_tx_set_locktime(struct bitcoin_tx *tx, u32 locktime);
  * Passing in just the {input_wscript}, we'll generate the scriptPubkey for you.
  * In some cases we may not have the wscript, in which case the scriptPubkey
  * should be provided. We'll check that it's P2WSH before saving it */
-int bitcoin_tx_add_input(struct bitcoin_tx *tx, const struct bitcoin_txid *txid,
-			 u32 outnum, u32 sequence, const u8 *scriptSig,
+int bitcoin_tx_add_input(struct bitcoin_tx *tx,
+			 const struct bitcoin_outpoint *outpoint,
+			 u32 sequence, const u8 *scriptSig,
 			 struct amount_sat amount, const u8 *scriptPubkey,
 			 const u8 *input_wscript);
 
 /* This helps is useful because wally uses a raw byte array for txids */
 bool wally_tx_input_spends(const struct wally_tx_input *input,
-			   const struct bitcoin_txid *txid,
-			   int outnum);
+			   const struct bitcoin_outpoint *outpoint);
 
 struct amount_asset
 wally_tx_output_get_amount(const struct wally_tx_output *output);
@@ -193,17 +196,23 @@ const u8 *bitcoin_tx_input_get_witness(const tal_t *ctx,
 /**
  * Wrap the raw txhash in the wally_tx_input into a bitcoin_txid
  */
+void bitcoin_tx_input_get_outpoint(const struct bitcoin_tx *tx,
+				   int innum,
+				   struct bitcoin_outpoint *outpoint);
+
 void bitcoin_tx_input_get_txid(const struct bitcoin_tx *tx, int innum,
 			       struct bitcoin_txid *out);
 void wally_tx_input_get_txid(const struct wally_tx_input *in,
 			     struct bitcoin_txid *txid);
 
+void wally_tx_input_get_outpoint(const struct wally_tx_input *in,
+				 struct bitcoin_outpoint *outpoint);
+
 /**
  * Overwrite the txhash and index in the wally_tx_input
  */
-void bitcoin_tx_input_set_txid(struct bitcoin_tx *tx, int innum,
-			       const struct bitcoin_txid *txid,
-			       u32 index);
+void bitcoin_tx_input_set_outpoint(struct bitcoin_tx *tx, int innum,
+				   const struct bitcoin_outpoint *outpoint);
 
 /**
  * Check a transaction for consistency.

channeld/channeld.c

@@ -235,12 +235,12 @@ static struct amount_msat advertized_htlc_max(const struct channel *channel)
 	struct amount_msat lower_bound_msat;
 
 	/* This shouldn't fail */
-	if (!amount_sat_sub(&lower_bound, channel->funding,
+	if (!amount_sat_sub(&lower_bound, channel->funding_sats,
 			    channel->config[REMOTE].channel_reserve)) {
 		status_failed(STATUS_FAIL_INTERNAL_ERROR,
 			      "funding %s - remote reserve %s?",
 			      type_to_string(tmpctx, struct amount_sat,
-					     &channel->funding),
+					     &channel->funding_sats),
 			      type_to_string(tmpctx, struct amount_sat,
 					     &channel->config[REMOTE]
 					     .channel_reserve));
@@ -454,7 +454,8 @@ static void make_channel_local_active(struct peer *peer)
 
 	/* Tell gossipd about local channel. */
 	msg = towire_gossip_store_private_channel(NULL,
-						  peer->channel->funding, ann);
+						  peer->channel->funding_sats,
+						  ann);
  	wire_sync_write(peer->pps->gossip_fd, take(msg));
 
 	/* Tell gossipd and the other side what parameters we expect should
@@ -3755,12 +3756,11 @@ static void req_in(struct peer *peer, const u8 *msg)
 static void init_channel(struct peer *peer)
 {
 	struct basepoints points[NUM_SIDES];
-	struct amount_sat funding;
-	u16 funding_txout;
+	struct amount_sat funding_sats;
 	struct amount_msat local_msat;
 	struct pubkey funding_pubkey[NUM_SIDES];
 	struct channel_config conf[NUM_SIDES];
-	struct bitcoin_txid funding_txid;
+	struct bitcoin_outpoint funding;
 	enum side opener;
 	struct existing_htlc **htlcs;
 	bool reconnected;
@@ -3786,8 +3786,8 @@ static void init_channel(struct peer *peer)
 				   &chainparams,
 				   &peer->our_features,
 				   &peer->channel_id,
-				   &funding_txid, &funding_txout,
 				   &funding,
+				   &funding_sats,
 				   &minimum_depth,
 				   &peer->our_blockheight,
 				   &blockheight_states,
@@ -3899,12 +3899,11 @@ static void init_channel(struct peer *peer)
 				 &peer->next_local_per_commit, NULL);
 
 	peer->channel = new_full_channel(peer, &peer->channel_id,
-					 &funding_txid,
-					 funding_txout,
+					 &funding,
 					 minimum_depth,
 					 take(blockheight_states),
 					 lease_expiry,
-					 funding,
+					 funding_sats,
 					 local_msat,
 					 take(fee_states),
 					 &conf[LOCAL], &conf[REMOTE],

channeld/channeld_wire.csv

@@ -15,8 +15,7 @@ msgtype,channeld_init,1000
 msgdata,channeld_init,chainparams,chainparams,
 msgdata,channeld_init,our_features,feature_set,
 msgdata,channeld_init,channel_id,channel_id,
-msgdata,channeld_init,funding_txid,bitcoin_txid,
-msgdata,channeld_init,funding_txout,u16,
+msgdata,channeld_init,funding,bitcoin_outpoint,
 msgdata,channeld_init,funding_satoshi,amount_sat,
 msgdata,channeld_init,minimum_depth,u32,
 msgdata,channeld_init,our_blockheight,u32,