EXPERIMENTAL: quiesence protocol

channeld/channeld.c

@@ -159,6 +159,17 @@ struct peer {
 	/* If master told us to send wrong_funding */
 	struct bitcoin_outpoint *shutdown_wrong_funding;
 
+#if EXPERIMENTAL_FEATURES
+	/* Do we want quiescence? */
+	bool stfu;
+	/* Which side is considered the initiator? */
+	enum side stfu_initiator;
+	/* Has stfu been sent by each side? */
+	bool stfu_sent[NUM_SIDES];
+	/* Updates master asked, which we've deferred while quiescing */
+	struct msg_queue *update_queue;
+#endif
+
 	/* Information used for reestablishment. */
 	bool last_was_revoke;
 	struct changed_htlc *last_sent_commit;
@@ -273,6 +284,101 @@ static struct amount_msat advertized_htlc_max(const struct channel *channel)
 	return lower_bound_msat;
 }
 
+#if EXPERIMENTAL_FEATURES
+static void maybe_send_stfu(struct peer *peer)
+{
+	if (!peer->stfu)
+		return;
+
+	if (!peer->stfu_sent[LOCAL] && !pending_updates(peer->channel, LOCAL)) {
+		u8 *msg = towire_stfu(NULL, &peer->channel_id,
+				      peer->stfu_initiator == LOCAL);
+		sync_crypto_write(peer->pps, take(msg));
+		peer->stfu_sent[LOCAL] = true;
+	}
+
+	if (peer->stfu_sent[LOCAL] && peer->stfu_sent[REMOTE]) {
+		status_unusual("STFU complete: we are quiescent");
+		wire_sync_write(MASTER_FD,
+				towire_channeld_dev_quiesce_reply(tmpctx));
+	}
+}
+
+static void handle_stfu(struct peer *peer, const u8 *stfu)
+{
+	struct channel_id channel_id;
+	u8 remote_initiated;
+
+	if (!fromwire_stfu(stfu, &channel_id, &remote_initiated))
+		peer_failed_warn(peer->pps, &peer->channel_id,
+				 "Bad stfu %s", tal_hex(peer, stfu));
+
+	if (!channel_id_eq(&channel_id, &peer->channel_id)) {
+		peer_failed_err(peer->pps, &channel_id,
+				"Wrong stfu channel_id: expected %s, got %s",
+				type_to_string(tmpctx, struct channel_id,
+					       &peer->channel_id),
+				type_to_string(tmpctx, struct channel_id,
+					       &channel_id));
+	}
+
+	/* Sanity check */
+	if (pending_updates(peer->channel, REMOTE))
+		peer_failed_warn(peer->pps, &peer->channel_id,
+				 "STFU but you still have updates pending?");
+
+	if (!peer->stfu) {
+		peer->stfu = true;
+		if (!remote_initiated)
+			peer_failed_warn(peer->pps, &peer->channel_id,
+					 "Unsolicited STFU but you said"
+					 " you didn't initiate?");
+		peer->stfu_initiator = REMOTE;
+	} else {
+		/* BOLT-quiescent #2:
+		 *
+		 * If both sides send `stfu` simultaneously, they will both
+		 * set `initiator` to `1`, in which case the "initiator" is
+		 * arbitrarily considered to be the channel funder (the sender
+		 * of `open_channel`).
+		 */
+		if (remote_initiated)
+			peer->stfu_initiator = peer->channel->opener;
+	}
+
+	/* BOLT-quiescent #2:
+	 * The receiver of `stfu`:
+	 *   - if it has sent `stfu` then:
+	 *     - MUST now consider the channel to be quiescent
+	 *   - otherwise:
+	 *     - SHOULD NOT send any more update messages.
+	 *     - MUST reply with `stfu` once it can do so.
+	 */
+	peer->stfu_sent[REMOTE] = true;
+
+	maybe_send_stfu(peer);
+}
+
+/* Returns true if we queued this for later handling (steals if true) */
+static bool handle_master_request_later(struct peer *peer, const u8 *msg)
+{
+	if (peer->stfu) {
+		msg_enqueue(peer->update_queue, take(msg));
+		return true;
+	}
+	return false;
+}
+#else /* !EXPERIMENTAL_FEATURES */
+static bool handle_master_request_later(struct peer *peer, const u8 *msg)
+{
+	return false;
+}
+
+static void maybe_send_stfu(struct peer *peer)
+{
+}
+#endif
+
 /* Create and send channel_update to gossipd (and maybe peer) */
 static void send_channel_update(struct peer *peer, int disable_flag)
 {
@@ -952,6 +1058,12 @@ static bool want_fee_update(const struct peer *peer, u32 *target)
 	if (peer->channel->opener != LOCAL)
 		return false;
 
+#if EXPERIMENTAL_FEATURES
+	/* No fee update while quiescing! */
+	if (peer->stfu)
+		return false;
+#endif
+
 	max = approx_max_feerate(peer->channel);
 	val = peer->desired_feerate;
 
@@ -1408,6 +1520,9 @@ static void handle_peer_commit_sig(struct peer *peer, const u8 *msg)
 	send_revocation(peer,
 			&commit_sig, htlc_sigs, changed_htlcs, txs[0]);
 
+	/* We may now be quiescent on our side. */
+	maybe_send_stfu(peer);
+
 	/* This might have synced the feerates: if so, we may want to
 	 * update */
 	if (want_fee_update(peer, NULL))
@@ -1537,6 +1652,9 @@ static void handle_peer_revoke_and_ack(struct peer *peer, const u8 *msg)
 		     type_to_string(tmpctx, struct pubkey,
 				    &peer->old_remote_per_commit));
 
+	/* We may now be quiescent on our side. */
+	maybe_send_stfu(peer);
+
 	start_commit_timer(peer);
 }
 
@@ -1931,6 +2049,11 @@ static void peer_in(struct peer *peer, const u8 *msg)
 		handle_peer_shutdown(peer, msg);
 		return;
 
+#if EXPERIMENTAL_FEATURES
+	case WIRE_STFU:
+		handle_stfu(peer, msg);
+		return;
+#endif
 	case WIRE_INIT:
 	case WIRE_OPEN_CHANNEL:
 	case WIRE_ACCEPT_CHANNEL:
@@ -2958,6 +3081,22 @@ static void channeld_send_custommsg(struct peer *peer, const u8 *msg)
 		master_badmsg(WIRE_CUSTOMMSG_OUT, msg);
 	sync_crypto_write(peer->pps, take(inner));
 }
+
+#if EXPERIMENTAL_FEATURES
+static void handle_dev_quiesce(struct peer *peer, const u8 *msg)
+{
+	if (!fromwire_channeld_dev_quiesce(msg))
+		master_badmsg(WIRE_CHANNELD_DEV_QUIESCE, msg);
+
+	/* Don't do this twice. */
+	if (peer->stfu)
+		status_failed(STATUS_FAIL_MASTER_IO, "dev_quiesce already");
+
+	peer->stfu = true;
+	peer->stfu_initiator = LOCAL;
+	maybe_send_stfu(peer);
+}
+#endif /* EXPERIMENTAL_FEATURES */
 #endif /* DEVELOPER */
 
 static void req_in(struct peer *peer, const u8 *msg)
@@ -2969,18 +3108,28 @@ static void req_in(struct peer *peer, const u8 *msg)
 		handle_funding_depth(peer, msg);
 		return;
 	case WIRE_CHANNELD_OFFER_HTLC:
+		if (handle_master_request_later(peer, msg))
+			return;
 		handle_offer_htlc(peer, msg);
 		return;
 	case WIRE_CHANNELD_FEERATES:
+		if (handle_master_request_later(peer, msg))
+			return;
 		handle_feerates(peer, msg);
 		return;
 	case WIRE_CHANNELD_FULFILL_HTLC:
+		if (handle_master_request_later(peer, msg))
+			return;
 		handle_preimage(peer, msg);
 		return;
 	case WIRE_CHANNELD_FAIL_HTLC:
+		if (handle_master_request_later(peer, msg))
+			return;
 		handle_fail(peer, msg);
 		return;
 	case WIRE_CHANNELD_SPECIFIC_FEERATES:
+		if (handle_master_request_later(peer, msg))
+			return;
 		handle_specific_feerates(peer, msg);
 		return;
 	case WIRE_CHANNELD_SEND_SHUTDOWN:
@@ -2996,9 +3145,15 @@ static void req_in(struct peer *peer, const u8 *msg)
 	case WIRE_CHANNELD_DEV_MEMLEAK:
 		handle_dev_memleak(peer, msg);
 		return;
+	case WIRE_CHANNELD_DEV_QUIESCE:
+#if EXPERIMENTAL_FEATURES
+		handle_dev_quiesce(peer, msg);
+		return;
+#endif /* EXPERIMENTAL_FEATURES */
 #else
 	case WIRE_CHANNELD_DEV_REENABLE_COMMIT:
 	case WIRE_CHANNELD_DEV_MEMLEAK:
+	case WIRE_CHANNELD_DEV_QUIESCE:
 #endif /* DEVELOPER */
 	case WIRE_CHANNELD_INIT:
 	case WIRE_CHANNELD_OFFER_HTLC_REPLY:
@@ -3016,6 +3171,7 @@ static void req_in(struct peer *peer, const u8 *msg)
 	case WIRE_CHANNELD_FAIL_FALLEN_BEHIND:
 	case WIRE_CHANNELD_DEV_MEMLEAK_REPLY:
 	case WIRE_CHANNELD_SEND_ERROR_REPLY:
+	case WIRE_CHANNELD_DEV_QUIESCE_REPLY:
 		break;
 	}
 
@@ -3263,6 +3419,11 @@ int main(int argc, char *argv[])
 	/* We actually received it in the previous daemon, but near enough */
 	peer->last_recv = time_now();
 	peer->last_empty_commitment = 0;
+#if EXPERIMENTAL_FEATURES
+	peer->stfu = false;
+	peer->stfu_sent[LOCAL] = peer->stfu_sent[REMOTE] = false;
+	peer->update_queue = msg_queue_new(peer);
+#endif
 
 	/* We send these to HSM to get real signatures; don't have valgrind
 	 * complain. */

channeld/channeld_wire.csv

@@ -217,3 +217,7 @@ msgdata,channeld_send_error,reason,wirestring,
 
 # Tell master channeld has sent the error message.
 msgtype,channeld_send_error_reply,1108
+
+# Ask channeld to quiesce.
+msgtype,channeld_dev_quiesce,1009
+msgtype,channeld_dev_quiesce_reply,1109

channeld/full_channel.c

@@ -1252,6 +1252,34 @@ static bool adjust_balance(struct balance view_owed[NUM_SIDES][NUM_SIDES],
 	return true;
 }
 
+bool pending_updates(const struct channel *channel, enum side side)
+{
+	struct htlc_map_iter it;
+	const struct htlc *htlc;
+
+	/* Initiator might have fee changes in play. */
+	if (side == channel->opener) {
+		if (!feerate_changes_done(channel->fee_states))
+			return true;
+	}
+
+	for (htlc = htlc_map_first(channel->htlcs, &it);
+	     htlc;
+	     htlc = htlc_map_next(channel->htlcs, &it)) {
+		/* If it's still being added, it's owner added it. */
+		if (htlc_state_flags(htlc->state) & HTLC_ADDING) {
+			if (htlc_owner(htlc) == side)
+				return true;
+		/* If it's being removed, non-owner removed it */
+		} else if (htlc_state_flags(htlc->state) & HTLC_REMOVING) {
+			if (htlc_owner(htlc) != side)
+				return true;
+		}
+	}
+
+	return false;
+}
+
 bool channel_force_htlcs(struct channel *channel,
 			 const struct existing_htlc **htlcs)
 {

channeld/full_channel.h

@@ -254,6 +254,13 @@ bool channel_force_htlcs(struct channel *channel,
  */
 void dump_htlcs(const struct channel *channel, const char *prefix);
 
+/**
+ * pending_updates: does this side have updates pending in channel?
+ * @channel: the channel
+ * @side: the side who is offering or failing/fulfilling HTLC, or feechange
+ */
+bool pending_updates(const struct channel *channel, enum side side);
+
 const char *channel_add_err_name(enum channel_add_err e);
 const char *channel_remove_err_name(enum channel_remove_err e);