Three misc cleanups

bitcoin/script.c

@@ -8,6 +8,7 @@
 #include <ccan/crypto/sha256/sha256.h>
 #include <ccan/endian/endian.h>
 #include <ccan/mem/mem.h>
+#include <common/utils.h>
 
 /* Some standard ops */
 #define OP_0		0x00
@@ -451,8 +452,7 @@ u8 **bitcoin_witness_sig_and_element(const tal_t *ctx,
 
 	witness[0] = stack_sig(witness, sig);
 	witness[1] = tal_dup_arr(witness, u8, elem, elemsize, 0);
-	witness[2] = tal_dup_arr(witness, u8,
-				 witnessscript, tal_count(witnessscript), 0);
+	witness[2] = tal_dup_talarr(witness, u8, witnessscript);
 
 	return witness;
 }
@@ -675,7 +675,7 @@ u8 **bitcoin_witness_htlc_timeout_tx(const tal_t *ctx,
 	witness[1] = stack_sig(witness, remotehtlcsig);
 	witness[2] = stack_sig(witness, localhtlcsig);
 	witness[3] = stack_number(witness, 0);
-	witness[4] = tal_dup_arr(witness, u8, wscript, tal_count(wscript), 0);
+	witness[4] = tal_dup_talarr(witness, u8, wscript);
 
 	return witness;
 }
@@ -692,7 +692,7 @@ u8 **bitcoin_witness_htlc_success_tx(const tal_t *ctx,
 	witness[1] = stack_sig(witness, remotesig);
 	witness[2] = stack_sig(witness, localhtlcsig);
 	witness[3] = stack_preimage(witness, preimage);
-	witness[4] = tal_dup_arr(witness, u8, wscript, tal_count(wscript), 0);
+	witness[4] = tal_dup_talarr(witness, u8, wscript);
 
 	return witness;
 }

common/close_tx.c

@@ -3,6 +3,7 @@
 #include "close_tx.h"
 #include "permute_tx.h"
 #include <assert.h>
+#include <common/utils.h>
 
 struct bitcoin_tx *create_close_tx(const tal_t *ctx,
 				   const struct chainparams *chainparams,
@@ -41,16 +42,14 @@ struct bitcoin_tx *create_close_tx(const tal_t *ctx,
 			     BITCOIN_TX_DEFAULT_SEQUENCE, funding, NULL);
 
 	if (amount_sat_greater_eq(to_us, dust_limit)) {
-		script =
-		    tal_dup_arr(tx, u8, our_script, tal_count(our_script), 0);
+		script = tal_dup_talarr(tx, u8, our_script);
 		/* One output is to us. */
 		bitcoin_tx_add_output(tx, script, to_us);
 		num_outputs++;
 	}
 
 	if (amount_sat_greater_eq(to_them, dust_limit)) {
-		script = tal_dup_arr(tx, u8, their_script,
-				     tal_count(their_script), 0);
+		script = tal_dup_talarr(tx, u8, their_script);
 		/* Other output is to them. */
 		bitcoin_tx_add_output(tx, script, to_them);
 		num_outputs++;

common/json.c

@@ -4,7 +4,10 @@
 #include <bitcoin/preimage.h>
 #include <bitcoin/privkey.h>
 #include <bitcoin/pubkey.h>
+#include <bitcoin/short_channel_id.h>
+#include <bitcoin/tx.h>
 #include <ccan/build_assert/build_assert.h>
+#include <ccan/json_escape/json_escape.h>
 #include <ccan/mem/mem.h>
 #include <ccan/str/hex/hex.h>
 #include <ccan/tal/str/str.h>
@@ -14,6 +17,7 @@
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/overflows.h>
+#include <common/type_to_string.h>
 #include <common/utils.h>
 #include <common/wireaddr.h>
 #include <ctype.h>

common/msg_queue.c

@@ -16,7 +16,7 @@ struct msg_queue *msg_queue_new(const tal_t *ctx)
 
 static void do_enqueue(struct msg_queue *q, const u8 *add TAKES)
 {
-	tal_arr_expand(&q->q, tal_dup_arr(q, u8, add, tal_count(add), 0));
+	tal_arr_expand(&q->q, tal_dup_talarr(q, u8, add));
 
 	/* In case someone is waiting */
 	io_wake(q);

common/onionreply.c

@@ -28,13 +28,13 @@ struct onionreply *dup_onionreply(const tal_t *ctx,
 		return cast_const(struct onionreply *, tal_steal(ctx, r));
 
 	n = tal(ctx, struct onionreply);
-	n->contents = tal_dup_arr(n, u8, r->contents, tal_count(r->contents), 0);
+	n->contents = tal_dup_talarr(n, u8, r->contents);
 	return n;
 }
 
 struct onionreply *new_onionreply(const tal_t *ctx, const u8 *contents TAKES)
 {
 	struct onionreply *r = tal(ctx, struct onionreply);
-	r->contents = tal_dup_arr(r, u8, contents, tal_count(contents), 0);
+	r->contents = tal_dup_talarr(r, u8, contents);
 	return r;
 }

common/param.c

@@ -214,8 +214,7 @@ static bool check_params(struct param *params)
 		return false;
 
 	/* duplicate so we can sort */
-	struct param *copy = tal_dup_arr(params, struct param,
-					 params, tal_count(params), 0);
+	struct param *copy = tal_dup_talarr(params, struct param, params);
 
 	/* check for repeated names and args */
 	if (!check_unique(copy, comp_by_name))

common/read_peer_msg.c

@@ -116,7 +116,7 @@ void handle_gossip_msg(struct per_peer_state *pps, const u8 *msg TAKES)
 		goto out;
 	} else
 		/* It's a raw gossip msg: this copies or takes() */
-		gossip = tal_dup_arr(tmpctx, u8, msg, tal_bytelen(msg), 0);
+		gossip = tal_dup_talarr(tmpctx, u8, msg);
 
 	/* Gossipd can send us gossip messages, OR errors */
 	if (fromwire_peektype(gossip) == WIRE_ERROR) {

common/sphinx.c

@@ -62,8 +62,7 @@ struct sphinx_path {
 struct sphinx_path *sphinx_path_new(const tal_t *ctx, const u8 *associated_data)
 {
 	struct sphinx_path *sp = tal(ctx, struct sphinx_path);
-	sp->associated_data = tal_dup_arr(sp, u8, associated_data,
-					  tal_bytelen(associated_data), 0);
+	sp->associated_data = tal_dup_talarr(sp, u8, associated_data);
 	sp->session_key = NULL;
 	sp->hops = tal_arr(sp, struct sphinx_hop, 0);
 	return sp;
@@ -95,7 +94,7 @@ void sphinx_add_hop(struct sphinx_path *path, const struct pubkey *pubkey,
 		    const u8 *payload TAKES)
 {
 	struct sphinx_hop sp;
-	sp.raw_payload = tal_dup_arr(path, u8, payload, tal_count(payload), 0);
+	sp.raw_payload = tal_dup_talarr(path, u8, payload);
 	sp.pubkey = *pubkey;
 	tal_arr_expand(&path->hops, sp);
 }
@@ -175,11 +174,19 @@ static void xorbytes(uint8_t *d, const uint8_t *a, const uint8_t *b, size_t len)
  */
 static void generate_cipher_stream(void *dst, const u8 *k, size_t dstlen)
 {
-	u8 nonce[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+	const u8 nonce[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
 
 	crypto_stream_chacha20(dst, dstlen, nonce, k);
 }
 
+/* xor cipher stream into dst */
+static void xor_cipher_stream(void *dst, const u8 *k, size_t dstlen)
+{
+	const u8 nonce[8] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+
+	crypto_stream_chacha20_xor(dst, dst, dstlen, nonce, k);
+}
+
 static bool compute_hmac(
 	void *dst,
 	const void *src,
@@ -393,7 +400,6 @@ struct onionpacket *create_onionpacket(
 	struct keyset keys;
 	u8 padkey[KEY_LEN];
 	u8 nexthmac[HMAC_SIZE];
-	u8 stream[ROUTING_INFO_SIZE];
 	struct hop_params *params;
 	struct secret *secrets = tal_arr(ctx, struct secret, num_hops);
 
@@ -432,14 +438,14 @@ struct onionpacket *create_onionpacket(
 	for (i = num_hops - 1; i >= 0; i--) {
 		memcpy(sp->hops[i].hmac, nexthmac, HMAC_SIZE);
 		generate_key_set(&params[i].secret, &keys);
-		generate_cipher_stream(stream, keys.rho, ROUTING_INFO_SIZE);
 
 		/* Rightshift mix-header by FRAME_SIZE */
 		size_t shiftSize = sphinx_hop_size(&sp->hops[i]);
 		memmove(packet->routinginfo + shiftSize, packet->routinginfo,
 			ROUTING_INFO_SIZE-shiftSize);
 		sphinx_write_frame(packet->routinginfo, &sp->hops[i]);
-		xorbytes(packet->routinginfo, packet->routinginfo, stream, ROUTING_INFO_SIZE);
+		xor_cipher_stream(packet->routinginfo, keys.rho,
+				  ROUTING_INFO_SIZE);
 
 		if (i == num_hops - 1) {
 			memcpy(packet->routinginfo + ROUTING_INFO_SIZE - fillerSize, filler, fillerSize);
@@ -479,7 +485,6 @@ struct route_step *process_onionpacket(
 	u8 hmac[HMAC_SIZE];
 	struct keyset keys;
 	u8 blind[BLINDING_FACTOR_SIZE];
-	u8 stream[NUM_STREAM_BYTES];
 	u8 paddedheader[2*ROUTING_INFO_SIZE];
 	size_t payload_size;
 	bigsize_t shift_size;
@@ -498,11 +503,9 @@ struct route_step *process_onionpacket(
 	}
 
 	//FIXME:store seen secrets to avoid replay attacks
-	generate_cipher_stream(stream, keys.rho, sizeof(stream));
-
 	memset(paddedheader, 0, sizeof(paddedheader));
 	memcpy(paddedheader, msg->routinginfo, ROUTING_INFO_SIZE);
-	xorbytes(paddedheader, paddedheader, stream, sizeof(stream));
+	xor_cipher_stream(paddedheader, keys.rho, sizeof(paddedheader));
 
 	compute_blinding_factor(&msg->ephemeralkey, shared_secret, blind);
 	if (!blind_group_element(&step->next->ephemeralkey, &msg->ephemeralkey, blind))
@@ -597,8 +600,6 @@ struct onionreply *wrap_onionreply(const tal_t *ctx,
 				   const struct onionreply *reply)
 {
 	u8 key[KEY_LEN];
-	size_t streamlen = tal_count(reply->contents);
-	u8 stream[streamlen];
 	struct onionreply *result = tal(ctx, struct onionreply);
 
 	/* BOLT #4:
@@ -610,9 +611,8 @@ struct onionreply *wrap_onionreply(const tal_t *ctx,
 	 * The obfuscation step is repeated by every hop along the return path.
 	 */
 	generate_key(key, "ammag", 5, shared_secret);
-	generate_cipher_stream(stream, key, streamlen);
-	result->contents = tal_arr(result, u8, streamlen);
-	xorbytes(result->contents, stream, reply->contents, streamlen);
+	result->contents = tal_dup_talarr(result, u8, reply->contents);
+	xor_cipher_stream(result->contents, key, tal_bytelen(result->contents));
 	return result;
 }
 

common/test/run-features.c

@@ -112,14 +112,14 @@ int main(void)
 
 	/* We can add random odd features, no problem. */
 	for (size_t i = 1; i < 16; i += 2) {
-		bits = tal_dup_arr(tmpctx, u8, lf, tal_count(lf), 0);
+		bits = tal_dup_talarr(tmpctx, u8, lf);
 		set_feature_bit(&bits, i);
 		assert(features_unsupported(bits) == -1);
 	}
 
 	/* We can't add random even features. */
 	for (size_t i = 0; i < 16; i += 2) {
-		bits = tal_dup_arr(tmpctx, u8, lf, tal_count(lf), 0);
+		bits = tal_dup_talarr(tmpctx, u8, lf);
 		set_feature_bit(&bits, i);
 
 		/* Special case for missing compulsory feature */

common/utils.c

@@ -141,3 +141,10 @@ void tal_arr_remove_(void *p, size_t elemsize, size_t n)
 	    len - (elemsize * (n+1)));
     tal_resize((char **)p, len - elemsize);
 }
+
+void *tal_dup_talarr_(const tal_t *ctx, const tal_t *src, const char *label)
+{
+	if (!src)
+		return NULL;
+	return tal_dup_(ctx, src, 1, tal_bytelen(src), 0, label);
+}

common/utils.h

@@ -56,6 +56,17 @@ void clear_softref_(const tal_t *outer, size_t outersize, void **ptr);
 #define tal_arr_remove(p, n) tal_arr_remove_((p), sizeof(**p), (n))
 void tal_arr_remove_(void *p, size_t elemsize, size_t n);
 
+/**
+ * The comon case of duplicating an entire tal array.
+ *
+ * A macro because we must not double-evaluate p.
+ */
+#define tal_dup_talarr(ctx, type, p)					\
+	((type *)tal_dup_talarr_((ctx), tal_typechk_(p, type *),	\
+				 TAL_LABEL(type, "[]")))
+void *tal_dup_talarr_(const tal_t *ctx, const tal_t *src TAKES,
+		      const char *label);
+
 /* Use the POSIX C locale. */
 void setup_locale(void);
 

common/wireaddr.h

@@ -3,7 +3,6 @@
 #include "config.h"
 #include <ccan/short_types/short_types.h>
 #include <ccan/tal/tal.h>
-#include <lightningd/lightningd.h>
 #include <stdbool.h>
 #include <stdlib.h>
 #include <sys/socket.h>

connectd/connectd.c

@@ -395,9 +395,9 @@ static struct io_plan *peer_reconnected(struct io_conn *conn,
 	pr->cs = *cs;
 	pr->addr = *addr;
 
-	/*~ Note that tal_dup_arr() will do handle the take() of features
+	/*~ Note that tal_dup_talarr() will do handle the take() of features
 	 * (turning it into a simply tal_steal() in those cases). */
-	pr->features = tal_dup_arr(pr, u8, features, tal_count(features), 0);
+	pr->features = tal_dup_talarr(pr, u8, features);
 
 	/*~ ccan/io supports waiting on an address: in this case, the key in
 	 * the peer set.  When someone calls `io_wake()` on that address, it

devtools/print_wire.h

@@ -4,6 +4,7 @@
 #include <bitcoin/tx.h>
 #include <common/wireaddr.h>
 #include <wire/gen_peer_wire.h>
+#include <wire/gen_onion_wire.h>
 
 struct tlv_print_record_type {
 	u64 type;

gossipd/gossip_generation.c

@@ -2,7 +2,9 @@
  * generation, man!" */
 #include <ccan/mem/mem.h>
 #include <common/memleak.h>
+#include <common/status.h>
 #include <common/timeout.h>
+#include <common/type_to_string.h>
 #include <common/utils.h>
 #include <common/wireaddr.h>
 #include <errno.h>

gossipd/gossipd.c

@@ -1421,8 +1421,7 @@ static u8 *patch_channel_update(const tal_t *ctx, u8 *channel_update TAKES)
 			tal_free(channel_update);
 		return fixed;
 	} else {
-		return tal_dup_arr(ctx, u8,
-				   channel_update, tal_count(channel_update), 0);
+		return tal_dup_talarr(ctx, u8, channel_update);
 	}
 }
 

gossipd/routing.c

@@ -1650,7 +1650,7 @@ bool routing_add_channel_announcement(struct routing_state *rstate,
 	}
 
 	uc = tal(rstate, struct unupdated_channel);
-	uc->channel_announce = tal_dup_arr(uc, u8, msg, tal_count(msg), 0);
+	uc->channel_announce = tal_dup_talarr(uc, u8, msg);
 	uc->added = gossip_time_now(rstate);
 	uc->index = index;
 	uc->sat = sat;
@@ -1694,8 +1694,7 @@ u8 *handle_channel_announcement(struct routing_state *rstate,
 	pending->updates[0] = NULL;
 	pending->updates[1] = NULL;
 	pending->update_peer_softref[0] = pending->update_peer_softref[1] = NULL;
-	pending->announce = tal_dup_arr(pending, u8,
-					announce, tal_count(announce), 0);
+	pending->announce = tal_dup_talarr(pending, u8, announce);
 	pending->update_timestamps[0] = pending->update_timestamps[1] = 0;
 
 	if (!fromwire_channel_announcement(pending, pending->announce,
@@ -1973,7 +1972,8 @@ static void update_pending(struct pending_cannouncement *pending,
 					  "Replacing existing update");
 			tal_free(pending->updates[direction]);
 		}
-		pending->updates[direction] = tal_dup_arr(pending, u8, update, tal_count(update), 0);
+		pending->updates[direction]
+			= tal_dup_talarr(pending, u8, update);
 		pending->update_timestamps[direction] = timestamp;
 		clear_softref(pending, &pending->update_peer_softref[direction]);
 		set_softref(pending, &pending->update_peer_softref[direction],
@@ -2473,9 +2473,7 @@ bool routing_add_node_announcement(struct routing_state *rstate,
 		pna->index = index;
 		tal_free(pna->node_announcement);
 		clear_softref(pna, &pna->peer_softref);
-		pna->node_announcement = tal_dup_arr(pna, u8, msg,
-						     tal_count(msg),
-						     0);
+		pna->node_announcement = tal_dup_talarr(pna, u8, msg);
 		set_softref(pna, &pna->peer_softref, peer);
 		return true;
 	}

gossipd/test/run-bench-find_route.c

@@ -4,6 +4,7 @@
 #include <ccan/opt/opt.h>
 #include <ccan/tal/str/str.h>
 #include <ccan/time/time.h>
+#include <common/json_stream.h>
 #include <common/pseudorand.h>
 #include <common/status.h>
 #include <common/type_to_string.h>

gossipd/test/run-crc32_of_update.c

@@ -3,6 +3,7 @@ int unused_main(int argc, char *argv[]);
 #include "../queries.c"
 #include "../gossip_generation.c"
 #undef main
+#include <common/json_stream.h>
 #include <stdio.h>
 
 /* AUTOGENERATED MOCKS START */

gossipd/test/run-extended-info.c

@@ -6,6 +6,7 @@
 #include <ccan/str/hex/hex.h>
 #include <common/json.h>
 #include <common/json_helpers.h>
+#include <common/json_stream.h>
 #include <stdio.h>
 
 #ifdef NDEBUG

gossipd/test/run-find_route-specific.c

@@ -5,6 +5,7 @@
  * getchannels:
  * {'channels': [{'active': True, 'short_id': '6990x2x1/1', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 1, 'destination': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'source': '02ea622d5c8d6143f15ed3ce1d501dd0d3d09d3b1c83a44d0034949f8a9ab60f06', 'last_update': 1504064344}, {'active': True, 'short_id': '6989x2x1/0', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 0, 'destination': '03c173897878996287a8100469f954dd820fcd8941daed91c327f168f3329be0bf', 'source': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'last_update': 1504064344}, {'active': True, 'short_id': '6990x2x1/0', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 0, 'destination': '02ea622d5c8d6143f15ed3ce1d501dd0d3d09d3b1c83a44d0034949f8a9ab60f06', 'source': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'last_update': 1504064344}, {'active': True, 'short_id': '6989x2x1/1', 'fee_per_kw': 10, 'delay': 5, 'message_flags': 0, 'channel_flags': 1, 'destination': '0230ad0e74ea03976b28fda587bb75bdd357a1938af4424156a18265167f5e40ae', 'source': '03c173897878996287a8100469f954dd820fcd8941daed91c327f168f3329be0bf', 'last_update': 1504064344}]}
  */
+#include <common/json_stream.h>
 #include <common/status.h>
 
 #include <stdio.h>