Multiple channel support.

channeld/channeld.c

@@ -2682,7 +2682,7 @@ static bool fromwire_channel_reestablish_notlvs(const void *p, struct channel_id
 
 static void peer_reconnect(struct peer *peer,
 			   const struct secret *last_remote_per_commit_secret,
-			   u8 *reestablish_only)
+			   bool reestablish_only)
 {
 	struct channel_id channel_id;
 	/* Note: BOLT #2 uses these names! */
@@ -2820,23 +2820,23 @@ static void peer_reconnect(struct peer *peer,
 
 	peer_billboard(false, "Sent reestablish, waiting for theirs");
 
-	/* If they sent reestablish, we analyze it for courtesy, but also
-	 * in case *they* are ahead of us! */
-	if (reestablish_only) {
-		msg = reestablish_only;
-		goto got_reestablish;
-	}
-
 	/* Read until they say something interesting (don't forward
 	 * gossip *to* them yet: we might try sending channel_update
 	 * before we've reestablished channel). */
 	do {
 		clean_tmpctx();
 		msg = peer_read(tmpctx, peer->pps);
+
+		/* connectd promised us the msg was reestablish? */
+		if (reestablish_only) {
+			if (fromwire_peektype(msg) != WIRE_CHANNEL_REESTABLISH)
+				status_failed(STATUS_FAIL_INTERNAL_ERROR,
+					      "Expected reestablish, got: %s",
+					      tal_hex(tmpctx, msg));
+		}
 	} while (handle_peer_error(peer->pps, &peer->channel_id, msg) ||
 		 capture_premature_msg(&premature_msgs, msg));
 
-got_reestablish:
 #if EXPERIMENTAL_FEATURES
 	recv_tlvs = tlv_channel_reestablish_tlvs_new(tmpctx);
 
@@ -3730,7 +3730,7 @@ static void init_channel(struct peer *peer)
 	secp256k1_ecdsa_signature *remote_ann_node_sig;
 	secp256k1_ecdsa_signature *remote_ann_bitcoin_sig;
 	struct penalty_base *pbases;
-	u8 *reestablish_only;
+	bool reestablish_only;
 	struct channel_type *channel_type;
 	u32 *dev_disable_commit; /* Always NULL */
 	bool dev_fast_gossip;

channeld/channeld_wire.csv

@@ -80,8 +80,7 @@ msgdata,channeld_init,dev_fail_process_onionpacket,bool,
 msgdata,channeld_init,dev_disable_commit,?u32,
 msgdata,channeld_init,num_penalty_bases,u32,
 msgdata,channeld_init,pbases,penalty_base,num_penalty_bases
-msgdata,channeld_init,reestablish_only_len,u16,
-msgdata,channeld_init,reestablish_only,u8,reestablish_only_len
+msgdata,channeld_init,reestablish_only,bool,
 msgdata,channeld_init,channel_update_len,u16,
 msgdata,channeld_init,channel_update,u8,channel_update_len
 

connectd/connectd.c

@@ -307,13 +307,13 @@ static struct peer *new_peer(struct daemon *daemon,
 	peer->id = *id;
 	peer->cs = *cs;
 	peer->final_msg = NULL;
-	peer->subd_in = NULL;
+	peer->subds = tal_arr(peer, struct subd *, 0);
 	peer->peer_in = NULL;
 	peer->sent_to_peer = NULL;
 	peer->urgent = false;
-	peer->told_to_close = false;
+	peer->ready_to_die = false;
+	peer->active = false;
 	peer->peer_outq = msg_queue_new(peer, false);
-	peer->subd_outq = msg_queue_new(peer, false);
 
 #if DEVELOPER
 	peer->dev_writes_enabled = NULL;
@@ -322,11 +322,6 @@ static struct peer *new_peer(struct daemon *daemon,
 
 	peer->to_peer = conn;
 
-	/* Aim for connection to shuffle data back and forth: sets up
-	 * peer->to_subd */
-	if (!multiplex_subd_setup(peer, fd_for_subd))
-		return tal_free(peer);
-
 	/* Now we own it */
 	tal_steal(peer, peer->to_peer);
 	peer_htable_add(&daemon->peers, peer);
@@ -425,9 +420,9 @@ struct io_plan *peer_connected(struct io_conn *conn,
 
 	/*~ daemon_conn is a message queue for inter-daemon communication: we
 	 * queue up the `connect_peer_connected` message to tell lightningd
-	 * we have connected, and give the peer fd. */
+	 * we have connected.  Once it says something interesting, we tell
+	 * it that, too. */
 	daemon_conn_send(daemon->master, take(msg));
-	daemon_conn_send_fd(daemon->master, subd_fd);
 
 	/*~ Now we set up this connection to read/write from subd */
 	return multiplex_peer_setup(conn, peer);
@@ -1792,10 +1787,16 @@ static void try_connect_peer(struct daemon *daemon,
 	existing = peer_htable_get(&daemon->peers, id);
 	if (existing) {
 		/* If it's exiting now, we've raced: reconnect after */
-		if (existing->to_subd
+		if ((tal_count(existing->subds) != 0 || !existing->active)
 		    && existing->to_peer
-		    && !existing->told_to_close)
+		    && !existing->ready_to_die) {
+			/* Tell it it's already connected so it doesn't
+			 * wait forever. */
+			daemon_conn_send(daemon->master,
+					 take(towire_connectd_peer_already_connected
+					      (NULL, id)));
 			return;
+		}
 	}
 
 	/* If we're trying to connect it right now, that's OK. */
@@ -1892,14 +1893,20 @@ void peer_conn_closed(struct peer *peer)
 	struct connecting *connect = find_connecting(peer->daemon, &peer->id);
 
 	/* These should be closed already! */
-	assert(!peer->to_subd);
+	assert(tal_count(peer->subds) == 0);
 	assert(!peer->to_peer);
-	assert(peer->told_to_close);
+	assert(peer->ready_to_die || !peer->active);
+
+	status_peer_debug(&peer->id, "peer_conn_closed");
 
 	/* Tell gossipd to stop asking this peer gossip queries */
 	daemon_conn_send(peer->daemon->gossipd,
 			 take(towire_gossipd_peer_gone(NULL, &peer->id)));
 
+	/* Tell lightningd it's really disconnected */
+	daemon_conn_send(peer->daemon->master,
+			 take(towire_connectd_peer_disconnect_done(NULL,
+								   &peer->id)));
 	/* Wake up in case there's a reconnecting peer waiting in io_wait. */
 	io_wake(peer);
 
@@ -1914,34 +1921,26 @@ void peer_conn_closed(struct peer *peer)
 		try_connect_one_addr(connect);
 }
 
-/* A peer is gone: clean things up. */
-static void cleanup_dead_peer(struct daemon *daemon, const struct node_id *id)
+/* lightningd tells us a peer should be disconnected. */
+static void peer_discard(struct daemon *daemon, const u8 *msg)
 {
+	struct node_id id;
 	struct peer *peer;
 
-	/* We should stay in sync with lightningd at all times. */
-	peer = peer_htable_get(&daemon->peers, id);
+	if (!fromwire_connectd_discard_peer(msg, &id))
+		master_badmsg(WIRE_CONNECTD_DISCARD_PEER, msg);
+
+	/* We should stay in sync with lightningd, but this can happen
+	 * under stress. */
+	peer = peer_htable_get(&daemon->peers, &id);
 	if (!peer)
-		status_failed(STATUS_FAIL_INTERNAL_ERROR,
-			      "peer_disconnected unknown peer: %s",
-			      type_to_string(tmpctx, struct node_id, id));
-	status_peer_debug(id, "disconnect");
+		return;
+	status_peer_debug(&id, "disconnect");
 
 	/* When it's finished, it will call peer_conn_closed() */
 	close_peer_conn(peer);
 }
 
-/* lightningd tells us a peer has disconnected. */
-static void peer_disconnected(struct daemon *daemon, const u8 *msg)
-{
-	struct node_id id;
-
-	if (!fromwire_connectd_peer_disconnected(msg, &id))
-		master_badmsg(WIRE_CONNECTD_PEER_DISCONNECTED, msg);
-
-	cleanup_dead_peer(daemon, &id);
-}
-
 /* lightningd tells us to send a msg and disconnect. */
 static void peer_final_msg(struct io_conn *conn,
 			   struct daemon *daemon, const u8 *msg)
@@ -2002,8 +2001,8 @@ static struct io_plan *recv_req(struct io_conn *conn,
 		connect_to_peer(daemon, msg);
 		goto out;
 
-	case WIRE_CONNECTD_PEER_DISCONNECTED:
-		peer_disconnected(daemon, msg);
+	case WIRE_CONNECTD_DISCARD_PEER:
+		peer_discard(daemon, msg);
 		goto out;
 
 	case WIRE_CONNECTD_PEER_FINAL_MSG:
@@ -2022,6 +2021,10 @@ static struct io_plan *recv_req(struct io_conn *conn,
 		send_custommsg(daemon, msg);
 		goto out;
 
+	case WIRE_CONNECTD_PEER_MAKE_ACTIVE:
+		peer_make_active(daemon, msg);
+		goto out;
+
 	case WIRE_CONNECTD_DEV_MEMLEAK:
 #if DEVELOPER
 		dev_connect_memleak(daemon, msg);
@@ -2031,12 +2034,15 @@ static struct io_plan *recv_req(struct io_conn *conn,
 	case WIRE_CONNECTD_INIT_REPLY:
 	case WIRE_CONNECTD_ACTIVATE_REPLY:
 	case WIRE_CONNECTD_PEER_CONNECTED:
+	case WIRE_CONNECTD_PEER_ALREADY_CONNECTED:
+	case WIRE_CONNECTD_PEER_ACTIVE:
 	case WIRE_CONNECTD_RECONNECTED:
 	case WIRE_CONNECTD_CONNECT_FAILED:
 	case WIRE_CONNECTD_DEV_MEMLEAK_REPLY:
 	case WIRE_CONNECTD_PING_REPLY:
 	case WIRE_CONNECTD_GOT_ONIONMSG_TO_US:
 	case WIRE_CONNECTD_CUSTOMMSG_IN:
+	case WIRE_CONNECTD_PEER_DISCONNECT_DONE:
 		break;
 	}
 

connectd/connectd.h

@@ -5,6 +5,7 @@
 #include <ccan/crypto/siphash24/siphash24.h>
 #include <ccan/htable/htable_type.h>
 #include <ccan/timer/timer.h>
+#include <common/channel_id.h>
 #include <common/crypto_state.h>
 #include <common/node_id.h>
 #include <common/pseudorand.h>
@@ -52,23 +53,27 @@ struct peer {
 	/* Connection to the peer */
 	struct io_conn *to_peer;
 
-	/* Connection to the subdaemon */
-	struct io_conn *to_subd;
+	/* Connections to the subdaemons */
+	struct subd **subds;
 
 	/* Final message to send to peer (and hangup) */
 	u8 *final_msg;
 
-	/* Set when we want to close. */
-	bool told_to_close;
+	/* Set once lightningd says it's OK to close (subd tells it
+	 * it's done). */
+	bool ready_to_die;
+
+	/* Has this ever been active?  (i.e. ever had a subd attached?) */
+	bool active;
 
 	/* When socket has Nagle overridden */
 	bool urgent;
 
-	/* Input buffers. */
-	u8 *subd_in, *peer_in;
+	/* Input buffer. */
+	u8 *peer_in;
 
-	/* Output buffers. */
-	struct msg_queue *subd_outq, *peer_outq;
+	/* Output buffer. */
+	struct msg_queue *peer_outq;
 
 	/* Peer sent buffer (for freeing after sending) */
 	const u8 *sent_to_peer;

connectd/connectd_wire.csv

@@ -1,4 +1,5 @@
 #include <bitcoin/block.h>
+#include <common/channel_id.h>
 #include <common/cryptomsg.h>
 #include <common/features.h>
 #include <common/node_id.h>
@@ -62,7 +63,7 @@ msgdata,connectd_connect_failed,failreason,wirestring,
 msgdata,connectd_connect_failed,seconds_to_delay,u32,
 msgdata,connectd_connect_failed,addrhint,?wireaddr_internal,
 
-# Connectd -> master: we got a peer. Plus fd for peer daemon
+# Connectd -> master: we got a peer.
 msgtype,connectd_peer_connected,2002
 msgdata,connectd_peer_connected,id,node_id,
 msgdata,connectd_peer_connected,addr,wireaddr_internal,
@@ -71,16 +72,36 @@ msgdata,connectd_peer_connected,incoming,bool,
 msgdata,connectd_peer_connected,flen,u16,
 msgdata,connectd_peer_connected,features,u8,flen
 
-# master -> connectd: peer has disconnected.
-msgtype,connectd_peer_disconnected,2015
-msgdata,connectd_peer_disconnected,id,node_id,
+# connectd -> master: peer disconnected.
+msgtype,connectd_peer_disconnect_done,2006
+msgdata,connectd_peer_disconnect_done,id,node_id,
+
+# Master -> connectd: make peer active immediately (we want to talk)
+msgtype,connectd_peer_make_active,2004
+msgdata,connectd_peer_make_active,id,node_id,
+msgdata,connectd_peer_make_active,channel_id,channel_id,
+
+# Connectd -> master: peer said something interesting (or you said make_active)
+# Plus fd for peer daemon.
+msgtype,connectd_peer_active,2005
+msgdata,connectd_peer_active,id,node_id,
+msgdata,connectd_peer_active,msgtype,?u16,
+msgdata,connectd_peer_active,channel_id,channel_id,
+
+# master -> connectd: peer no longer wanted, you can disconnect.
+msgtype,connectd_discard_peer,2015
+msgdata,connectd_discard_peer,id,node_id,
 
 # master -> connectd: give message to peer and disconnect.
 msgtype,connectd_peer_final_msg,2003
 msgdata,connectd_peer_final_msg,id,node_id,
 msgdata,connectd_peer_final_msg,len,u16,
 msgdata,connectd_peer_final_msg,msg,u8,len
 
+# connectd->master: You said to connect, but we already were.
+msgtype,connectd_peer_already_connected,2007
+msgdata,connectd_peer_already_connected,id,node_id,
+
 # master -> connectd: do you have a memleak?
 msgtype,connectd_dev_memleak,2033