Merge branch 'master' of ssh://gnunet.org/gnunet
[oweals/gnunet.git] / src / cadet / gnunet-service-cadet-new_tunnels.c
index 4d4cdc2dcaa7fc24454faf96259ba1a8c34d8343..d50860629312b54edd47d843109a8bee3a97927d 100644 (file)
  * @author Christian Grothoff
  *
  * FIXME:
- * - KX:
- *   + clean up KX logic, including adding sender authentication
- *   + implement rekeying
- *   + check KX estate machine -- make sure it is never stuck!
- * - connection management
- *   + properly (evaluate, kill old ones, search for new ones)
- *   + when managing connections, distinguish those that
- *     have (recently) had traffic from those that were
- *     never ready (or not recently)
+ * - proper connection evaluation during connection management:
+ *   + consider quality (or quality spread?) of current connection set
+ *     when deciding how often to do maintenance
+ *   + interact with PEER to drive DHT GET/PUT operations based
+ *     on how much we like our connections
  */
 #include "platform.h"
 #include "gnunet_util_lib.h"
  */
 #define IDLE_DESTROY_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
 
+/**
+ * How long do we wait initially before retransmitting the KX?
+ * TODO: replace by 2 RTT if/once we have connection-level RTT data!
+ */
+#define INITIAL_KX_RETRY_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 250)
+
 /**
  * Maximum number of skipped keys we keep in memory per tunnel.
  */
@@ -173,12 +175,12 @@ struct CadetTunnelAxolotl
   /**
    * ECDH for key exchange (A0 / B0).
    */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *kx_0;
+  struct GNUNET_CRYPTO_EcdhePrivateKey kx_0;
 
   /**
    * ECDH Ratchet key (our private key in the current DH).
    */
-  struct GNUNET_CRYPTO_EcdhePrivateKey *DHRs;
+  struct GNUNET_CRYPTO_EcdhePrivateKey DHRs;
 
   /**
    * ECDH Ratchet key (other peer's public key in the current DH).
@@ -268,7 +270,7 @@ struct CadetTunnelQueueEntry
   /**
    * Continuation to call once sent (on the channel layer).
    */
-  GNUNET_SCHEDULER_TaskCallback cont;
+  GCT_SendContinuation cont;
 
   /**
    * Closure for @c cont.
@@ -359,14 +361,24 @@ struct CadetTunnel
   struct GNUNET_MQ_Handle *mq;
 
   /**
-   * DLL of connections that are actively used to reach the destination peer.
+   * DLL of ready connections that are actively used to reach the destination peer.
    */
-  struct CadetTConnection *connection_head;
+  struct CadetTConnection *connection_ready_head;
 
   /**
-   * DLL of connections that are actively used to reach the destination peer.
+   * DLL of ready connections that are actively used to reach the destination peer.
    */
-  struct CadetTConnection *connection_tail;
+  struct CadetTConnection *connection_ready_tail;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_head;
+
+  /**
+   * DLL of connections that we maintain that might be used to reach the destination peer.
+   */
+  struct CadetTConnection *connection_busy_tail;
 
   /**
    * Channels inside this tunnel. Maps
@@ -389,6 +401,13 @@ struct CadetTunnel
    */
   struct CadetTunnelQueueEntry *tq_tail;
 
+  /**
+   * Identification of the connection from which we are currently processing
+   * a message. Only valid (non-NULL) during #handle_decrypted() and the
+   * handle-*()-functions called from our @e mq during that function.
+   */
+  struct CadetTConnection *current_ct;
+
   /**
    * How long do we wait until we retry the KX?
    */
@@ -400,9 +419,14 @@ struct CadetTunnel
   struct GNUNET_TIME_Absolute next_kx_attempt;
 
   /**
-   * Number of connections in the @e connection_head DLL.
+   * Number of connections in the @e connection_ready_head DLL.
    */
-  unsigned int num_connections;
+  unsigned int num_ready_connections;
+
+  /**
+   * Number of connections in the @e connection_busy_head DLL.
+   */
+  unsigned int num_busy_connections;
 
   /**
    * How often have we tried and failed to decrypt a message using
@@ -421,9 +445,39 @@ struct CadetTunnel
    */
   enum CadetTunnelEState estate;
 
+  /**
+   * Force triggering KX_AUTH independent of @e estate.
+   */
+  int kx_auth_requested;
+
 };
 
 
+/**
+ * Connection @a ct is now unready, clear it's ready flag
+ * and move it from the ready DLL to the busy DLL.
+ *
+ * @param ct connection to move to unready status
+ */
+static void
+mark_connection_unready (struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = ct->t;
+
+  GNUNET_assert (GNUNET_YES == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                               t->connection_ready_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_ready_connections);
+  t->num_ready_connections--;
+  ct->is_ready = GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  t->num_busy_connections++;
+}
+
+
 /**
  * Get the static string for the peer this tunnel is directed.
  *
@@ -460,17 +514,24 @@ estate2s (enum CadetTunnelEState es)
 
   switch (es)
   {
-    case CADET_TUNNEL_KEY_UNINITIALIZED:
-      return "CADET_TUNNEL_KEY_UNINITIALIZED";
-    case CADET_TUNNEL_KEY_SENT:
-      return "CADET_TUNNEL_KEY_SENT";
-    case CADET_TUNNEL_KEY_PING:
-      return "CADET_TUNNEL_KEY_PING";
-    case CADET_TUNNEL_KEY_OK:
-      return "CADET_TUNNEL_KEY_OK";
-    default:
-      SPRINTF (buf, "%u (UNKNOWN STATE)", es);
-      return buf;
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    return "CADET_TUNNEL_KEY_UNINITIALIZED";
+  case CADET_TUNNEL_KEY_AX_RECV:
+    return "CADET_TUNNEL_KEY_AX_RECV";
+  case CADET_TUNNEL_KEY_AX_SENT:
+    return "CADET_TUNNEL_KEY_AX_SENT";
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    return "CADET_TUNNEL_KEY_AX_SENT_AND_RECV";
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    return "CADET_TUNNEL_KEY_AX_AUTH_SENT";
+  case CADET_TUNNEL_KEY_OK:
+    return "CADET_TUNNEL_KEY_OK";
+  default:
+    GNUNET_snprintf (buf,
+                     sizeof (buf),
+                     "%u (UNKNOWN STATE)",
+                     es);
+    return buf;
   }
 }
 
@@ -526,9 +587,9 @@ lookup_channel (struct CadetTunnel *t,
  * @return Number of connections created, either being established or ready.
  */
 unsigned int
-GCT_count_any_connections (struct CadetTunnel *t)
+GCT_count_any_connections (const struct CadetTunnel *t)
 {
-  return t->num_connections;
+  return t->num_ready_connections + t->num_busy_connections;
 }
 
 
@@ -542,25 +603,11 @@ GCT_count_any_connections (struct CadetTunnel *t)
 static struct CadetTConnection *
 get_ready_connection (struct CadetTunnel *t)
 {
-  for (struct CadetTConnection *pos = t->connection_head;
-       NULL != pos;
-       pos = pos->next)
-    if (GNUNET_YES == pos->is_ready)
-    {
-      if (pos != t->connection_tail)
-      {
-        /* move 'pos' to the end, so we try other ready connections
-           first next time (round-robin, modulo availability) */
-        GNUNET_CONTAINER_DLL_remove (t->connection_head,
-                                     t->connection_tail,
-                                     pos);
-        GNUNET_CONTAINER_DLL_insert_tail (t->connection_head,
-                                          t->connection_tail,
-                                          pos);
-      }
-      return pos;
-    }
-  return NULL;
+  struct CadetTConnection *hd = t->connection_ready_head;
+
+  GNUNET_assert ( (NULL == hd) ||
+                  (GNUNET_YES == hd->is_ready) );
+  return hd;
 }
 
 
@@ -601,8 +648,10 @@ trigger_transmissions (void *cls);
 static void
 new_ephemeral (struct CadetTunnelAxolotl *ax)
 {
-  GNUNET_free_non_null (ax->DHRs);
-  ax->DHRs = GNUNET_CRYPTO_ecdhe_key_create ();
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating new ephemeral ratchet key (DHRs)\n");
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdhe_key_create2 (&ax->DHRs));
 }
 
 
@@ -737,7 +786,7 @@ t_ax_encrypt (struct CadetTunnelAxolotl *ax,
     ax->HKs = ax->NHKs;
 
     /* RK, NHKs, CKs = KDF( HMAC-HASH(RK, DH(DHRs, DHRr)) ) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs,
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
                             &ax->DHRr,
                             &dh);
     t_ax_hmac_hash (&ax->RK,
@@ -1143,7 +1192,7 @@ t_ax_decrypt_and_validate (struct CadetTunnelAxolotl *ax,
                    PNp);
 
     /* RKp, NHKp, CKp = KDF (HMAC-HASH (RK, DH (DHRp, DHRs))) */
-    GNUNET_CRYPTO_ecc_ecdh (ax->DHRs,
+    GNUNET_CRYPTO_ecc_ecdh (&ax->DHRs,
                             DHRp,
                             &dh);
     t_ax_hmac_hash (&ax->RK,
@@ -1227,10 +1276,10 @@ GCT_change_estate (struct CadetTunnel *t,
 
   t->estate = state;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s estate changed from %d to %d\n",
+       "%s estate changed from %s to %s\n",
        GCT_2s (t),
-       old,
-       state);
+       estate2s (old),
+       estate2s (state));
 
   if ( (CADET_TUNNEL_KEY_OK != old) &&
        (CADET_TUNNEL_KEY_OK == t->estate) )
@@ -1244,6 +1293,10 @@ GCT_change_estate (struct CadetTunnel *t,
     GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
                                              &notify_tunnel_up_cb,
                                              t);
+    if (NULL != t->send_task)
+      GNUNET_SCHEDULER_cancel (t->send_task);
+    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
+                                             t);
   }
 }
 
@@ -1251,57 +1304,135 @@ GCT_change_estate (struct CadetTunnel *t,
 /**
  * Send a KX message.
  *
- * FIXME: does not take care of sender-authentication yet!
- *
- * @param t Tunnel on which to send it.
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
  * @param ax axolotl key context to use
- * @param force_reply Force the other peer to reply with a KX message.
  */
 static void
 send_kx (struct CadetTunnel *t,
-         struct CadetTunnelAxolotl *ax,
-         int force_reply)
+         struct CadetTConnection *ct,
+         struct CadetTunnelAxolotl *ax)
 {
-  struct CadetTConnection *ct;
   struct CadetConnection *cc;
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_TunnelKeyExchangeMessage *msg;
   enum GNUNET_CADET_KX_Flags flags;
 
-  ct = get_ready_connection (t);
+  if ( (NULL == ct) ||
+       (GNUNET_NO == ct->is_ready) )
+    ct = get_ready_connection (t);
   if (NULL == ct)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Wanted to send KX on tunnel %s, but no connection is ready, deferring\n",
-         GCT_2s (t));
+         "Wanted to send %s in state %s, but no connection is ready, deferring\n",
+         GCT_2s (t),
+         estate2s (t->estate));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
     return;
   }
   cc = ct->cc;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Sending KX on tunnel %s using connection %s\n",
+       "Sending KX on %s via %s in state %s\n",
        GCT_2s (t),
-       GCC_2s (ct->cc));
-
-  // GNUNET_assert (GNUNET_NO == GCT_is_loopback (t));
+       GCC_2s (cc),
+       estate2s (t->estate));
   env = GNUNET_MQ_msg (msg,
                        GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX);
-  flags = GNUNET_CADET_KX_FLAG_NONE;
-  if (GNUNET_YES == force_reply)
-    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
+  flags = GNUNET_CADET_KX_FLAG_FORCE_REPLY; /* always for KX */
   msg->flags = htonl (flags);
   msg->cid = *GCC_get_id (cc);
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->kx_0,
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
                                       &msg->ephemeral_key);
-  GNUNET_CRYPTO_ecdhe_key_get_public (ax->DHRs,
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
                                       &msg->ratchet_key);
-  ct->is_ready = GNUNET_NO;
-  GCC_transmit (cc,
-                env);
+  mark_connection_unready (ct);
   t->kx_retry_delay = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
   t->next_kx_attempt = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
   if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
     GCT_change_estate (t,
-                       CADET_TUNNEL_KEY_SENT);
+                       CADET_TUNNEL_KEY_AX_SENT);
+  else if (CADET_TUNNEL_KEY_AX_RECV == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+  GCC_transmit (cc,
+                env);
+}
+
+
+/**
+ * Send a KX_AUTH message.
+ *
+ * @param t tunnel on which to send the KX_AUTH
+ * @param ct Tunnel and connection on which to send the KX_AUTH, NULL if
+ *           we are to find one that is ready.
+ * @param ax axolotl key context to use
+ * @param force_reply Force the other peer to reply with a KX_AUTH message
+ *         (set if we would like to transmit right now, but cannot)
+ */
+static void
+send_kx_auth (struct CadetTunnel *t,
+              struct CadetTConnection *ct,
+              struct CadetTunnelAxolotl *ax,
+              int force_reply)
+{
+  struct CadetConnection *cc;
+  struct GNUNET_MQ_Envelope *env;
+  struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg;
+  enum GNUNET_CADET_KX_Flags flags;
+
+  if ( (NULL == ct) ||
+       (GNUNET_NO == ct->is_ready) )
+    ct = get_ready_connection (t);
+  if (NULL == ct)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Wanted to send KX_AUTH on %s, but no connection is ready, deferring\n",
+         GCT_2s (t));
+    t->next_kx_attempt = GNUNET_TIME_absolute_get ();
+    t->kx_auth_requested = GNUNET_YES; /* queue KX_AUTH independent of estate */
+    return;
+  }
+  t->kx_auth_requested = GNUNET_NO; /* clear flag */
+  cc = ct->cc;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending KX_AUTH on %s using %s\n",
+       GCT_2s (t),
+       GCC_2s (ct->cc));
+
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_TUNNEL_KX_AUTH);
+  flags = GNUNET_CADET_KX_FLAG_NONE;
+  if (GNUNET_YES == force_reply)
+    flags |= GNUNET_CADET_KX_FLAG_FORCE_REPLY;
+  msg->kx.flags = htonl (flags);
+  msg->kx.cid = *GCC_get_id (cc);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->kx_0,
+                                      &msg->kx.ephemeral_key);
+  GNUNET_CRYPTO_ecdhe_key_get_public (&ax->DHRs,
+                                      &msg->kx.ratchet_key);
+  /* Compute authenticator (this is the main difference to #send_kx()) */
+  GNUNET_CRYPTO_hash (&ax->RK,
+                      sizeof (ax->RK),
+                      &msg->auth);
+
+  /* Compute when to be triggered again; actual job will
+     be scheduled via #connection_ready_cb() */
+  t->kx_retry_delay
+    = GNUNET_TIME_STD_BACKOFF (t->kx_retry_delay);
+  t->next_kx_attempt
+    = GNUNET_TIME_relative_to_absolute (t->kx_retry_delay);
+
+  /* Send via cc, mark it as unready */
+  mark_connection_unready (ct);
+
+  /* Update state machine, unless we are already OK */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_AUTH_SENT);
+
+  GCC_transmit (cc,
+                env);
 }
 
 
@@ -1317,8 +1448,8 @@ cleanup_ax (struct CadetTunnelAxolotl *ax)
     delete_skipped_key (ax,
                         ax->skipped_head);
   GNUNET_assert (0 == ax->skipped);
-  GNUNET_free_non_null (ax->kx_0);
-  GNUNET_free_non_null (ax->DHRs);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->kx_0);
+  GNUNET_CRYPTO_ecdhe_key_clear (&ax->DHRs);
 }
 
 
@@ -1378,7 +1509,7 @@ update_ax_by_kx (struct CadetTunnelAxolotl *ax,
   }
   else
   {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* B0 */
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* B0 */
                               &pid->public_key,    /* A */
                               &key_material[0]);
   }
@@ -1386,7 +1517,7 @@ update_ax_by_kx (struct CadetTunnelAxolotl *ax,
   /* ECDH A0 B */
   if (GNUNET_YES == am_I_alice)
   {
-    GNUNET_CRYPTO_ecdh_eddsa (ax->kx_0,            /* A0 */
+    GNUNET_CRYPTO_ecdh_eddsa (&ax->kx_0,            /* A0 */
                               &pid->public_key,    /* B */
                               &key_material[1]);
   }
@@ -1402,7 +1533,7 @@ update_ax_by_kx (struct CadetTunnelAxolotl *ax,
   /* ECDH A0 B0 */
   /* (This is the triple-DH, we could probably safely skip this,
      as A0/B0 are already in the key material.) */
-  GNUNET_CRYPTO_ecc_ecdh (ax->kx_0,             /* A0 or B0 */
+  GNUNET_CRYPTO_ecc_ecdh (&ax->kx_0,             /* A0 or B0 */
                           ephemeral_key,  /* B0 or A0 */
                           &key_material[2]);
 
@@ -1445,6 +1576,95 @@ update_ax_by_kx (struct CadetTunnelAxolotl *ax,
 }
 
 
+/**
+ * Try to redo the KX or KX_AUTH handshake, if we can.
+ *
+ * @param cls the `struct CadetTunnel` to do KX for.
+ */
+static void
+retry_kx (void *cls)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetTunnelAxolotl *ax;
+
+  t->kx_task = NULL;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Trying to make KX progress on %s in state %s\n",
+       GCT_2s (t),
+       estate2s (t->estate));
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED: /* first attempt */
+  case CADET_TUNNEL_KEY_AX_SENT:       /* trying again */
+    send_kx (t,
+             NULL,
+             &t->ax);
+    break;
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* We are responding, so only require reply
+       if WE have a channel waiting. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Send AX_AUTH so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* How can this be? */
+      GNUNET_break (0);
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  (0 == GCT_count_channels (t))
+                  ? GNUNET_NO
+                  : GNUNET_YES);
+    break;
+  case CADET_TUNNEL_KEY_OK:
+    /* Must have been the *other* peer asking us to
+       respond with a KX_AUTH. */
+    if (NULL != t->unverified_ax)
+    {
+      /* Sending AX_AUTH in response to AX so we might get this one verified */
+      ax = t->unverified_ax;
+    }
+    else
+    {
+      /* Sending AX_AUTH in response to AX_AUTH */
+      ax = &t->ax;
+    }
+    send_kx_auth (t,
+                  NULL,
+                  ax,
+                  GNUNET_NO);
+    break;
+  }
+}
+
+
 /**
  * Handle KX message that lacks authentication (and which will thus
  * only be considered authenticated after we respond with our own
@@ -1461,50 +1681,64 @@ GCT_handle_kx (struct CadetTConnection *ct,
   struct CadetTunnelAxolotl *ax;
   int ret;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Handling KX message for tunnel %s\n",
-       GCT_2s (t));
+  if (0 ==
+      memcmp (&t->ax.DHRr,
+              &msg->ratchet_key,
+              sizeof (msg->ratchet_key)))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Got duplicate KX. Firing back KX_AUTH.\n");
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_NO);
+    return;
+  }
 
   /* We only keep ONE unverified KX around, so if there is an existing one,
      clean it up. */
   if (NULL != t->unverified_ax)
   {
+    if (0 ==
+        memcmp (&t->unverified_ax->DHRr,
+                &msg->ratchet_key,
+                sizeof (msg->ratchet_key)))
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "Got duplicate unverified KX on %s. Fire back KX_AUTH again.\n",
+           GCT_2s (t));
+      send_kx_auth (t,
+                    ct,
+                    t->unverified_ax,
+                    GNUNET_NO);
+      return;
+    }
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Dropping old unverified KX state, got a fresh one.\n",
-         t->unverified_attempts);
-    cleanup_ax (t->unverified_ax);
+         "Dropping old unverified KX state. Got a fresh KX for %s.\n",
+         GCT_2s (t));
     memset (t->unverified_ax,
             0,
             sizeof (struct CadetTunnelAxolotl));
-    new_ephemeral (t->unverified_ax);
-    t->unverified_ax->kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
+    t->unverified_ax->DHRs = t->ax.DHRs;
+    t->unverified_ax->kx_0 = t->ax.kx_0;
   }
   else
   {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Creating fresh unverified KX for %s.\n",
+         GCT_2s (t));
     t->unverified_ax = GNUNET_new (struct CadetTunnelAxolotl);
-    new_ephemeral (t->unverified_ax);
-    t->unverified_ax->kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
+    t->unverified_ax->DHRs = t->ax.DHRs;
+    t->unverified_ax->kx_0 = t->ax.kx_0;
   }
-  /* Set as the 'current' RK the one we are currently using,
+  /* Set as the 'current' RK/DHRr the one we are currently using,
      so that the duplicate-detection logic of
      #update_ax_by_kx can work. */
   t->unverified_ax->RK = t->ax.RK;
+  t->unverified_ax->DHRr = t->ax.DHRr;
   t->unverified_attempts = 0;
   ax = t->unverified_ax;
 
-  /* FIXME: why this? Investigate use of kx_task! */
-  if (0 != (GNUNET_CADET_KX_FLAG_FORCE_REPLY & ntohl (msg->flags)))
-  {
-    if (NULL != t->kx_task)
-    {
-      GNUNET_SCHEDULER_cancel (t->kx_task);
-      t->kx_task = NULL;
-    }
-    send_kx (t,
-             ax,
-             GNUNET_NO);
-  }
-
   /* Update 'ax' by the new key material */
   ret = update_ax_by_kx (ax,
                          GCP_get_id (t->destination),
@@ -1514,34 +1748,125 @@ GCT_handle_kx (struct CadetTConnection *ct,
   if (GNUNET_OK != ret)
     return; /* duplicate KX, nothing to do */
 
+  /* move ahead in our state machine */
+  if (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_RECV);
+  else if (CADET_TUNNEL_KEY_AX_SENT == t->estate)
+    GCT_change_estate (t,
+                       CADET_TUNNEL_KEY_AX_SENT_AND_RECV);
+
+  /* KX is still not done, try again our end. */
+  if (CADET_TUNNEL_KEY_OK != t->estate)
+  {
+    if (NULL != t->kx_task)
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+    t->kx_task
+      = GNUNET_SCHEDULER_add_now (&retry_kx,
+                                  t);
+  }
+}
+
+
+/**
+ * Handle KX_AUTH message.
+ *
+ * @param ct connection/tunnel combo that received encrypted message
+ * @param msg the key exchange message
+ */
+void
+GCT_handle_kx_auth (struct CadetTConnection *ct,
+                    const struct GNUNET_CADET_TunnelKeyExchangeAuthMessage *msg)
+{
+  struct CadetTunnel *t = ct->t;
+  struct CadetTunnelAxolotl ax_tmp;
+  struct GNUNET_HashCode kx_auth;
+  int ret;
+
+  if ( (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate) ||
+       (CADET_TUNNEL_KEY_AX_RECV == t->estate) )
+  {
+    /* Confusing, we got a KX_AUTH before we even send our own
+       KX. This should not happen. We'll send our own KX ASAP anyway,
+       so let's ignore this here. */
+    GNUNET_break_op (0);
+    return;
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Handling KX_AUTH message for %s\n",
+       GCT_2s (t));
+
+  /* We do everything in ax_tmp until we've checked the authentication
+     so we don't clobber anything we care about by accident. */
+  ax_tmp = t->ax;
+
+  /* Update 'ax' by the new key material */
+  ret = update_ax_by_kx (&ax_tmp,
+                         GCP_get_id (t->destination),
+                         &msg->kx.ephemeral_key,
+                         &msg->kx.ratchet_key);
+  if (GNUNET_OK != ret)
+  {
+    if (GNUNET_NO == ret)
+      GNUNET_STATISTICS_update (stats,
+                                "# redundant KX_AUTH received",
+                                1,
+                                GNUNET_NO);
+    else
+      GNUNET_break (0); /* connect to self!? */
+    return;
+  }
+  GNUNET_CRYPTO_hash (&ax_tmp.RK,
+                      sizeof (ax_tmp.RK),
+                      &kx_auth);
+  if (0 != memcmp (&kx_auth,
+                   &msg->auth,
+                   sizeof (kx_auth)))
+  {
+    /* This KX_AUTH is not using the latest KX/KX_AUTH data
+       we transmitted to the sender, refuse it, try KX again. */
+    GNUNET_STATISTICS_update (stats,
+                              "# KX_AUTH not using our last KX received (auth failure)",
+                              1,
+                              GNUNET_NO);
+    send_kx (t,
+             ct,
+             &t->ax);
+    return;
+  }
+  /* Yep, we're good. */
+  t->ax = ax_tmp;
+  if (NULL != t->unverified_ax)
+  {
+    /* We got some "stale" KX before, drop that. */
+    cleanup_ax (t->unverified_ax);
+    GNUNET_free (t->unverified_ax);
+    t->unverified_ax = NULL;
+  }
+
   /* move ahead in our state machine */
   switch (t->estate)
   {
   case CADET_TUNNEL_KEY_UNINITIALIZED:
-    GCT_change_estate (t,
-                       CADET_TUNNEL_KEY_PING);
+  case CADET_TUNNEL_KEY_AX_RECV:
+    /* Checked above, this is impossible. */
+    GNUNET_assert (0);
     break;
-  case CADET_TUNNEL_KEY_SENT:
-    /* Got a response to us sending our key; now
-       we can start transmitting! */
+  case CADET_TUNNEL_KEY_AX_SENT:      /* This is the normal case */
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV: /* both peers started KX */
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT: /* both peers now did KX_AUTH */
     GCT_change_estate (t,
                        CADET_TUNNEL_KEY_OK);
-    if (NULL != t->send_task)
-      GNUNET_SCHEDULER_cancel (t->send_task);
-    t->send_task = GNUNET_SCHEDULER_add_now (&trigger_transmissions,
-                                             t);
-    break;
-  case CADET_TUNNEL_KEY_PING:
-    /* Got a key yet again; need encrypted payload or KX_AUTH
-       to advance to #CADET_TUNNEL_KEY_OK! */
     break;
   case CADET_TUNNEL_KEY_OK:
-    /* Did not expect a key, but so what. */
+    /* Did not expect another KX_AUTH, but so what, still acceptable.
+       Nothing to do here. */
     break;
   }
 }
 
 
+
 /* ************************************** end core crypto ***************************** */
 
 
@@ -1571,12 +1896,12 @@ get_next_free_ctn (struct CadetTunnel *t)
   ctn = ntohl (t->next_ctn.cn);
   while (NULL !=
          GNUNET_CONTAINER_multihashmap32_get (t->channels,
-                                              ctn))
+                                              ctn | highbit))
   {
-    ctn = ((ctn + 1) & (~ HIGH_BIT)) | highbit;
+    ctn = ((ctn + 1) & (~ HIGH_BIT));
   }
-  t->next_ctn.cn = htonl (((ctn + 1) & (~ HIGH_BIT)) | highbit);
-  ret.cn = ntohl (ctn);
+  t->next_ctn.cn = htonl ((ctn + 1) & (~ HIGH_BIT));
+  ret.cn = htonl (ctn | highbit);
   return ret;
 }
 
@@ -1597,17 +1922,44 @@ GCT_add_channel (struct CadetTunnel *t,
   struct GNUNET_CADET_ChannelTunnelNumber ctn;
 
   ctn = get_next_free_ctn (t);
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap32_put (t->channels,
                                                       ntohl (ctn.cn),
                                                       ch,
                                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Adding channel %s to tunnel %s\n",
+       "Adding %s to %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  if (CADET_TUNNEL_KEY_OK == t->estate)
+  switch (t->estate)
+  {
+  case CADET_TUNNEL_KEY_UNINITIALIZED:
+    /* waiting for connection to start KX */
+    break;
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* we're currently waiting for KX to complete */
+    break;
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* waiting for OTHER peer to send us data,
+       we might need to prompt more aggressively! */
+    if (NULL == t->kx_task)
+      t->kx_task
+        = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
+                                   &retry_kx,
+                                   t);
+    break;
+  case CADET_TUNNEL_KEY_OK:
+    /* We are ready. Tell the new channel that we are up. */
     GCCH_tunnel_up (ch);
+    break;
+  }
   return ctn;
 }
 
@@ -1623,13 +1975,43 @@ GCT_connection_lost (struct CadetTConnection *ct)
 {
   struct CadetTunnel *t = ct->t;
 
-  GNUNET_CONTAINER_DLL_remove (t->connection_head,
-                               t->connection_tail,
-                               ct);
+  if (GNUNET_YES == ct->is_ready)
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_ready_head,
+                                 t->connection_ready_tail,
+                                 ct);
+    t->num_ready_connections--;
+  }
+  else
+  {
+    GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                                 t->connection_busy_tail,
+                                 ct);
+    t->num_busy_connections--;
+  }
   GNUNET_free (ct);
 }
 
 
+/**
+ * Clean up connection @a ct of a tunnel.
+ *
+ * @param cls the `struct CadetTunnel`
+ * @param ct connection to clean up
+ */
+static void
+destroy_t_connection (void *cls,
+                      struct CadetTConnection *ct)
+{
+  struct CadetTunnel *t = cls;
+  struct CadetConnection *cc = ct->cc;
+
+  GNUNET_assert (ct->t == t);
+  GCT_connection_lost (ct);
+  GCC_destroy_without_tunnel (cc);
+}
+
+
 /**
  * This tunnel is no longer used, destroy it.
  *
@@ -1639,27 +2021,23 @@ static void
 destroy_tunnel (void *cls)
 {
   struct CadetTunnel *t = cls;
-  struct CadetTConnection *ct;
   struct CadetTunnelQueueEntry *tq;
 
   t->destroy_task = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Destroying idle tunnel %s\n",
+       "Destroying idle %s\n",
        GCT_2s (t));
-  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (t->channels));
-  while (NULL != (ct = t->connection_head))
-  {
-    struct CadetConnection *cc;
-
-    GNUNET_assert (ct->t == t);
-    cc = ct->cc;
-    GCT_connection_lost (ct);
-    GCC_destroy_without_tunnel (cc);
-  }
+  GNUNET_assert (0 == GCT_count_channels (t));
+  GCT_iterate_connections (t,
+                           &destroy_t_connection,
+                           t);
+  GNUNET_assert (NULL == t->connection_ready_head);
+  GNUNET_assert (NULL == t->connection_busy_head);
   while (NULL != (tq = t->tq_head))
   {
     if (NULL != tq->cont)
-      tq->cont (tq->cont_cls);
+      tq->cont (tq->cont_cls,
+                NULL);
     GCT_send_cancel (tq);
   }
   GCP_drop_tunnel (t->destination,
@@ -1682,12 +2060,13 @@ destroy_tunnel (void *cls)
   }
   GNUNET_MST_destroy (t->mst);
   GNUNET_MQ_destroy (t->mq);
-  cleanup_ax (&t->ax);
   if (NULL != t->unverified_ax)
   {
     cleanup_ax (t->unverified_ax);
     GNUNET_free (t->unverified_ax);
   }
+  cleanup_ax (&t->ax);
+  GNUNET_assert (NULL == t->destroy_task);
   GNUNET_free (t);
 }
 
@@ -1705,19 +2084,21 @@ GCT_remove_channel (struct CadetTunnel *t,
                     struct GNUNET_CADET_ChannelTunnelNumber ctn)
 {
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Removing channel %s from tunnel %s\n",
+       "Removing %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
   GNUNET_assert (GNUNET_YES ==
                  GNUNET_CONTAINER_multihashmap32_remove (t->channels,
                                                          ntohl (ctn.cn),
                                                          ch));
-  if (0 ==
-      GNUNET_CONTAINER_multihashmap32_size (t->channels))
+  if ( (0 ==
+        GCT_count_channels (t)) &&
+       (NULL == t->destroy_task) )
   {
-    t->destroy_task = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
-                                                    &destroy_tunnel,
-                                                    t);
+    t->destroy_task
+      = GNUNET_SCHEDULER_add_delayed (IDLE_DESTROY_DELAY,
+                                      &destroy_tunnel,
+                                      t);
   }
 }
 
@@ -1737,7 +2118,8 @@ destroy_remaining_channels (void *cls,
 {
   struct CadetChannel *ch = value;
 
-  GCCH_handle_remote_destroy (ch);
+  GCCH_handle_remote_destroy (ch,
+                              NULL);
   return GNUNET_OK;
 }
 
@@ -1755,7 +2137,7 @@ GCT_destroy_tunnel_now (struct CadetTunnel *t)
                                            &destroy_remaining_channels,
                                            t);
   GNUNET_assert (0 ==
-                 GNUNET_CONTAINER_multihashmap32_size (t->channels));
+                 GCT_count_channels (t));
   if (NULL != t->destroy_task)
   {
     GNUNET_SCHEDULER_cancel (t->destroy_task);
@@ -1765,26 +2147,6 @@ GCT_destroy_tunnel_now (struct CadetTunnel *t)
 }
 
 
-/**
- * It's been a while, we should try to redo the KX, if we can.
- *
- * @param cls the `struct CadetTunnel` to do KX for.
- */
-static void
-retry_kx (void *cls)
-{
-  struct CadetTunnel *t = cls;
-
-  t->kx_task = NULL;
-  send_kx (t,
-           &t->ax,
-           ( (CADET_TUNNEL_KEY_UNINITIALIZED == t->estate) ||
-             (CADET_TUNNEL_KEY_SENT == t->estate) )
-           ? GNUNET_YES
-           : GNUNET_NO);
-}
-
-
 /**
  * Send normal payload from queue in @a t via connection @a ct.
  * Does nothing if our payload queue is empty.
@@ -1816,7 +2178,7 @@ try_send_normal_payload (struct CadetTunnel *t,
                                tq);
   if (NULL != tq->cid)
     *tq->cid = *GCC_get_id (ct->cc);
-  ct->is_ready = GNUNET_NO;
+  mark_connection_unready (ct);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Sending payload of %s on %s\n",
        GCT_2s (t),
@@ -1824,7 +2186,8 @@ try_send_normal_payload (struct CadetTunnel *t,
   GCC_transmit (ct->cc,
                 tq->env);
   if (NULL != tq->cont)
-    tq->cont (tq->cont_cls);
+    tq->cont (tq->cont_cls,
+              GCC_get_id (ct->cc));
   GNUNET_free (tq);
 }
 
@@ -1847,37 +2210,71 @@ connection_ready_cb (void *cls,
   if (GNUNET_NO == is_ready)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Connection %s no longer ready for tunnel %s\n",
+         "%s no longer ready for %s\n",
          GCC_2s (ct->cc),
          GCT_2s (t));
-    ct->is_ready = GNUNET_NO;
+    mark_connection_unready (ct);
     return;
   }
+  GNUNET_assert (GNUNET_NO == ct->is_ready);
+  GNUNET_CONTAINER_DLL_remove (t->connection_busy_head,
+                               t->connection_busy_tail,
+                               ct);
+  GNUNET_assert (0 < t->num_busy_connections);
+  t->num_busy_connections--;
   ct->is_ready = GNUNET_YES;
+  GNUNET_CONTAINER_DLL_insert_tail (t->connection_ready_head,
+                                    t->connection_ready_tail,
+                                    ct);
+  t->num_ready_connections++;
+
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Connection %s now ready for tunnel %s in state %s\n",
+       "%s now ready for %s in state %s\n",
        GCC_2s (ct->cc),
        GCT_2s (t),
        estate2s (t->estate));
   switch (t->estate)
   {
   case CADET_TUNNEL_KEY_UNINITIALIZED:
+    /* Do not begin KX if WE have no channels waiting! */
+    if (0 == GCT_count_channels (t))
+      return;
+    /* We are uninitialized, just transmit immediately,
+       without undue delay. */
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
     send_kx (t,
-             &t->ax,
-             GNUNET_YES);
+             ct,
+             &t->ax);
     break;
-  case CADET_TUNNEL_KEY_SENT:
-  case CADET_TUNNEL_KEY_PING:
-    /* opportunity to #retry_kx() starts now, schedule job */
+  case CADET_TUNNEL_KEY_AX_RECV:
+  case CADET_TUNNEL_KEY_AX_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* we're currently waiting for KX to complete, schedule job */
     if (NULL == t->kx_task)
-    {
       t->kx_task
         = GNUNET_SCHEDULER_add_at (t->next_kx_attempt,
                                    &retry_kx,
                                    t);
-    }
     break;
   case CADET_TUNNEL_KEY_OK:
+    if (GNUNET_YES == t->kx_auth_requested)
+    {
+      if (NULL != t->kx_task)
+      {
+        GNUNET_SCHEDULER_cancel (t->kx_task);
+        t->kx_task = NULL;
+      }
+      send_kx_auth (t,
+                    ct,
+                    &t->ax,
+                    GNUNET_NO);
+      return;
+    }
     try_send_normal_payload (t,
                              ct);
     break;
@@ -1910,6 +2307,118 @@ trigger_transmissions (void *cls)
 }
 
 
+/**
+ * Closure for #evaluate_connection. Used to assemble summary information
+ * about the existing connections so we can evaluate a new path.
+ */
+struct EvaluationSummary
+{
+
+  /**
+   * Minimum length of any of our connections, `UINT_MAX` if we have none.
+   */
+  unsigned int min_length;
+
+  /**
+   * Maximum length of any of our connections, 0 if we have none.
+   */
+  unsigned int max_length;
+
+  /**
+   * Minimum desirability of any of our connections, UINT64_MAX if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType min_desire;
+
+  /**
+   * Maximum desirability of any of our connections, 0 if we have none.
+   */
+  GNUNET_CONTAINER_HeapCostType max_desire;
+
+  /**
+   * Path we are comparing against for #evaluate_connection, can be NULL.
+   */
+  struct CadetPeerPath *path;
+
+  /**
+   * Connection deemed the "worst" so far encountered by #evaluate_connection,
+   * NULL if we did not yet encounter any connections.
+   */
+  struct CadetTConnection *worst;
+
+  /**
+   * Numeric score of @e worst, only set if @e worst is non-NULL.
+   */
+  double worst_score;
+
+  /**
+   * Set to #GNUNET_YES if we have a connection over @e path already.
+   */
+  int duplicate;
+
+};
+
+
+/**
+ * Evaluate a connection, updating our summary information in @a cls about
+ * what kinds of connections we have.
+ *
+ * @param cls the `struct EvaluationSummary *` to update
+ * @param ct a connection to include in the summary
+ */
+static void
+evaluate_connection (void *cls,
+                     struct CadetTConnection *ct)
+{
+  struct EvaluationSummary *es = cls;
+  struct CadetConnection *cc = ct->cc;
+  struct CadetPeerPath *ps = GCC_get_path (cc);
+  const struct CadetConnectionMetrics *metrics;
+  GNUNET_CONTAINER_HeapCostType ct_desirability;
+  struct GNUNET_TIME_Relative uptime;
+  struct GNUNET_TIME_Relative last_use;
+  uint32_t ct_length;
+  double score;
+  double success_rate;
+
+  if (ps == es->path)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "Ignoring duplicate path %s.\n",
+         GCPP_2s (es->path));
+    es->duplicate = GNUNET_YES;
+    return;
+  }
+  ct_desirability = GCPP_get_desirability (ps);
+  ct_length = GCPP_get_length (ps);
+  metrics = GCC_get_metrics (cc);
+  uptime = GNUNET_TIME_absolute_get_duration (metrics->age);
+  last_use = GNUNET_TIME_absolute_get_duration (metrics->last_use);
+  /* We add 1.0 here to avoid division by zero. */
+  success_rate = (metrics->num_acked_transmissions + 1.0) / (metrics->num_successes + 1.0);
+  score
+    = ct_desirability
+    + 100.0 / (1.0 + ct_length) /* longer paths = better */
+    + sqrt (uptime.rel_value_us / 60000000LL) /* larger uptime = better */
+    - last_use.rel_value_us / 1000L;          /* longer idle = worse */
+  score *= success_rate;        /* weigh overall by success rate */
+
+  if ( (NULL == es->worst) ||
+       (score < es->worst_score) )
+  {
+    es->worst = ct;
+    es->worst_score = score;
+  }
+  es->min_length = GNUNET_MIN (es->min_length,
+                               ct_length);
+  es->max_length = GNUNET_MAX (es->max_length,
+                               ct_length);
+  es->min_desire = GNUNET_MIN (es->min_desire,
+                               ct_desirability);
+  es->max_desire = GNUNET_MAX (es->max_desire,
+                               ct_desirability);
+}
+
+
 /**
  * Consider using the path @a p for the tunnel @a t.
  * The tunnel destination is at offset @a off in path @a p.
@@ -1925,31 +2434,24 @@ consider_path_cb (void *cls,
                   unsigned int off)
 {
   struct CadetTunnel *t = cls;
-  unsigned int min_length = UINT_MAX;
-  GNUNET_CONTAINER_HeapCostType max_desire = 0;
+  struct EvaluationSummary es;
   struct CadetTConnection *ct;
 
-  /* Check if we care about the new path. */
-  for (ct = t->connection_head;
-       NULL != ct;
-       ct = ct->next)
-  {
-    struct CadetPeerPath *ps;
-
-    ps = GCC_get_path (ct->cc);
-    if (ps == path)
-    {
-      LOG (GNUNET_ERROR_TYPE_DEBUG,
-           "Ignoring duplicate path %s for tunnel %s.\n",
-           GCPP_2s (path),
-           GCT_2s (t));
-      return GNUNET_YES; /* duplicate */
-    }
-    min_length = GNUNET_MIN (min_length,
-                             GCPP_get_length (ps));
-    max_desire = GNUNET_MAX (max_desire,
-                             GCPP_get_desirability (ps));
-  }
+  GNUNET_assert (off < GCPP_get_length (path));
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = path;
+  es.duplicate = GNUNET_NO;
+  es.worst = NULL;
+
+  /* Compute evaluation summary over existing connections. */
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if (GNUNET_YES == es.duplicate)
+    return GNUNET_YES;
 
   /* FIXME: not sure we should really just count
      'num_connections' here, as they may all have
@@ -1958,18 +2460,20 @@ consider_path_cb (void *cls,
   /* We iterate by increasing path length; if we have enough paths and
      this one is more than twice as long than what we are currently
      using, then ignore all of these super-long ones! */
-  if ( (t->num_connections > DESIRED_CONNECTIONS_PER_TUNNEL) &&
-       (min_length * 2 < off) )
+  if ( (GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (es.min_length * 2 < off) &&
+       (es.max_length < off) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Ignoring paths of length %u, they are way too long.\n",
-         min_length * 2);
+         es.min_length * 2);
     return GNUNET_NO;
   }
   /* If we have enough paths and this one looks no better, ignore it. */
-  if ( (t->num_connections >= DESIRED_CONNECTIONS_PER_TUNNEL) &&
-       (min_length < GCPP_get_length (path)) &&
-       (max_desire > GCPP_get_desirability (path)) )
+  if ( (GCT_count_any_connections (t) >= DESIRED_CONNECTIONS_PER_TUNNEL) &&
+       (es.min_length < GCPP_get_length (path)) &&
+       (es.min_desire > GCPP_get_desirability (path)) &&
+       (es.max_length < off) )
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
          "Ignoring path (%u/%llu) to %s, got something better already.\n",
@@ -1986,19 +2490,22 @@ consider_path_cb (void *cls,
   ct->t = t;
   ct->cc = GCC_create (t->destination,
                        path,
+                       off,
+                       GNUNET_CADET_OPTION_DEFAULT, /* FIXME: set based on what channels want/need! */
                        ct,
                        &connection_ready_cb,
                        ct);
+
   /* FIXME: schedule job to kill connection (and path?)  if it takes
      too long to get ready! (And track performance data on how long
      other connections took with the tunnel!)
      => Note: to be done within 'connection'-logic! */
-  GNUNET_CONTAINER_DLL_insert (t->connection_head,
-                               t->connection_tail,
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
                                ct);
-  t->num_connections++;
+  t->num_busy_connections++;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Found interesting path %s for tunnel %s, created connection %s\n",
+       "Found interesting path %s for %s, created %s\n",
        GCPP_2s (path),
        GCT_2s (t),
        GCC_2s (ct->cc));
@@ -2023,17 +2530,50 @@ static void
 maintain_connections_cb (void *cls)
 {
   struct CadetTunnel *t = cls;
+  struct GNUNET_TIME_Relative delay;
+  struct EvaluationSummary es;
 
   t->maintain_connections_task = NULL;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Performing connection maintenance for tunnel %s.\n",
+       "Performing connection maintenance for %s.\n",
        GCT_2s (t));
 
+  es.min_length = UINT_MAX;
+  es.max_length = 0;
+  es.max_desire = 0;
+  es.min_desire = UINT64_MAX;
+  es.path = NULL;
+  es.worst = NULL;
+  es.duplicate = GNUNET_NO;
+  GCT_iterate_connections (t,
+                           &evaluate_connection,
+                           &es);
+  if ( (NULL != es.worst) &&
+       (GCT_count_any_connections (t) > DESIRED_CONNECTIONS_PER_TUNNEL) )
+  {
+    /* Clear out worst-performing connection 'es.worst'. */
+    destroy_t_connection (t,
+                          es.worst);
+  }
+
+  /* Consider additional paths */
   (void) GCP_iterate_paths (t->destination,
                             &consider_path_cb,
                             t);
 
-  GNUNET_break (0); // FIXME: implement!
+  /* FIXME: calculate when to try again based on how well we are doing;
+     in particular, if we have to few connections, we might be able
+     to do without this (as PATHS should tell us whenever a new path
+     is available instantly; however, need to make sure this job is
+     restarted after that happens).
+     Furthermore, if the paths we do know are in a reasonably narrow
+     quality band and are plentyful, we might also consider us stabilized
+     and then reduce the frequency accordingly.  */
+  delay = GNUNET_TIME_UNIT_MINUTES;
+  t->maintain_connections_task
+    = GNUNET_SCHEDULER_add_delayed (delay,
+                                    &maintain_connections_cb,
+                                    t);
 }
 
 
@@ -2050,6 +2590,10 @@ GCT_consider_path (struct CadetTunnel *t,
                    struct CadetPeerPath *p,
                    unsigned int off)
 {
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Considering %s for %s\n",
+       GCPP_2s (p),
+       GCT_2s (t));
   (void) consider_path_cb (t,
                            p,
                            off);
@@ -2069,7 +2613,7 @@ handle_plaintext_keepalive (void *cls,
   struct CadetTunnel *t = cls;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Received KEEPALIVE on tunnel %s\n",
+       "Received KEEPALIVE on %s\n",
        GCT_2s (t));
   GNUNET_STATISTICS_update (stats,
                             "# keepalives received",
@@ -2114,7 +2658,7 @@ handle_plaintext_data (void *cls,
     /* We don't know about such a channel, might have been destroyed on our
        end in the meantime, or never existed. Send back a DESTROY. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved %u bytes of application data for unknown channel %u, sending DESTROY\n",
+         "Received %u bytes of application data for unknown channel %u, sending DESTROY\n",
          (unsigned int) (ntohs (msg->header.size) - sizeof (*msg)),
          ntohl (msg->ctn.cn));
     GCT_send_channel_destroy (t,
@@ -2122,6 +2666,7 @@ handle_plaintext_data (void *cls,
     return;
   }
   GCCH_handle_channel_plaintext_data (ch,
+                                      GCC_get_id (t->current_ct->cc),
                                       msg);
 }
 
@@ -2148,13 +2693,14 @@ handle_plaintext_data_ack (void *cls,
     /* We don't know about such a channel, might have been destroyed on our
        end in the meantime, or never existed. Send back a DESTROY. */
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved DATA_ACK for unknown channel %u, sending DESTROY\n",
+         "Received DATA_ACK for unknown channel %u, sending DESTROY\n",
          ntohl (ack->ctn.cn));
     GCT_send_channel_destroy (t,
                               ack->ctn);
     return;
   }
   GCCH_handle_channel_plaintext_data_ack (ch,
+                                          GCC_get_id (t->current_ct->cc),
                                           ack);
 }
 
@@ -2178,21 +2724,27 @@ handle_plaintext_channel_open (void *cls,
   if (NULL != ch)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Receicved duplicate channel OPEN on port %s from %s (%s), resending ACK\n",
+         "Received duplicate channel CHANNEL_OPEN on port %s from %s (%s), resending ACK\n",
          GNUNET_h2s (&copen->port),
          GCT_2s (t),
          GCCH_2s (ch));
-    GCCH_handle_duplicate_open (ch);
+    GCCH_handle_duplicate_open (ch,
+                                GCC_get_id (t->current_ct->cc));
     return;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Receicved channel OPEN on port %s from %s\n",
+       "Received CHANNEL_OPEN on port %s from %s\n",
        GNUNET_h2s (&copen->port),
        GCT_2s (t));
   ch = GCCH_channel_incoming_new (t,
                                   copen->ctn,
                                   &copen->port,
                                   ntohl (copen->opt));
+  if (NULL != t->destroy_task)
+  {
+    GNUNET_SCHEDULER_cancel (t->destroy_task);
+    t->destroy_task = NULL;
+  }
   GNUNET_assert (GNUNET_OK ==
                  GNUNET_CONTAINER_multihashmap32_put (t->channels,
                                                       ntohl (copen->ctn.cn),
@@ -2259,7 +2811,8 @@ handle_plaintext_channel_open_ack (void *cls,
        "Received channel OPEN_ACK on channel %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  GCCH_handle_channel_open_ack (ch);
+  GCCH_handle_channel_open_ack (ch,
+                                GCC_get_id (t->current_ct->cc));
 }
 
 
@@ -2289,10 +2842,11 @@ handle_plaintext_channel_destroy (void *cls,
     return;
   }
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Receicved channel DESTROY on %s from %s\n",
+       "Received channel DESTROY on %s from %s\n",
        GCCH_2s (ch),
        GCT_2s (t));
-  GCCH_handle_remote_destroy (ch);
+  GCCH_handle_remote_destroy (ch,
+                              GCC_get_id (t->current_ct->cc));
 }
 
 
@@ -2310,6 +2864,7 @@ handle_decrypted (void *cls,
 {
   struct CadetTunnel *t = cls;
 
+  GNUNET_assert (NULL != t->current_ct);
   GNUNET_MQ_inject_message (t->mq,
                             msg);
   return GNUNET_OK;
@@ -2370,8 +2925,10 @@ GCT_create_tunnel (struct CadetPeer *destination)
     GNUNET_MQ_handler_end ()
   };
 
+  t->kx_retry_delay = INITIAL_KX_RETRY_DELAY;
   new_ephemeral (&t->ax);
-  t->ax.kx_0 = GNUNET_CRYPTO_ecdhe_key_create ();
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CRYPTO_ecdhe_key_create2 (&t->ax.kx_0));
   t->destination = destination;
   t->channels = GNUNET_CONTAINER_multihashmap32_create (8);
   t->maintain_connections_task
@@ -2395,6 +2952,7 @@ GCT_create_tunnel (struct CadetPeer *destination)
  *
  * @param t a tunnel
  * @param cid connection identifer to use for the connection
+ * @param options options for the connection
  * @param path path to use for the connection
  * @return #GNUNET_OK on success,
  *         #GNUNET_SYSERR on failure (duplicate connection)
@@ -2402,6 +2960,7 @@ GCT_create_tunnel (struct CadetPeer *destination)
 int
 GCT_add_inbound_connection (struct CadetTunnel *t,
                             const struct GNUNET_CADET_ConnectionTunnelIdentifier *cid,
+                            enum GNUNET_CADET_ChannelOption options,
                             struct CadetPeerPath *path)
 {
   struct CadetTConnection *ct;
@@ -2411,6 +2970,7 @@ GCT_add_inbound_connection (struct CadetTunnel *t,
   ct->t = t;
   ct->cc = GCC_create_inbound (t->destination,
                                path,
+                               options,
                                ct,
                                cid,
                                &connection_ready_cb,
@@ -2418,7 +2978,7 @@ GCT_add_inbound_connection (struct CadetTunnel *t,
   if (NULL == ct->cc)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Tunnel %s refused inbound connection %s (duplicate)\n",
+         "%s refused inbound %s (duplicate)\n",
          GCT_2s (t),
          GCC_2s (ct->cc));
     GNUNET_free (ct);
@@ -2428,12 +2988,12 @@ GCT_add_inbound_connection (struct CadetTunnel *t,
      too long to get ready! (And track performance data on how long
      other connections took with the tunnel!)
      => Note: to be done within 'connection'-logic! */
-  GNUNET_CONTAINER_DLL_insert (t->connection_head,
-                               t->connection_tail,
+  GNUNET_CONTAINER_DLL_insert (t->connection_busy_head,
+                               t->connection_busy_tail,
                                ct);
-  t->num_connections++;
+  t->num_busy_connections++;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s has new connection %s\n",
+       "%s has new %s\n",
        GCT_2s (t),
        GCC_2s (ct->cc));
   return GNUNET_OK;
@@ -2456,7 +3016,7 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
   ssize_t decrypted_size;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel %s received %u bytes of encrypted data in state %d\n",
+       "%s received %u bytes of encrypted data in state %d\n",
        GCT_2s (t),
        (unsigned int) size,
        t->estate);
@@ -2464,32 +3024,64 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
   switch (t->estate)
   {
   case CADET_TUNNEL_KEY_UNINITIALIZED:
+  case CADET_TUNNEL_KEY_AX_RECV:
     /* We did not even SEND our KX, how can the other peer
-       send us encrypted data? */
-    GNUNET_break_op (0);
+       send us encrypted data? Must have been that we went
+       down and the other peer still things we are up.
+       Let's send it KX back. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without any KX",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
-  case CADET_TUNNEL_KEY_SENT:
+  case CADET_TUNNEL_KEY_AX_SENT_AND_RECV:
+    /* We send KX, and other peer send KX to us at the same time.
+       Neither KX is AUTH'ed, so let's try KX_AUTH this time. */
+    GNUNET_STATISTICS_update (stats,
+                              "# received encrypted without KX_AUTH",
+                              1,
+                              GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx_auth (t,
+                  ct,
+                  &t->ax,
+                  GNUNET_YES);
+    return;
+  case CADET_TUNNEL_KEY_AX_SENT:
     /* We did not get the KX of the other peer, but that
-       might have been lost.  Ask for KX again. */
+       might have been lost.  Send our KX again immediately. */
     GNUNET_STATISTICS_update (stats,
                               "# received encrypted without KX",
                               1,
                               GNUNET_NO);
     if (NULL != t->kx_task)
+    {
       GNUNET_SCHEDULER_cancel (t->kx_task);
-    t->kx_task = GNUNET_SCHEDULER_add_now (&retry_kx,
-                                           t);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
-  case CADET_TUNNEL_KEY_PING:
-    /* Great, first payload, we might graduate to OK */
+  case CADET_TUNNEL_KEY_AX_AUTH_SENT:
+    /* Great, first payload, we might graduate to OK! */
   case CADET_TUNNEL_KEY_OK:
+    /* We are up and running, all good. */
     break;
   }
 
-  GNUNET_STATISTICS_update (stats,
-                            "# received encrypted",
-                            1,
-                            GNUNET_NO);
   decrypted_size = -1;
   if (CADET_TUNNEL_KEY_OK == t->estate)
   {
@@ -2519,7 +3111,7 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
       GNUNET_free (t->unverified_ax);
       t->unverified_ax = NULL;
     }
-    if (CADET_TUNNEL_KEY_PING == t->estate)
+    if (CADET_TUNNEL_KEY_AX_AUTH_SENT == t->estate)
     {
       /* First time it worked, move tunnel into production! */
       GCT_change_estate (t,
@@ -2553,24 +3145,37 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
   if (-1 == decrypted_size)
   {
     /* Decryption failed for good, complain. */
-    GNUNET_break_op (0);
     LOG (GNUNET_ERROR_TYPE_WARNING,
-         "Tunnel %s failed to decrypt and validate encrypted data\n",
+         "%s failed to decrypt and validate encrypted data, retrying KX\n",
          GCT_2s (t));
     GNUNET_STATISTICS_update (stats,
                               "# unable to decrypt",
                               1,
                               GNUNET_NO);
+    if (NULL != t->kx_task)
+    {
+      GNUNET_SCHEDULER_cancel (t->kx_task);
+      t->kx_task = NULL;
+    }
+    send_kx (t,
+             ct,
+             &t->ax);
     return;
   }
+  GNUNET_STATISTICS_update (stats,
+                            "# decrypted bytes",
+                            decrypted_size,
+                            GNUNET_NO);
 
   /* The MST will ultimately call #handle_decrypted() on each message. */
+  t->current_ct = ct;
   GNUNET_break_op (GNUNET_OK ==
                    GNUNET_MST_from_buffer (t->mst,
                                            cbuf,
                                            decrypted_size,
                                            GNUNET_YES,
                                            GNUNET_NO));
+  t->current_ct = NULL;
 }
 
 
@@ -2587,7 +3192,7 @@ GCT_handle_encrypted (struct CadetTConnection *ct,
 struct CadetTunnelQueueEntry *
 GCT_send (struct CadetTunnel *t,
           const struct GNUNET_MessageHeader *message,
-          GNUNET_SCHEDULER_TaskCallback cont,
+          GCT_SendContinuation cont,
           void *cont_cls)
 {
   struct CadetTunnelQueueEntry *tq;
@@ -2602,7 +3207,7 @@ GCT_send (struct CadetTunnel *t,
   }
   payload_size = ntohs (message->size);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Encrypting %u bytes for tunnel %s\n",
+       "Encrypting %u bytes for %s\n",
        (unsigned int) payload_size,
        GCT_2s (t));
   env = GNUNET_MQ_msg_extra (ax_msg,
@@ -2612,12 +3217,16 @@ GCT_send (struct CadetTunnel *t,
                 &ax_msg[1],
                 message,
                 payload_size);
+  GNUNET_STATISTICS_update (stats,
+                            "# encrypted bytes",
+                            payload_size,
+                            GNUNET_NO);
   ax_msg->ax_header.Ns = htonl (t->ax.Ns++);
   ax_msg->ax_header.PNs = htonl (t->ax.PNs);
   /* FIXME: we should do this once, not once per message;
      this is a point multiplication, and DHRs does not
      change all the time. */
-  GNUNET_CRYPTO_ecdhe_key_get_public (t->ax.DHRs,
+  GNUNET_CRYPTO_ecdhe_key_get_public (&t->ax.DHRs,
                                       &ax_msg->ax_header.DHRs);
   t_h_encrypt (&t->ax,
                ax_msg);
@@ -2679,11 +3288,23 @@ GCT_iterate_connections (struct CadetTunnel *t,
                          GCT_ConnectionIterator iter,
                          void *iter_cls)
 {
-  for (struct CadetTConnection *ct = t->connection_head;
+  struct CadetTConnection *n;
+  for (struct CadetTConnection *ct = t->connection_ready_head;
+       NULL != ct;
+       ct = n)
+  {
+    n = ct->next;
+    iter (iter_cls,
+          ct);
+  }
+  for (struct CadetTConnection *ct = t->connection_busy_head;
        NULL != ct;
-       ct = ct->next)
+       ct = n)
+  {
+    n = ct->next;
     iter (iter_cls,
-          ct->cc);
+          ct);
+  }
 }
 
 
@@ -2797,7 +3418,7 @@ GCT_debug (const struct CadetTunnel *t,
         GCT_2s (t),
         estate2s (t->estate),
         t->tq_len,
-        t->num_connections);
+        GCT_count_any_connections (t));
   LOG2 (level,
         "TTT channels:\n");
   GNUNET_CONTAINER_multihashmap32_iterate (t->channels,
@@ -2805,7 +3426,10 @@ GCT_debug (const struct CadetTunnel *t,
                                            &level);
   LOG2 (level,
         "TTT connections:\n");
-  for (iter_c = t->connection_head; NULL != iter_c; iter_c = iter_c->next)
+  for (iter_c = t->connection_ready_head; NULL != iter_c; iter_c = iter_c->next)
+    GCC_debug (iter_c->cc,
+               level);
+  for (iter_c = t->connection_busy_head; NULL != iter_c; iter_c = iter_c->next)
     GCC_debug (iter_c->cc,
                level);