X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fmesh%2Fgnunet-service-mesh_tunnel.c;h=f192bb05b5ac49cc6a30c603164bff95c882721c;hb=a107ae89650629fb1d6981bff567ebbdae3dcb8b;hp=d46137cb95a8292ad6c649f69d6b44485ffc59ea;hpb=f43d37c03f088f8c42301e9dba6aed6cb4d843cf;p=oweals%2Fgnunet.git

diff --git a/src/mesh/gnunet-service-mesh_tunnel.c b/src/mesh/gnunet-service-mesh_tunnel.c
index d46137cb9..f192bb05b 100644
--- a/src/mesh/gnunet-service-mesh_tunnel.c
+++ b/src/mesh/gnunet-service-mesh_tunnel.c
@@ -24,7 +24,7 @@
 #include "gnunet_signatures.h"
 #include "gnunet_statistics_service.h"
 
-#include "mesh_protocol_enc.h"
+#include "mesh_protocol.h"
 #include "mesh_path.h"
 
 #include "gnunet-service-mesh_tunnel.h"
@@ -34,7 +34,7 @@
 
 #define LOG(level, ...) GNUNET_log_from(level,"mesh-tun",__VA_ARGS__)
 
-#define REKEY_WAIT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 30)
+#define REKEY_WAIT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 5)
 
 /******************************************************************************/
 /********************************   STRUCTS  **********************************/
@@ -82,9 +82,14 @@ struct MeshTunnel3
   struct MeshPeer *peer;
 
     /**
-     * State of the tunnel.
+     * State of the tunnel connectivity.
      */
-  enum MeshTunnel3State state;
+  enum MeshTunnel3CState cstate;
+
+  /**
+   * State of the tunnel encryption.
+   */
+  enum MeshTunnel3EState estate;
 
   /**
    * Key eXchange context.
@@ -136,26 +141,31 @@ struct MeshTunnel3
   /**
    * Queued messages, to transmit once tunnel gets connected.
    */
-  struct MeshTunnelQueue *tq_head;
-  struct MeshTunnelQueue *tq_tail;
+  struct MeshTunnelDelayed *tq_head;
+  struct MeshTunnelDelayed *tq_tail;
 };
 
 
 /**
- * Struct used to queue messages in a tunnel.
+ * Struct used to save messages in a non-ready tunnel to send once connected.
  */
-struct MeshTunnelQueue
+struct MeshTunnelDelayed
 {
   /**
    * DLL
    */
-  struct MeshTunnelQueue *next;
-  struct MeshTunnelQueue *prev;
+  struct MeshTunnelDelayed *next;
+  struct MeshTunnelDelayed *prev;
 
   /**
-   * Channel.
+   * Tunnel.
    */
-  struct MeshChannel *ch;
+  struct MeshTunnel3 *t;
+
+  /**
+   * Tunnel queue given to the channel to cancel request. Update on send_queued.
+   */
+  struct MeshTunnel3Queue *tq;
 
   /**
    * Message to send.
@@ -164,6 +174,33 @@ struct MeshTunnelQueue
 };
 
 
+/**
+ * Handle for messages queued but not yet sent.
+ */
+struct MeshTunnel3Queue
+{
+  /**
+   * Connection queue handle, to cancel if necessary.
+   */
+  struct MeshConnectionQueue *cq;
+
+  /**
+   * Handle in case message hasn't been given to a connection yet.
+   */
+  struct MeshTunnelDelayed *tqd;
+
+  /**
+   * Continuation to call once sent.
+   */
+  GMT_sent cont;
+
+  /**
+   * Closure for @c cont.
+   */
+  void *cont_cls;
+};
+
+
 /******************************************************************************/
 /*******************************   GLOBALS  ***********************************/
 /******************************************************************************/
@@ -225,18 +262,18 @@ static struct GNUNET_TIME_Relative rekey_period;
 /******************************************************************************/
 
 /**
- * Get string description for tunnel state.
+ * Get string description for tunnel connectivity state.
  *
- * @param s Tunnel state.
+ * @param cs Tunnel state.
  *
  * @return String representation.
  */
 static const char *
-GMT_state2s (enum MeshTunnel3State s)
+cstate2s (enum MeshTunnel3CState cs)
 {
   static char buf[128];
 
-  switch (s)
+  switch (cs)
   {
     case MESH_TUNNEL3_NEW:
       return "MESH_TUNNEL3_NEW";
@@ -244,21 +281,65 @@ GMT_state2s (enum MeshTunnel3State s)
       return "MESH_TUNNEL3_SEARCHING";
     case MESH_TUNNEL3_WAITING:
       return "MESH_TUNNEL3_WAITING";
-    case MESH_TUNNEL3_KEY_SENT:
-      return "MESH_TUNNEL3_KEY_SENT";
-    case MESH_TUNNEL3_PING_SENT:
-      return "MESH_TUNNEL3_PING_SENT";
     case MESH_TUNNEL3_READY:
       return "MESH_TUNNEL3_READY";
-    case MESH_TUNNEL3_RECONNECTING:
-      return "MESH_TUNNEL3_RECONNECTING";
-    case MESH_TUNNEL3_REKEY:
-      return "MESH_TUNNEL3_REKEY";
 
     default:
-      sprintf (buf, "%u (UNKNOWN STATE)", s);
+      sprintf (buf, "%u (UNKNOWN STATE)", cs);
+      return buf;
+  }
+  return "";
+}
+
+
+/**
+ * Get string description for tunnel encryption state.
+ *
+ * @param es Tunnel state.
+ *
+ * @return String representation.
+ */
+static const char *
+estate2s (enum MeshTunnel3EState es)
+{
+  static char buf[128];
+
+  switch (es)
+  {
+    case MESH_TUNNEL3_KEY_UNINITIALIZED:
+      return "MESH_TUNNEL3_KEY_UNINITIALIZED";
+    case MESH_TUNNEL3_KEY_SENT:
+      return "MESH_TUNNEL3_KEY_SENT";
+    case MESH_TUNNEL3_KEY_PING:
+      return "MESH_TUNNEL3_KEY_PING";
+    case MESH_TUNNEL3_KEY_OK:
+      return "MESH_TUNNEL3_KEY_OK";
+
+    default:
+      sprintf (buf, "%u (UNKNOWN STATE)", es);
       return buf;
   }
+  return "";
+}
+
+
+/**
+ * @brief Check if tunnel is ready to send traffic.
+ *
+ * Tunnel must be connected and with encryption correctly set up.
+ *
+ * @param t Tunnel to check.
+ *
+ * @return #GNUNET_YES if ready, #GNUNET_NO otherwise
+ */
+static int
+is_ready (struct MeshTunnel3 *t)
+{
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  ready: cs=%s, es=%s\n",
+       cstate2s (t->cstate), estate2s (t->estate));
+  return (MESH_TUNNEL3_READY == t->cstate
+          && MESH_TUNNEL3_KEY_OK == t->estate)
+         || GMT_is_loopback (t);
 }
 
 
@@ -290,6 +371,82 @@ ping_encryption_size (void)
 }
 
 
+/**
+ * Get the channel's buffer. ONLY FOR NON-LOOPBACK CHANNELS!!
+ *
+ * @param tch Tunnel's channel handle.
+ *
+ * @return Amount of messages the channel can still buffer towards the client.
+ */
+static unsigned int
+get_channel_buffer (const struct MeshTChannel *tch)
+{
+  int fwd;
+
+  /* If channel is outgoing, is origin in the FWD direction and fwd is YES */
+  fwd = GMCH_is_origin (tch->ch, GNUNET_YES);
+
+  return GMCH_get_buffer (tch->ch, fwd);
+}
+
+
+/**
+ * Get the channel's allowance status.
+ *
+ * @param tch Tunnel's channel handle.
+ *
+ * @return #GNUNET_YES if we allowed the client to send data to us.
+ */
+static int
+get_channel_allowed (const struct MeshTChannel *tch)
+{
+  int fwd;
+
+  /* If channel is outgoing, is origin in the FWD direction and fwd is YES */
+  fwd = GMCH_is_origin (tch->ch, GNUNET_YES);
+
+  return GMCH_get_allowed (tch->ch, fwd);
+}
+
+
+/**
+ * Get the connection's buffer.
+ *
+ * @param tc Tunnel's connection handle.
+ *
+ * @return Amount of messages the connection can still buffer.
+ */
+static unsigned int
+get_connection_buffer (const struct MeshTConnection *tc)
+{
+  int fwd;
+
+  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
+  fwd = GMC_is_origin (tc->c, GNUNET_YES);
+
+  return GMC_get_buffer (tc->c, fwd);
+}
+
+
+/**
+ * Get the connection's allowance.
+ *
+ * @param tc Tunnel's connection handle.
+ *
+ * @return Amount of messages we have allowed the next peer to send us.
+ */
+static unsigned int
+get_connection_allowed (const struct MeshTConnection *tc)
+{
+  int fwd;
+
+  /* If connection is outgoing, is origin in the FWD direction and fwd is YES */
+  fwd = GMC_is_origin (tc->c, GNUNET_YES);
+
+  return GMC_get_allowed (tc->c, fwd);
+}
+
+
 /**
  * Check that a ephemeral key message s well formed and correctly signed.
  *
@@ -299,7 +456,7 @@ ping_encryption_size (void)
  * @return GNUNET_OK if message is fine, GNUNET_SYSERR otherwise.
  */
 int
-check_ephemeral (struct MeshTunnel3 *t, 
+check_ephemeral (struct MeshTunnel3 *t,
                  const struct GNUNET_MESH_KX_Ephemeral *msg)
 {
   /* Check message size */
@@ -365,7 +522,7 @@ t_decrypt (struct MeshTunnel3 *t,
 {
   struct GNUNET_CRYPTO_SymmetricInitializationVector siv;
 
-  GNUNET_CRYPTO_symmetric_derive_iv (&siv, &t->e_key, &iv, sizeof (uint32_t), NULL);
+  GNUNET_CRYPTO_symmetric_derive_iv (&siv, &t->d_key, &iv, sizeof (uint32_t), NULL);
   return GNUNET_CRYPTO_symmetric_decrypt (src, size, &t->d_key, &siv, dst);
 }
 
@@ -435,7 +592,7 @@ tunnel_get_connection (struct MeshTunnel3 *t)
   for (iter = t->connection_head; NULL != iter; iter = iter->next)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG, "  connection %s: %u\n",
-         GNUNET_h2s (GMC_get_id (iter->c)), GMC_get_state (iter->c));
+         GMC_2s (iter->c), GMC_get_state (iter->c));
     if (MESH_CONNECTION_READY == GMC_get_state (iter->c))
     {
       qn = GMC_get_qn (iter->c, GMC_is_origin (iter->c, GNUNET_YES));
@@ -447,82 +604,235 @@ tunnel_get_connection (struct MeshTunnel3 *t)
       }
     }
   }
+  LOG (GNUNET_ERROR_TYPE_DEBUG, " selected: connection %s\n", GMC_2s (best));
   return best;
 }
 
 
 /**
- * Send all cached messages that we can, tunnel is online.
+ * Callback called when a queued message is sent.
+ *
+ * Calculates the average time and connection packet tracking.
  *
- * @param t Tunnel that holds the messages.
- * @param fwd Is this fwd?
+ * @param cls Closure (TunnelQueue handle).
+ * @param c Connection this message was on.
+ * @param q Connection queue handle (unused).
+ * @param type Type of message sent.
+ * @param fwd Was this a FWD going message?
+ * @param size Size of the message.
  */
 static void
-send_queued_data (struct MeshTunnel3 *t, int fwd)
+message_sent (void *cls,
+              struct MeshConnection *c,
+              struct MeshConnectionQueue *q,
+              uint16_t type, int fwd, size_t size)
 {
-  struct MeshTunnelQueue *tq;
-  struct MeshTunnelQueue *next;
-  unsigned int room;
+  struct MeshTunnel3Queue *qt = cls;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-              "GMT_send_queued_data on tunnel %s\n",
-              GMT_2s (t));
+  GNUNET_assert (NULL != qt->cont);
+  qt->cont (qt->cont_cls, GMC_get_tunnel (c), qt, type, size);
+  GNUNET_free (qt);
+}
 
-  if (NULL == t->channel_head ||
-      GNUNET_NO == GMCH_is_origin (t->channel_head->ch, fwd))
-    return;
 
-  room = GMT_get_buffer (t, fwd);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer space: %u\n", room);
-  for (tq = t->tq_head; NULL != tq && room > 0; tq = next)
-  {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " data on channel %s\n", GMCH_2s (tq->ch));
-    next = tq->next;
-    room--;
-    GNUNET_CONTAINER_DLL_remove (t->tq_head, t->tq_tail, tq);
-    GMCH_send_prebuilt_message ((struct GNUNET_MessageHeader *) &tq[1],
-                                tq->ch, fwd);
+/**
+ * Delete a queued message: either was sent or the channel was destroyed
+ * before the tunnel's key exchange had a chance to finish.
+ *
+ * @param tq Queue handle.
+ */
+static void
+unqueue_data (struct MeshTunnelDelayed *tq)
+{
+  GNUNET_CONTAINER_DLL_remove (tq->t->tq_head, tq->t->tq_tail, tq);
+  GNUNET_free (tq);
+}
 
-    GNUNET_free (tq);
+
+/**
+ * Cache a message to be sent once tunnel is online.
+ *
+ * @param t Tunnel to hold the message.
+ * @param msg Message itself (copy will be made).
+ */
+static struct MeshTunnelDelayed *
+queue_data (struct MeshTunnel3 *t, const struct GNUNET_MessageHeader *msg)
+{
+  struct MeshTunnelDelayed *tqd;
+  uint16_t size = ntohs (msg->size);
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "queue data on Tunnel %s\n", GMT_2s (t));
+
+  if (GNUNET_YES == is_ready (t))
+  {
+    GNUNET_break (0);
+    return NULL;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "GMT_send_queued_data end\n",
-       GMP_2s (t->peer));
+
+  tqd = GNUNET_malloc (sizeof (struct MeshTunnelDelayed) + size);
+
+  tqd->t = t;
+  memcpy (&tqd[1], msg, size);
+  GNUNET_CONTAINER_DLL_insert_tail (t->tq_head, t->tq_tail, tqd);
+  return tqd;
 }
 
 
 
+/**
+ * Sends an already built message on a tunnel, encrypting it and
+ * choosing the best connection.
+ *
+ * @param message Message to send. Function modifies it.
+ * @param t Tunnel on which this message is transmitted.
+ * @param force Force the tunnel to take the message (buffer overfill).
+ * @param cont Continuation to call once message is really sent.
+ * @param cont_cls Closure for @c cont.
+ * @param existing_q In case this a transmission of previously queued data,
+ *                   this should be TunnelQueue given to the client.
+ *                   Otherwise, NULL.
+ *
+ * @return Handle to cancel message. NULL if @c cont is NULL.
+ */
+static struct MeshTunnel3Queue *
+send_prebuilt_message (const struct GNUNET_MessageHeader *message,
+                       struct MeshTunnel3 *t, int force,
+                       GMT_sent cont, void *cont_cls,
+                       struct MeshTunnel3Queue *existing_q)
+{
+  struct MeshTunnel3Queue *tq;
+  struct MeshConnection *c;
+  struct GNUNET_MESH_Encrypted *msg;
+  size_t size = ntohs (message->size);
+  char cbuf[sizeof (struct GNUNET_MESH_Encrypted) + size];
+  uint32_t iv;
+  uint16_t type;
+  int fwd;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT Send on Tunnel %s\n", GMT_2s (t));
+
+  if (GNUNET_NO == is_ready (t))
+  {
+    struct MeshTunnelDelayed *tqd;
+    /* A non null existing_q indicates sending of queued data.
+     * Should only happen after tunnel becomes ready.
+     */
+    GNUNET_assert (NULL == existing_q);
+    tqd = queue_data (t, message);
+    if (NULL == cont)
+      return NULL;
+    tq = GNUNET_new (struct MeshTunnel3Queue);
+    tq->tqd = tqd;
+    tqd->tq = tq;
+    tq->cont = cont;
+    tq->cont_cls = cont_cls;
+    return tq;
+  }
+
+  GNUNET_assert (GNUNET_NO == GMT_is_loopback (t));
+
+  iv = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
+  msg = (struct GNUNET_MESH_Encrypted *) cbuf;
+  msg->header.type = htons (GNUNET_MESSAGE_TYPE_MESH_ENCRYPTED);
+  msg->iv = iv;
+  GNUNET_assert (t_encrypt (t, &msg[1], message, size, iv) == size);
+  msg->header.size = htons (sizeof (struct GNUNET_MESH_Encrypted) + size);
+  c = tunnel_get_connection (t);
+  if (NULL == c)
+  {
+    GNUNET_break (GNUNET_YES == t->destroy);
+    return NULL;
+  }
+  type = ntohs (message->type);
+  switch (type)
+  {
+    case GNUNET_MESSAGE_TYPE_MESH_DATA:
+    case GNUNET_MESSAGE_TYPE_MESH_DATA_ACK:
+    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_CREATE:
+    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_DESTROY:
+    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_ACK:
+      msg->cid = *GMC_get_id (c);
+      msg->ttl = htonl (default_ttl);
+      break;
+    default:
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "unkown type %s\n",
+           GM_m2s (type));
+      GNUNET_break (0);
+  }
+
+  fwd = GMC_is_origin (c, GNUNET_YES);
+
+  if (NULL == cont)
+  {
+    (void) GMC_send_prebuilt_message (&msg->header, c, fwd, force, NULL, NULL);
+    return NULL;
+  }
+  if (NULL == existing_q)
+  {
+    tq = GNUNET_new (struct MeshTunnel3Queue); /* FIXME valgrind: leak*/
+  }
+  else
+  {
+    tq = existing_q;
+    tq->tqd = NULL;
+  }
+  tq->cq = GMC_send_prebuilt_message (&msg->header, c, fwd, force,
+                                      &message_sent, tq);
+  tq->cont = cont;
+  tq->cont_cls = cont_cls;
+
+  return tq;
+}
+
 
 /**
- * Cache a message to be sent once tunnel is online.
+ * Send all cached messages that we can, tunnel is online.
  *
- * @param t Tunnel to hold the message.
- * @param ch Channel the message is about.
- * @param msg Message itself (copy will be made).
+ * @param t Tunnel that holds the messages. Cannot be loopback.
  */
 static void
-queue_data (struct MeshTunnel3 *t,
-            struct MeshChannel *ch,
-            const struct GNUNET_MessageHeader *msg)
+send_queued_data (struct MeshTunnel3 *t)
 {
-  struct MeshTunnelQueue *tq;
-  uint16_t size = ntohs (msg->size);
+  struct MeshTunnelDelayed *tqd;
+  struct MeshTunnelDelayed *next;
+  unsigned int room;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "GMT_send_queued_data on tunnel %s\n",
+       GMT_2s (t));
 
-  if (MESH_TUNNEL3_READY == t->state)
+  if (GMT_is_loopback (t))
   {
     GNUNET_break (0);
     return;
   }
 
-  tq = GNUNET_malloc (sizeof (struct MeshTunnelQueue) + size);
+  if (GNUNET_NO == is_ready (t))
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  not ready yet: %s/%s\n",
+         estate2s (t->estate), cstate2s (t->cstate));
+    return;
+  }
 
-  tq->ch = ch;
-  memcpy (&tq[1], msg, size);
-  GNUNET_CONTAINER_DLL_insert_tail (t->tq_head, t->tq_tail, tq);
+  room = GMT_get_connections_buffer (t);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  buffer space: %u\n", room);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  tq head: %p\n", t->tq_head);
+  for (tqd = t->tq_head; NULL != tqd && room > 0; tqd = next)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, " sending queued data\n");
+    next = tqd->next;
+    room--;
+    send_prebuilt_message ((struct GNUNET_MessageHeader *) &tqd[1],
+                           tqd->t, GNUNET_YES,
+                           NULL != tqd->tq ? tqd->tq->cont : NULL,
+                           NULL != tqd->tq ? tqd->tq->cont_cls : NULL,
+                           tqd->tq);
+    unqueue_data (tqd);
+  }
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT_send_queued_data end\n", GMP_2s (t->peer));
 }
 
 
-
 /**
  * Sends key exchange message on a tunnel, choosing the best connection.
  * Should not be called on loopback tunnels.
@@ -544,7 +854,7 @@ send_kx (struct MeshTunnel3 *t,
   LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT KX on Tunnel %s\n", GMT_2s (t));
 
   /* Avoid loopback. */
-  if (myid == GMP_get_short_id (t->peer))
+  if (GMT_is_loopback (t))
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG, "  loopback!\n");
     GNUNET_break (0);
@@ -554,7 +864,7 @@ send_kx (struct MeshTunnel3 *t,
   /* Must have a connection. */
   if (NULL == t->connection_head)
   {
-    GNUNET_break (0);
+    GNUNET_break (MESH_TUNNEL3_SEARCHING == t->cstate);
     return;
   }
 
@@ -564,7 +874,7 @@ send_kx (struct MeshTunnel3 *t,
   c = tunnel_get_connection (t);
   if (NULL == c)
   {
-    GNUNET_break (GNUNET_YES == t->destroy);
+    GNUNET_break (GNUNET_YES == t->destroy || MESH_TUNNEL3_READY != t->cstate);
     return;
   }
   type = ntohs (message->type);
@@ -573,18 +883,18 @@ send_kx (struct MeshTunnel3 *t,
     case GNUNET_MESSAGE_TYPE_MESH_KX_EPHEMERAL:
     case GNUNET_MESSAGE_TYPE_MESH_KX_PING:
     case GNUNET_MESSAGE_TYPE_MESH_KX_PONG:
-      msg->cid = *GMC_get_id (c);
       msg->reserved = htonl (0);
       memcpy (&msg[1], message, size);
       break;
     default:
       LOG (GNUNET_ERROR_TYPE_DEBUG, "unkown type %s\n",
-           GNUNET_MESH_DEBUG_M2S (type));
+           GM_m2s (type));
       GNUNET_break (0);
   }
 
   fwd = GMC_is_origin (t->connection_head->c, GNUNET_YES);
-  GMC_send_prebuilt_message (&msg->header, c, fwd);
+  /* TODO save handle and cancel in case of a unneeded retransmission */
+  GMC_send_prebuilt_message (&msg->header, c, fwd, GNUNET_YES, NULL, NULL);
 }
 
 
@@ -596,8 +906,9 @@ send_kx (struct MeshTunnel3 *t,
 static void
 send_ephemeral (struct MeshTunnel3 *t)
 {
-  kx_msg.sender_status = htonl (t->state);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "%s()\n", __FUNCTION__);
 
+  kx_msg.sender_status = htonl (t->estate);
   send_kx (t, &kx_msg.header);
 }
 
@@ -611,12 +922,18 @@ send_ping (struct MeshTunnel3 *t)
 {
   struct GNUNET_MESH_KX_Ping msg;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "%s()\n", __FUNCTION__);
   msg.header.size = htons (sizeof (msg));
   msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_KX_PING);
   msg.iv = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
   msg.target = *GMP_get_id (t->peer);
   msg.nonce = t->kx_ctx->challenge;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending %u\n", msg.nonce);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  towards %s\n", GNUNET_i2s (&msg.target));
   t_encrypt (t, &msg.target, &msg.target, ping_encryption_size(), msg.iv);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  e sending %u\n", msg.nonce);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  e towards %s\n", GNUNET_i2s (&msg.target));
 
   send_kx (t, &msg.header);
 }
@@ -633,11 +950,14 @@ send_pong (struct MeshTunnel3 *t, uint32_t challenge)
 {
   struct GNUNET_MESH_KX_Pong msg;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "%s()\n", __FUNCTION__);
   msg.header.size = htons (sizeof (msg));
   msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_KX_PONG);
   msg.iv = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
   msg.nonce = challenge;
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  sending %u\n", msg.nonce);
   t_encrypt (t, &msg.nonce, &msg.nonce, sizeof (msg.nonce), msg.iv);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  e sending %u\n", msg.nonce);
 
   send_kx (t, &msg.header);
 }
@@ -656,28 +976,37 @@ rekey_tunnel (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
   t->rekey_task = GNUNET_SCHEDULER_NO_TASK;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Re-key Tunnel\n");
   if (NULL != tc && 0 != (GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason))
     return;
 
-  t->kx_ctx = GNUNET_new (struct MeshTunnelKXCtx);
-  t->kx_ctx->challenge = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
-                                                   UINT32_MAX);
-  t->kx_ctx->d_key_old = t->d_key;
-  send_ephemeral (t);
-  if (MESH_TUNNEL3_READY == t->state)
+  if (NULL == t->kx_ctx)
   {
-    send_ping (t);
-    t->state = MESH_TUNNEL3_REKEY;
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  new kx ctx\n");
+    t->kx_ctx = GNUNET_new (struct MeshTunnelKXCtx);
+    t->kx_ctx->challenge = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
+                                                     UINT32_MAX);
+    t->kx_ctx->d_key_old = t->d_key;
   }
-  else if (MESH_TUNNEL3_WAITING == t->state)
-  {
-    t->state = MESH_TUNNEL3_KEY_SENT;
-  }
-  else
+  send_ephemeral (t);
+  switch (t->estate)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "Unexpected state %u\n", t->state);
+    case MESH_TUNNEL3_KEY_UNINITIALIZED:
+      t->estate = MESH_TUNNEL3_KEY_SENT;
+      break;
+    case MESH_TUNNEL3_KEY_SENT:
+      break;
+    case MESH_TUNNEL3_KEY_PING:
+    case MESH_TUNNEL3_KEY_OK:
+      send_ping (t);
+      t->estate = MESH_TUNNEL3_KEY_PING;
+      break;
+    default:
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "Unexpected state %u\n", t->estate);
   }
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  next call in %s\n",
+       GNUNET_STRINGS_relative_time_to_string (REKEY_WAIT, GNUNET_YES));
   t->rekey_task = GNUNET_SCHEDULER_add_delayed (REKEY_WAIT, &rekey_tunnel, t);
 }
 
@@ -751,20 +1080,43 @@ rekey (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 }
 
 
+/**
+ * Called only on shutdown, destroy every tunnel.
+ *
+ * @param cls Closure (unused).
+ * @param key Current public key.
+ * @param value Value in the hash map (tunnel).
+ *
+ * @return #GNUNET_YES, so we should continue to iterate,
+ */
+static int
+destroy_iterator (void *cls,
+                const struct GNUNET_PeerIdentity *key,
+                void *value)
+{
+  struct MeshTunnel3 *t = value;
+
+  GMT_destroy (t);
+  return GNUNET_YES;
+}
+
+
 /**
  * Demultiplex data per channel and call appropriate channel handler.
  *
  * @param t Tunnel on which the data came.
  * @param msg Data message.
- * @param fwd Is this FWD data? (root -> dest)
+ * @param fwd Is this message fwd? This only is meaningful in loopback channels.
+ *            #GNUNET_YES if message is FWD on the respective channel (loopback)
+ *            #GNUNET_NO if message is BCK on the respective channel (loopback)
+ *            #GNUNET_SYSERR if message on a one-ended channel (remote)
  */
-void
+static void
 handle_data (struct MeshTunnel3 *t,
              const struct GNUNET_MESH_Data *msg,
              int fwd)
 {
   struct MeshChannel *ch;
-  uint16_t type;
   size_t size;
 
   /* Check size */
@@ -776,11 +1128,8 @@ handle_data (struct MeshTunnel3 *t,
     GNUNET_break (0);
     return;
   }
-  type = ntohs (msg->header.type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "got a %s message\n",
-              GNUNET_MESH_DEBUG_M2S (type));
   LOG (GNUNET_ERROR_TYPE_DEBUG, " payload of type %s\n",
-              GNUNET_MESH_DEBUG_M2S (ntohs (msg[1].header.type)));
+              GM_m2s (ntohs (msg[1].header.type)));
 
   /* Check channel */
   ch = GMT_get_channel (t, ntohl (msg->chid));
@@ -793,11 +1142,21 @@ handle_data (struct MeshTunnel3 *t,
     return;
   }
 
-  GMT_change_state (t, MESH_TUNNEL3_READY);
   GMCH_handle_data (ch, msg, fwd);
 }
 
-void
+
+/**
+ * Demultiplex data ACKs per channel and update appropriate channel buffer info.
+ *
+ * @param t Tunnel on which the DATA ACK came.
+ * @param msg DATA ACK message.
+ * @param fwd Is this message fwd? This only is meaningful in loopback channels.
+ *            #GNUNET_YES if message is FWD on the respective channel (loopback)
+ *            #GNUNET_NO if message is BCK on the respective channel (loopback)
+ *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ */
+static void
 handle_data_ack (struct MeshTunnel3 *t,
                  const struct GNUNET_MESH_DataACK *msg,
                  int fwd)
@@ -827,10 +1186,16 @@ handle_data_ack (struct MeshTunnel3 *t,
   GMCH_handle_data_ack (ch, msg, fwd);
 }
 
-void
+
+/**
+ * Handle channel create.
+ *
+ * @param t Tunnel on which the data came.
+ * @param msg Data message.
+ */
+static void
 handle_ch_create (struct MeshTunnel3 *t,
-                  const struct GNUNET_MESH_ChannelCreate *msg,
-                  int fwd)
+                  const struct GNUNET_MESH_ChannelCreate *msg)
 {
   struct MeshChannel *ch;
   size_t size;
@@ -845,19 +1210,68 @@ handle_ch_create (struct MeshTunnel3 *t,
 
   /* Check channel */
   ch = GMT_get_channel (t, ntohl (msg->chid));
-  if (NULL != ch)
+  if (NULL != ch && ! GMT_is_loopback (t))
   {
     /* Probably a retransmission, safe to ignore */
     LOG (GNUNET_ERROR_TYPE_DEBUG, "   already exists...\n");
   }
   else
   {
-    ch = GMCH_handle_create (t, msg, fwd);
+    ch = GMCH_handle_create (t, msg);
   }
-  GMT_add_channel (t, ch);
+  if (NULL != ch)
+    GMT_add_channel (t, ch);
 }
 
-void
+
+
+/**
+ * Handle channel NACK: check correctness and call channel handler for NACKs.
+ *
+ * @param t Tunnel on which the NACK came.
+ * @param msg NACK message.
+ */
+static void
+handle_ch_nack (struct MeshTunnel3 *t,
+                const struct GNUNET_MESH_ChannelManage *msg)
+{
+  struct MeshChannel *ch;
+  size_t size;
+
+  /* Check size */
+  size = ntohs (msg->header.size);
+  if (size != sizeof (struct GNUNET_MESH_ChannelManage))
+  {
+    GNUNET_break (0);
+    return;
+  }
+
+  /* Check channel */
+  ch = GMT_get_channel (t, ntohl (msg->chid));
+  if (NULL == ch)
+  {
+    GNUNET_STATISTICS_update (stats, "# channel NACK on unknown channel",
+                              1, GNUNET_NO);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "WARNING channel %u unknown\n",
+         ntohl (msg->chid));
+    return;
+  }
+
+  GMCH_handle_nack (ch);
+}
+
+
+/**
+ * Handle a CHANNEL ACK (SYNACK/ACK).
+ *
+ * @param t Tunnel on which the CHANNEL ACK came.
+ * @param msg CHANNEL ACK message.
+ * @param fwd Is this message fwd? This only is meaningful in loopback channels.
+ *            #GNUNET_YES if message is FWD on the respective channel (loopback)
+ *            #GNUNET_NO if message is BCK on the respective channel (loopback)
+ *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ */
+static void
 handle_ch_ack (struct MeshTunnel3 *t,
                const struct GNUNET_MESH_ChannelManage *msg,
                int fwd)
@@ -887,7 +1301,19 @@ handle_ch_ack (struct MeshTunnel3 *t,
   GMCH_handle_ack (ch, msg, fwd);
 }
 
-void
+
+
+/**
+ * Handle a channel destruction message.
+ *
+ * @param t Tunnel on which the message came.
+ * @param msg Channel destroy message.
+ * @param fwd Is this message fwd? This only is meaningful in loopback channels.
+ *            #GNUNET_YES if message is FWD on the respective channel (loopback)
+ *            #GNUNET_NO if message is BCK on the respective channel (loopback)
+ *            #GNUNET_SYSERR if message on a one-ended channel (remote)
+ */
+static void
 handle_ch_destroy (struct MeshTunnel3 *t,
                    const struct GNUNET_MESH_ChannelManage *msg,
                    int fwd)
@@ -934,12 +1360,14 @@ handle_ephemeral (struct MeshTunnel3 *t,
     return;
   }
   derive_key_material (&km, &msg->ephemeral_key);
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "  km is %s\n", GNUNET_h2s (&km));
   derive_symmertic (&t->e_key, &my_full_id, GMP_get_id (t->peer), &km);
   derive_symmertic (&t->d_key, GMP_get_id (t->peer), &my_full_id, &km);
-  if (MESH_TUNNEL3_KEY_SENT == t->state)
+  if (MESH_TUNNEL3_KEY_SENT == t->estate)
   {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  our key was sent, send ping\n");
     send_ping (t);
-    t->state = MESH_TUNNEL3_PING_SENT;
+    t->estate = MESH_TUNNEL3_KEY_PING;
   }
 }
 
@@ -957,17 +1385,25 @@ handle_ping (struct MeshTunnel3 *t,
 {
   struct GNUNET_MESH_KX_Ping res;
 
+  if (ntohs (msg->header.size) != sizeof (res))
+  {
+    GNUNET_break_op (0);
+    return;
+  }
+
   LOG (GNUNET_ERROR_TYPE_DEBUG, "  ping message\n");
-  t_decrypt (t, &res.target, &msg->target, ping_encryption_size(), msg->iv);
-  if (0 != memcmp (&my_full_id, &msg->target, sizeof (my_full_id)))
+  t_decrypt (t, &res.target, &msg->target, ping_encryption_size (), msg->iv);
+  if (0 != memcmp (&my_full_id, &res.target, sizeof (my_full_id)))
   {
-    GNUNET_break (0);
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  at %s\n", GNUNET_i2s (&my_full_id));
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  for %s\n", GNUNET_i2s (&msg->target));
+    GNUNET_break_op (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  e got %u\n", msg->nonce);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  e towards %s\n", GNUNET_i2s (&msg->target));
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  got %u\n", res.nonce);
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  towards %s\n", GNUNET_i2s (&res.target));
     return;
   }
 
-  send_pong (t, res.iv);
+  send_pong (t, res.nonce);
 }
 
 
@@ -996,8 +1432,8 @@ handle_pong (struct MeshTunnel3 *t,
   if (challenge != t->kx_ctx->challenge)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Wrong PONG challenge: %u. Expected: %u.\n",
-         challenge, t->kx_ctx->challenge);
+         "Wrong PONG challenge: %u (e: %u). Expected: %u.\n",
+         challenge, msg->nonce, t->kx_ctx->challenge);
     GNUNET_break_op (0);
     return;
   }
@@ -1005,9 +1441,7 @@ handle_pong (struct MeshTunnel3 *t,
   t->rekey_task = GNUNET_SCHEDULER_NO_TASK;
   GNUNET_free (t->kx_ctx);
   t->kx_ctx = NULL;
-  t->state = MESH_TUNNEL3_READY;
-  send_queued_data (t, GNUNET_YES);
-  send_queued_data (t, GNUNET_NO);
+  GMT_change_estate (t, MESH_TUNNEL3_KEY_OK);
 }
 
 
@@ -1017,7 +1451,10 @@ handle_pong (struct MeshTunnel3 *t,
  *
  * @param t Tunnel this message came on.
  * @param msgh Message header.
- * @param fwd Is this message fwd?
+ * @param fwd Is this message fwd? This only is meaningful in loopback channels.
+ *            #GNUNET_YES if message is FWD on the respective channel (loopback)
+ *            #GNUNET_NO if message is BCK on the respective channel (loopback)
+ *            #GNUNET_SYSERR if message on a one-ended channel (remote)
  */
 static void
 handle_decrypted (struct MeshTunnel3 *t,
@@ -1029,7 +1466,7 @@ handle_decrypted (struct MeshTunnel3 *t,
   type = ntohs (msgh->type);
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Got a %s message!\n",
-       GNUNET_MESH_DEBUG_M2S (type));
+       GM_m2s (type));
 
   switch (type)
   {
@@ -1044,8 +1481,12 @@ handle_decrypted (struct MeshTunnel3 *t,
 
     case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_CREATE:
       handle_ch_create (t,
-                        (struct GNUNET_MESH_ChannelCreate *) msgh,
-                        fwd);
+                        (struct GNUNET_MESH_ChannelCreate *) msgh);
+      break;
+
+    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_NACK:
+      handle_ch_nack (t,
+                      (struct GNUNET_MESH_ChannelManage *) msgh);
       break;
 
     case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_ACK:
@@ -1078,12 +1519,10 @@ handle_decrypted (struct MeshTunnel3 *t,
  *
  * @param t Tunnel this message came on.
  * @param msg Encrypted message.
- * @param fwd Is this message fwd?
  */
 void
 GMT_handle_encrypted (struct MeshTunnel3 *t,
-                      const struct GNUNET_MESH_Encrypted *msg,
-                      int fwd)
+                      const struct GNUNET_MESH_Encrypted *msg)
 {
   size_t size = ntohs (msg->header.size);
   size_t payload_size = size - sizeof (struct GNUNET_MESH_Encrypted);
@@ -1097,7 +1536,7 @@ GMT_handle_encrypted (struct MeshTunnel3 *t,
   while (off < decrypted_size)
   {
     msgh = (struct GNUNET_MessageHeader *) &cbuf[off];
-    handle_decrypted (t, msgh, fwd);
+    handle_decrypted (t, msgh, GNUNET_SYSERR);
     off += ntohs (msgh->size);
   }
 }
@@ -1187,6 +1626,7 @@ GMT_shutdown (void)
     GNUNET_SCHEDULER_cancel (rekey_task);
     rekey_task = GNUNET_SCHEDULER_NO_TASK;
   }
+  GNUNET_CONTAINER_multipeermap_iterate (tunnels, &destroy_iterator, NULL);
   GNUNET_CONTAINER_multipeermap_destroy (tunnels);
 }
 
@@ -1218,42 +1658,71 @@ GMT_new (struct MeshPeer *destination)
 
 
 /**
- * Change the tunnel state.
+ * Change the tunnel's connection state.
  *
- * @param t Tunnel whose state to change.
- * @param state New state.
+ * @param t Tunnel whose connection state to change.
+ * @param cstate New connection state.
  */
 void
-GMT_change_state (struct MeshTunnel3* t, enum MeshTunnel3State state)
+GMT_change_cstate (struct MeshTunnel3* t, enum MeshTunnel3CState cstate)
 {
   if (NULL == t)
     return;
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-              "Tunnel %s state was %s\n",
-              GMP_2s (t->peer),
-              GMT_state2s (t->state));
+              "Tunnel %s cstate was %s\n",
+              GMP_2s (t->peer), cstate2s (t->cstate));
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-              "Tunnel %s state is now %s\n",
-              GMP_2s (t->peer),
-              GMT_state2s (state));
+              "Tunnel %s cstate is now %s\n",
+              GMP_2s (t->peer), cstate2s (cstate));
   if (myid != GMP_get_short_id (t->peer) &&
-      MESH_TUNNEL3_WAITING == t->state && MESH_TUNNEL3_READY == state)
+      MESH_TUNNEL3_READY != t->cstate &&
+      MESH_TUNNEL3_READY == cstate)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  triggered rekey\n");
-    rekey_tunnel (t, NULL);
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "Tunnel %s state is now %s\n",
-         GMP_2s (t->peer),
-         GMT_state2s (t->state));
+    t->cstate = cstate;
+    if (MESH_TUNNEL3_KEY_OK == t->estate)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "  triggered send queued data\n");
+      send_queued_data (t);
+    }
+    else if (MESH_TUNNEL3_KEY_UNINITIALIZED == t->estate)
+    {
+      LOG (GNUNET_ERROR_TYPE_DEBUG, "  triggered rekey\n");
+      rekey_tunnel (t, NULL);
+    }
   }
-  else
+  t->cstate = cstate;
+
+  if (MESH_TUNNEL3_READY == cstate && 3 <= GMT_count_connections (t))
   {
-    t->state = state;
+    GMP_stop_search (t->peer);
   }
-  if (MESH_TUNNEL3_READY == state && 3 <= GMT_count_connections (t))
+}
+
+/**
+ * Change the tunnel encryption state.
+ *
+ * @param t Tunnel whose encryption state to change.
+ * @param state New encryption state.
+ */
+void
+GMT_change_estate (struct MeshTunnel3* t, enum MeshTunnel3EState state)
+{
+  if (NULL == t)
+    return;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Tunnel %s estate was %s\n",
+       GMP_2s (t->peer), estate2s (t->estate));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Tunnel %s estate is now %s\n",
+       GMP_2s (t->peer), estate2s (state));
+  if (myid != GMP_get_short_id (t->peer) &&
+      MESH_TUNNEL3_KEY_OK != t->estate && MESH_TUNNEL3_KEY_OK == state)
   {
-    GMP_stop_search (t->peer);
+    t->estate = state;
+    send_queued_data (t);
+    return;
   }
+  t->estate = state;
 }
 
 
@@ -1268,6 +1737,8 @@ GMT_add_connection (struct MeshTunnel3 *t, struct MeshConnection *c)
 {
   struct MeshTConnection *aux;
 
+  GNUNET_assert (NULL != c);
+
   for (aux = t->connection_head; aux != NULL; aux = aux->next)
     if (aux->c == c)
       return;
@@ -1288,14 +1759,39 @@ void
 GMT_remove_connection (struct MeshTunnel3 *t, struct MeshConnection *c)
 {
   struct MeshTConnection *aux;
+  struct MeshTConnection *next;
 
-  for (aux = t->connection_head; aux != NULL; aux = aux->next)
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Removing connection %s from tunnel %s\n",
+       GMC_2s (c), GMT_2s (t));
+  for (aux = t->connection_head; aux != NULL; aux = next)
+  {
+    next = aux->next;
     if (aux->c == c)
     {
       GNUNET_CONTAINER_DLL_remove (t->connection_head, t->connection_tail, aux);
       GNUNET_free (aux);
-      return;
     }
+  }
+
+  /* Start new connections if needed */
+  if (NULL == t->connection_head && GNUNET_NO == t->destroy)
+  {
+    LOG (GNUNET_ERROR_TYPE_DEBUG, "  no more connections\n");
+    GMP_connect (t->peer);
+    t->cstate = MESH_TUNNEL3_SEARCHING;
+    return;
+  }
+
+  /* If not marked as ready, no change is needed */
+  if (MESH_TUNNEL3_READY != t->cstate)
+    return;
+
+  /* Check if any connection is ready to maintaing cstate */
+  for (aux = t->connection_head; aux != NULL; aux = aux->next)
+    if (MESH_CONNECTION_READY == GMC_get_state (aux->c))
+      return;
+
+  t->cstate = MESH_TUNNEL3_WAITING;
 }
 
 
@@ -1310,6 +1806,8 @@ GMT_add_channel (struct MeshTunnel3 *t, struct MeshChannel *ch)
 {
   struct MeshTChannel *aux;
 
+  GNUNET_assert (NULL != ch);
+
   LOG (GNUNET_ERROR_TYPE_DEBUG, "Adding channel %p to tunnel %p\n", ch, t);
 
   for (aux = t->channel_head; aux != NULL; aux = aux->next)
@@ -1323,6 +1821,12 @@ GMT_add_channel (struct MeshTunnel3 *t, struct MeshChannel *ch)
   aux->ch = ch;
   LOG (GNUNET_ERROR_TYPE_DEBUG, " adding %p to %p\n", aux, t->channel_head);
   GNUNET_CONTAINER_DLL_insert_tail (t->channel_head, t->channel_tail, aux);
+
+  if (GNUNET_YES == t->destroy)
+  {
+    t->destroy = GNUNET_NO;
+    LOG (GNUNET_ERROR_TYPE_DEBUG, " undo destroy!\n");
+  }
 }
 
 
@@ -1389,11 +1893,13 @@ GMT_destroy_empty (struct MeshTunnel3 *t)
 {
   struct MeshTConnection *iter;
 
+  LOG (GNUNET_ERROR_TYPE_DEBUG, "Tunnel empty: destroying scheduled\n");
   for (iter = t->connection_head; NULL != iter; iter = iter->next)
   {
     GMC_send_destroy (iter->c);
   }
 
+  t->cstate = MESH_TUNNEL3_NEW;
   t->destroy = GNUNET_YES;
 }
 
@@ -1427,21 +1933,32 @@ GMT_destroy_if_empty (struct MeshTunnel3 *t)
 void
 GMT_destroy (struct MeshTunnel3 *t)
 {
-  struct MeshTConnection *iter;
-  struct MeshTConnection *next;
+  struct MeshTConnection *iter_c;
+  struct MeshTConnection *next_c;
+  struct MeshTChannel *iter_ch;
+  struct MeshTChannel *next_ch;
 
   if (NULL == t)
     return;
 
+  t->destroy = 2;
+
   LOG (GNUNET_ERROR_TYPE_DEBUG, "destroying tunnel %s\n", GMP_2s (t->peer));
 
-//   if (GNUNET_YES != GNUNET_CONTAINER_multihashmap_remove (tunnels, &t->id, t))
-//     GNUNET_break (0);
+  GNUNET_break (GNUNET_YES ==
+                GNUNET_CONTAINER_multipeermap_remove (tunnels,
+                                                      GMP_get_id (t->peer), t));
 
-  for (iter = t->connection_head; NULL != iter; iter = next)
+  for (iter_c = t->connection_head; NULL != iter_c; iter_c = next_c)
   {
-    next = iter->next;
-    GMC_destroy (iter->c);
+    next_c = iter_c->next;
+    GMC_destroy (iter_c->c);
+  }
+  for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = next_ch)
+  {
+    next_ch = iter_ch->next;
+    GMCH_destroy (iter_ch->ch);
+    /* Should only happen on shutdown, but it's ok. */
   }
 
   GNUNET_STATISTICS_update (stats, "# tunnels", -1, GNUNET_NO);
@@ -1484,12 +2001,17 @@ GMT_use_path (struct MeshTunnel3 *t, struct MeshPeerPath *p)
   }
   if (own_pos > p->length - 1)
   {
-    GNUNET_break (0);
+    GNUNET_break_op (0);
     return NULL;
   }
 
   GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_NONCE, &cid);
   c = GMC_new (&cid, t, p, own_pos);
+  if (NULL == c)
+  {
+    /* Path was flawed */
+    return NULL;
+  }
   GMT_add_connection (t, c);
   return c;
 }
@@ -1537,72 +2059,76 @@ GMT_count_channels (struct MeshTunnel3 *t)
 
 
 /**
- * Get the state of a tunnel.
+ * Get the connectivity state of a tunnel.
  *
  * @param t Tunnel.
  *
- * @return Tunnel's state.
+ * @return Tunnel's connectivity state.
  */
-enum MeshTunnel3State
-GMT_get_state (struct MeshTunnel3 *t)
+enum MeshTunnel3CState
+GMT_get_cstate (struct MeshTunnel3 *t)
 {
   if (NULL == t)
-    return (enum MeshTunnel3State) -1;
-  return t->state;
+  {
+    GNUNET_break (0);
+    return (enum MeshTunnel3CState) -1;
+  }
+  return t->cstate;
 }
 
+
 /**
- * Get the total buffer space for a tunnel.
- *
- * If terminal, use the biggest channel buffer (or 64) if no channel exists.
- * If not terminal, use the sum of all connection buffers.
+ * Get the maximum buffer space for a tunnel towards a local client.
  *
  * @param t Tunnel.
- * @param fwd Is this for FWD traffic?
  *
- * @return Buffer space offered by all entities (c/ch) in the tunnel.
+ * @return Biggest buffer space offered by any channel in the tunnel.
  */
 unsigned int
-GMT_get_buffer (struct MeshTunnel3 *t, int fwd)
+GMT_get_channels_buffer (struct MeshTunnel3 *t)
 {
-  struct MeshTConnection *iter;
+  struct MeshTChannel *iter;
   unsigned int buffer;
+  unsigned int ch_buf;
 
-  iter = t->connection_head;
-  buffer = 0;
+  if (NULL == t->channel_head)
+  {
+    /* Probably getting buffer for a channel create/handshake. */
+    return 64;
+  }
 
-  /* If terminal, return biggest channel buffer */
-  if (NULL == iter || GMC_is_terminal (iter->c, fwd))
+  buffer = 0;
+  for (iter = t->channel_head; NULL != iter; iter = iter->next)
   {
-    struct MeshTChannel *iter_ch;
-    unsigned int ch_buf;
+    ch_buf = get_channel_buffer (iter);
+    if (ch_buf > buffer)
+      buffer = ch_buf;
+  }
+  return buffer;
+}
 
-    if (NULL == t->channel_head)
-    {
-      /* Probably getting buffer for a channel create/handshake. */
-      return 64;
-    }
 
-    for (iter_ch = t->channel_head; NULL != iter_ch; iter_ch = iter_ch->next)
-    {
-      ch_buf = GMCH_get_buffer (iter_ch->ch, fwd);
-      if (ch_buf > buffer)
-        buffer = ch_buf;
-    }
-    return buffer;
-  }
+/**
+ * Get the total buffer space for a tunnel for P2P traffic.
+ *
+ * @param t Tunnel.
+ *
+ * @return Buffer space offered by all connections in the tunnel.
+ */
+unsigned int
+GMT_get_connections_buffer (struct MeshTunnel3 *t)
+{
+  struct MeshTConnection *iter;
+  unsigned int buffer;
 
-  /* If not terminal, return sum of connection buffers */
-  while (NULL != iter)
+  buffer = 0;
+  for (iter = t->connection_head; NULL != iter; iter = iter->next)
   {
     if (GMC_get_state (iter->c) != MESH_CONNECTION_READY)
     {
-      iter = iter->next;
       continue;
     }
-
-    buffer += GMC_get_buffer (iter->c, fwd);
-    iter = iter->next;
+    buffer += get_connection_buffer (iter);
   }
 
   return buffer;
@@ -1634,14 +2160,28 @@ MESH_ChannelNumber
 GMT_get_next_chid (struct MeshTunnel3 *t)
 {
   MESH_ChannelNumber chid;
+  MESH_ChannelNumber mask;
+  int result;
+
+  /* Set bit 30 depending on the ID relationship. Bit 31 is always 0 for GID.
+   * If our ID is bigger or loopback tunnel, start at 0, bit 30 = 0
+   * If peer's ID is bigger, start at 0x4... bit 30 = 1
+   */
+  result = GNUNET_CRYPTO_cmp_peer_identity (&my_full_id, GMP_get_id (t->peer));
+  if (0 > result)
+    mask = 0x4000000;
+  else
+    mask = 0x0;
 
   while (NULL != GMT_get_channel (t, t->next_chid))
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG, "Channel %u exists...\n", t->next_chid);
     t->next_chid = (t->next_chid + 1) & ~GNUNET_MESH_LOCAL_CHANNEL_ID_CLI;
+    t->next_chid |= mask;
   }
   chid = t->next_chid;
   t->next_chid = (t->next_chid + 1) & ~GNUNET_MESH_LOCAL_CHANNEL_ID_CLI;
+  t->next_chid |= mask;
 
   return chid;
 }
@@ -1651,10 +2191,9 @@ GMT_get_next_chid (struct MeshTunnel3 *t)
  * Send ACK on one or more channels due to buffer in connections.
  *
  * @param t Channel which has some free buffer space.
- * @param fwd Is this for FWD traffic? (ACK goes to root)
  */
 void
-GMT_unchoke_channels (struct MeshTunnel3 *t, int fwd)
+GMT_unchoke_channels (struct MeshTunnel3 *t)
 {
   struct MeshTChannel *iter;
   unsigned int buffer;
@@ -1667,14 +2206,8 @@ GMT_unchoke_channels (struct MeshTunnel3 *t, int fwd)
   if (NULL != t->channel_head)
     LOG (GNUNET_ERROR_TYPE_DEBUG, " head ch: %p\n", t->channel_head->ch);
 
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    return;
-  }
-
   /* Get buffer space */
-  buffer = GMT_get_buffer (t, fwd);
+  buffer = GMT_get_connections_buffer (t);
   if (0 == buffer)
   {
     return;
@@ -1684,7 +2217,7 @@ GMT_unchoke_channels (struct MeshTunnel3 *t, int fwd)
   choked_n = 0;
   for (iter = t->channel_head; NULL != iter; iter = iter->next)
   {
-    if (GNUNET_NO == GMCH_get_allowed (iter->ch, fwd))
+    if (GNUNET_NO == get_channel_allowed (iter))
     {
       choked[choked_n++] = iter->ch;
     }
@@ -1695,7 +2228,7 @@ GMT_unchoke_channels (struct MeshTunnel3 *t, int fwd)
   {
     unsigned int r = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
                                                choked_n);
-    GMCH_allow_client (choked[r], fwd);
+    GMCH_allow_client (choked[r], GMCH_is_origin (choked[r], GNUNET_YES));
     choked_n--;
     buffer--;
     choked[r] = choked[choked_n];
@@ -1709,10 +2242,9 @@ GMT_unchoke_channels (struct MeshTunnel3 *t, int fwd)
  * Iterates all connections of the tunnel and sends ACKs appropriately.
  *
  * @param t Tunnel.
- * @param fwd Is this in for FWD traffic? (ACK goes dest->root)
  */
 void
-GMT_send_acks (struct MeshTunnel3 *t, int fwd)
+GMT_send_connection_acks (struct MeshTunnel3 *t)
 {
   struct MeshTConnection *iter;
   uint32_t allowed;
@@ -1722,34 +2254,27 @@ GMT_send_acks (struct MeshTunnel3 *t, int fwd)
   unsigned int buffer;
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Tunnel send %s ACKs on %s\n",
-       fwd ? "FWD" : "BCK", GMT_2s (t));
+       "Tunnel send connection ACKs on %s\n",
+       GMT_2s (t));
 
   if (NULL == t)
   {
     GNUNET_break (0);
     return;
   }
-  if (NULL == t->channel_head ||
-      GNUNET_NO == GMCH_is_origin (t->channel_head->ch, !fwd))
-  {
-    GNUNET_break (0);
-    return;
-  }
 
-  buffer = GMT_get_buffer (t, fwd);
+  buffer = GMT_get_channels_buffer (t);
 
   /* Count connections, how many messages are already allowed */
   cs = GMT_count_connections (t);
   for (allowed = 0, iter = t->connection_head; NULL != iter; iter = iter->next)
   {
-    allowed += GMC_get_allowed (iter->c, fwd);
+    allowed += get_connection_allowed (iter);
   }
 
   /* Make sure there is no overflow */
   if (allowed > buffer)
   {
-    GNUNET_break (0);
     return;
   }
 
@@ -1761,11 +2286,11 @@ GMT_send_acks (struct MeshTunnel3 *t, int fwd)
     allow_per_connection = to_allow/cs;
     to_allow -= allow_per_connection;
     cs--;
-    if (GMC_get_allowed (iter->c, fwd) > 64 / 3)
+    if (get_connection_allowed (iter) > 64 / 3)
     {
       continue;
     }
-    GMC_allow (iter->c, buffer, fwd);
+    GMC_allow (iter->c, buffer, GMC_is_origin (iter->c, GNUNET_YES));
   }
 
   GNUNET_break (to_allow == 0);
@@ -1773,74 +2298,54 @@ GMT_send_acks (struct MeshTunnel3 *t, int fwd)
 
 
 /**
- * Sends an already built message on a tunnel, encrypting it and
- * choosing the best connection.
+ * Cancel a previously sent message while it's in the queue.
  *
- * @param message Message to send. Function modifies it.
- * @param t Tunnel on which this message is transmitted.
- * @param ch Channel on which this message is transmitted.
- * @param fwd Is this a fwd message on @c ch?
+ * ONLY can be called before the continuation given to the send function
+ * is called. Once the continuation is called, the message is no longer in the
+ * queue.
+ *
+ * @param q Handle to the queue.
  */
 void
-GMT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
-                           struct MeshTunnel3 *t,
-                           struct MeshChannel *ch,
-                           int fwd)
+GMT_cancel (struct MeshTunnel3Queue *q)
 {
-  struct MeshConnection *c;
-  struct GNUNET_MESH_Encrypted *msg;
-  size_t size = ntohs (message->size);
-  size_t encrypted_size;
-  char cbuf[sizeof (struct GNUNET_MESH_Encrypted) + size];
-  uint32_t iv;
-  uint16_t type;
-
-  if (MESH_TUNNEL3_READY != t->state)
-  {
-    queue_data (t, ch, message);
-    return;
-  }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GMT Send on Tunnel %s\n", GMT_2s (t));
-
-  if (myid == GMP_get_short_id (t->peer))
+  if (NULL != q->cq)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  loopback!\n");
-    handle_decrypted (t, message, fwd);
-    return;
+    GMC_cancel (q->cq);
+    /* message_sent() will be called and free q */
   }
-
-  iv = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, UINT32_MAX);
-  msg = (struct GNUNET_MESH_Encrypted *) cbuf;
-  msg->header.type = htons (GNUNET_MESSAGE_TYPE_MESH_ENCRYPTED);
-  msg->iv = iv;
-  encrypted_size = t_encrypt (t, &msg[1], message, size, iv);
-  msg->header.size = htons (sizeof (struct GNUNET_MESH_Encrypted) + encrypted_size);
-  c = tunnel_get_connection (t);
-  if (NULL == c)
+  else if (NULL != q->tqd)
   {
-    GNUNET_break (GNUNET_YES == t->destroy);
-    return;
+    unqueue_data (q->tqd);
   }
-  type = ntohs (message->type);
-  switch (type)
+  else
   {
-    case GNUNET_MESSAGE_TYPE_MESH_DATA:
-    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_CREATE:
-    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_DESTROY:
-    case GNUNET_MESSAGE_TYPE_MESH_CHANNEL_ACK:
-      msg->cid = *GMC_get_id (c);
-      msg->ttl = htonl (default_ttl);
-      break;
-    default:
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "unkown type %s\n",
-           GNUNET_MESH_DEBUG_M2S (type));
-      GNUNET_break (0);
+    GNUNET_break (0);
   }
+}
 
-  fwd = GMC_is_origin (c, GNUNET_YES);
-  GMC_send_prebuilt_message (&msg->header, c, fwd);
+
+/**
+ * Sends an already built message on a tunnel, encrypting it and
+ * choosing the best connection.
+ *
+ * @param message Message to send. Function modifies it.
+ * @param t Tunnel on which this message is transmitted.
+ * @param force Force the tunnel to take the message (buffer overfill).
+ * @param cont Continuation to call once message is really sent.
+ * @param cont_cls Closure for @c cont.
+ *
+ * @return Handle to cancel message. NULL if @c cont is NULL.
+ */
+struct MeshTunnel3Queue *
+GMT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
+                           struct MeshTunnel3 *t, int force,
+                           GMT_sent cont, void *cont_cls)
+{
+  return send_prebuilt_message (message, t, force, cont, cont_cls, NULL);
 }
 
+
 /**
  * Is the tunnel directed towards the local peer?
  *
@@ -1851,12 +2356,12 @@ GMT_send_prebuilt_message (const struct GNUNET_MessageHeader *message,
 int
 GMT_is_loopback (const struct MeshTunnel3 *t)
 {
-  return (myid == GMP_get_short_id(t->peer));
+  return (myid == GMP_get_short_id (t->peer));
 }
 
 
 /**
- * Is the tunnel using this path already?
+ * Is the tunnel this path already?
  *
  * @param t Tunnel.
  * @param p Path.
@@ -1889,6 +2394,7 @@ GMT_get_path_cost (const struct MeshTunnel3 *t,
                    const struct MeshPeerPath *path)
 {
   struct MeshTConnection *iter;
+  const struct MeshPeerPath *aux;
   unsigned int overlap;
   unsigned int i;
   unsigned int j;
@@ -1903,9 +2409,13 @@ GMT_get_path_cost (const struct MeshTunnel3 *t,
   {
     for (iter = t->connection_head; NULL != iter; iter = iter->next)
     {
-      for (j = 0; j < GMC_get_path (iter->c)->length; j++)
+      aux = GMC_get_path (iter->c);
+      if (NULL == aux)
+        continue;
+
+      for (j = 0; j < aux->length; j++)
       {
-        if (path->peers[i] == GMC_get_path (iter->c)->peers[j])
+        if (path->peers[i] == aux->peers[j])
         {
           overlap++;
           break;
@@ -1927,5 +2437,8 @@ GMT_get_path_cost (const struct MeshTunnel3 *t,
 const char *
 GMT_2s (const struct MeshTunnel3 *t)
 {
+  if (NULL == t)
+    return "(NULL)";
+
   return GMP_2s (t->peer);
-}
\ No newline at end of file
+}