- use message ID for end-to-end ACKs
[oweals/gnunet.git] / src / mesh / gnunet-service-mesh.c
index d97ae74b130961685535080853e9a618ba504585..7504fa223679d6c5a124a9bb405bc8f22ae9b878 100644 (file)
  * @brief GNUnet MESH service
  * @author Bartlomiej Polot
  *
+ *  FIXME in progress:
+ * - keep queues until receiving ACK
+ *
  * TODO:
  * - relay corking down to core
  * - set ttl relative to path length
  * - add signatures
  * - add encryption
- * - keep queues until receiving ACK
  * TODO END
  */
 
@@ -49,6 +51,8 @@
 #define MESH_MAX_POLL_TIME      GNUNET_TIME_relative_multiply (\
                                   GNUNET_TIME_UNIT_MINUTES,\
                                   10)
+#define MESH_RETRANSMIT_TIME    GNUNET_TIME_UNIT_SECONDS
+#define MESH_RETRANSMIT_MARGIN  4
 
 #if MESH_DEBUG_CONNECTION
 #define DEBUG_CONN(...) GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
@@ -279,6 +283,90 @@ struct MESH_TunnelID
 };
 
 
+/**
+ * Data needed for reliable tunnel endpoint retransmission management.
+ */
+struct MeshTunnelReliability;
+
+
+/**
+ * Info needed to retry a message in case it gets lost.
+ */
+struct MeshReliableMessage
+{
+    /**
+     * Double linked list, FIFO style
+     */
+  struct MeshReliableMessage    *next;
+  struct MeshReliableMessage    *prev;
+
+    /**
+     * Tunnel Reliability queue this message is in.
+     */
+  struct MeshTunnelReliability  *rel;
+
+    /**
+     * ID of the message (ACK needed to free)
+     */
+  uint64_t                      mid;
+
+    /**
+     * When was this message issued (to calculate ACK delay) FIXME update with traffic
+     */
+  struct GNUNET_TIME_Absolute   timestamp;
+
+  /* struct GNUNET_MESH_Data with payload */
+};
+
+
+struct MeshTunnelReliability
+{
+    /**
+     * Tunnel this is about.
+     */
+  struct MeshTunnel *t;
+
+    /**
+     * DLL of messages sent and not yet ACK'd.
+     */
+  struct MeshReliableMessage        *head_sent;
+  struct MeshReliableMessage        *tail_sent;
+
+    /**
+     * Messages pending
+     */
+  unsigned int                      n_sent;
+
+    /**
+     * Next MID to use
+     */
+  uint64_t                          mid_sent;
+
+    /**
+     * DLL of messages received out of order.
+     */
+  struct MeshReliableMessage        *head_recv;
+  struct MeshReliableMessage        *tail_recv;
+
+  uint64_t                          mid_recv;
+
+    /**
+     * Task to resend/poll in case no ACK is received.
+     */
+  GNUNET_SCHEDULER_TaskIdentifier   retry_task;
+
+    /**
+     * Counter for exponential backoff.
+     */
+  struct GNUNET_TIME_Relative       retry_timer;
+
+    /**
+     * How long does it usually take to get an ACK. FIXME update with traffic
+     */
+  struct GNUNET_TIME_Relative       expected_delay;
+};
+
+
 /**
  * Struct containing all information regarding a tunnel
  * For an intermediate node the improtant info used will be:
@@ -397,6 +485,18 @@ struct MeshTunnel
      * Total messages pending for this tunnels, payload or not.
      */
   unsigned int pending_messages;
+
+    /**
+     * Reliability data.
+     * Only present (non-NULL) at the owner of a tunnel.
+     */
+  struct MeshTunnelReliability *fwd_rel;
+
+    /**
+     * Reliability data.
+     * Only present (non-NULL) at the destination of a tunnel.
+     */
+  struct MeshTunnelReliability *bck_rel;
 };
 
 
@@ -420,12 +520,12 @@ struct MeshClient
     /**
      * Tunnels that belong to this client, indexed by local id
      */
-  struct GNUNET_CONTAINER_MultiHashMap *own_tunnels;
+  struct GNUNET_CONTAINER_MultiHashMap32 *own_tunnels;
 
    /**
      * Tunnels this client has accepted, indexed by incoming local id
      */
-  struct GNUNET_CONTAINER_MultiHashMap *incoming_tunnels;
+  struct GNUNET_CONTAINER_MultiHashMap32 *incoming_tunnels;
 
     /**
      * Handle to communicate with the client
@@ -434,9 +534,9 @@ struct MeshClient
 
     /**
      * Ports that this client has declared interest in.
-     * Indexed by a GMC_hash32 (type), contains *Client.
+     * Indexed by port, contains *Client.
      */
-  struct GNUNET_CONTAINER_MultiHashMap *ports;
+  struct GNUNET_CONTAINER_MultiHashMap32 *ports;
 
     /**
      * Whether the client is active or shutting down (don't send confirmations
@@ -538,11 +638,6 @@ static unsigned long long drop_percent;
 
 /*************************** Static global variables **************************/
 
-/**
- * Hostkey generation context
- */
-static struct GNUNET_CRYPTO_EccKeyGenerationContext *keygen;
-
 /**
  * DLL with all the clients, head.
  */
@@ -567,7 +662,7 @@ static unsigned long long n_tunnels;
  * Tunnels incoming, indexed by MESH_TunnelNumber
  * (which is greater than GNUNET_MESH_LOCAL_TUNNEL_ID_SERV).
  */
-static struct GNUNET_CONTAINER_MultiHashMap *incoming_tunnels;
+static struct GNUNET_CONTAINER_MultiHashMap32 *incoming_tunnels;
 
 /**
  * Peers known, indexed by PeerIdentity (MeshPeerInfo).
@@ -637,7 +732,7 @@ static MESH_TunnelNumber next_local_tid;
 /**
  * All ports clients of this peer have opened.
  */
-static struct GNUNET_CONTAINER_MultiHashMap *ports;
+static struct GNUNET_CONTAINER_MultiHashMap32 *ports;
 
 /**
  * Task to periodically announce itself in the network.
@@ -892,16 +987,7 @@ announce_id (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 static struct MeshClient *
 client_get (struct GNUNET_SERVER_Client *client)
 {
-  struct MeshClient *c;
-
-  c = clients_head;
-  while (NULL != c)
-  {
-    if (c->handle == client)
-      return c;
-    c = c->next;
-  }
-  return NULL;
+  return GNUNET_SERVER_client_get_user_context (client, struct MeshClient);
 }
 
 
@@ -915,27 +1001,19 @@ client_get (struct GNUNET_SERVER_Client *client)
 static void
 client_delete_tunnel (struct MeshClient *c, struct MeshTunnel *t)
 {
-  struct GNUNET_HashCode hash;
-
   if (c == t->owner)
   {
-    GMC_hash32 (t->local_tid, &hash);
     GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (c->own_tunnels,
-                                                         &hash,
-                                                         t));
+                   GNUNET_CONTAINER_multihashmap32_remove (c->own_tunnels,
+                                                           t->local_tid,
+                                                           t));
   }
-  else if (c == t->client)
+  if (c == t->client)
   {
-    GMC_hash32 (t->local_tid_dest, &hash);
     GNUNET_assert (GNUNET_YES ==
-                   GNUNET_CONTAINER_multihashmap_remove (c->incoming_tunnels,
-                                                         &hash,
-                                                         t));
-  }
-  else
-  {
-    GNUNET_break (0);
+                   GNUNET_CONTAINER_multihashmap32_remove (c->incoming_tunnels,
+                                                           t->local_tid_dest,
+                                                           t));
   }
 }
 
@@ -955,6 +1033,10 @@ send_client_tunnel_create (struct MeshTunnel *t)
   msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_CREATE);
   msg.tunnel_id = htonl (t->local_tid_dest);
   msg.port = htonl (t->port);
+  msg.options = 0;
+  msg.options |= GNUNET_YES == t->reliable ? GNUNET_MESH_OPTION_RELIABLE : 0;
+  msg.options |= GNUNET_YES == t->nobuffer ? GNUNET_MESH_OPTION_NOBUFFER : 0;
+  msg.options = htonl (msg.options);
   GNUNET_PEER_resolve (t->id.oid, &msg.peer);
   GNUNET_SERVER_notification_context_unicast (nc, t->client->handle,
                                               &msg.header, GNUNET_NO);
@@ -1581,17 +1663,22 @@ tunnel_poll (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     return;
   }
 
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " *** Polling!\n");
+
   msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_POLL);
   msg.header.size = htons (sizeof (msg));
   msg.tid = htonl (t->id.tid);
+  msg.pid = htonl (fc->last_pid_sent);
   GNUNET_PEER_resolve (t->id.oid, &msg.oid);
 
   if (fc == &t->prev_fc)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " *** prev peer!\n");
     peer = t->prev_hop;
   }
   else if (fc == &t->next_fc)
   {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " *** next peer!\n");
     peer = t->next_hop;
   }
   else
@@ -1739,11 +1826,8 @@ path_refresh (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
 static struct MeshTunnel *
 tunnel_get_incoming (MESH_TunnelNumber tid)
 {
-  struct GNUNET_HashCode hash;
-
   GNUNET_assert (tid >= GNUNET_MESH_LOCAL_TUNNEL_ID_SERV);
-  GMC_hash32 (tid, &hash);
-  return GNUNET_CONTAINER_multihashmap_get (incoming_tunnels, &hash);
+  return GNUNET_CONTAINER_multihashmap32_get (incoming_tunnels, tid);
 }
 
 
@@ -1764,10 +1848,8 @@ tunnel_get_by_local_id (struct MeshClient *c, MESH_TunnelNumber tid)
   }
   else
   {
-    struct GNUNET_HashCode hash;
-
-    GMC_hash32 (tid, &hash);
-    return GNUNET_CONTAINER_multihashmap_get (c->own_tunnels, &hash);
+    GNUNET_assert (tid >= GNUNET_MESH_LOCAL_TUNNEL_ID_CLI);
+    return GNUNET_CONTAINER_multihashmap32_get (c->own_tunnels, tid);
   }
 }
 
@@ -1818,24 +1900,23 @@ tunnel_get (const struct GNUNET_PeerIdentity *oid, MESH_TunnelNumber tid)
 static void
 tunnel_add_client (struct MeshTunnel *t, struct MeshClient *c)
 {
-  struct GNUNET_HashCode hash;
-
   if (NULL != t->client)
   {
     GNUNET_break(0);
     return;
   }
-  GMC_hash32 (t->local_tid_dest, &hash);
   if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (c->incoming_tunnels, &hash, t,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
+      GNUNET_CONTAINER_multihashmap32_put (c->incoming_tunnels,
+                                           t->local_tid_dest, t,
+                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
   {
     GNUNET_break (0);
     return;
   }
   if (GNUNET_OK !=
-      GNUNET_CONTAINER_multihashmap_put (incoming_tunnels, &hash, t,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
+      GNUNET_CONTAINER_multihashmap32_put (incoming_tunnels,
+                                           t->local_tid_dest, t,
+                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST))
   {
     GNUNET_break (0);
     return;
@@ -1930,18 +2011,19 @@ tunnel_notify_connection_broken (struct MeshTunnel *t, GNUNET_PEER_Id p1,
  * 
  * @param t Tunnel on which to send the ACK.
  * @param c Client to whom send the ACK.
- * @param ack Value of the ACK.
+ * @param is_fwd Set to GNUNET_YES for FWD ACK (dest->owner)
  */
 static void
-send_local_ack (struct MeshTunnel *t, struct MeshClient *c, uint32_t ack)
+send_local_ack (struct MeshTunnel *t,
+                struct MeshClient *c,
+                int is_fwd)
 {
   struct GNUNET_MESH_LocalAck msg;
 
   msg.header.size = htons (sizeof (msg));
   msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK);
-  msg.tunnel_id = htonl (t->owner == c ? t->local_tid : t->local_tid_dest);
-  msg.ack = htonl (ack); 
-  GNUNET_SERVER_notification_context_unicast(nc,
+  msg.tunnel_id = htonl (is_fwd ? t->local_tid : t->local_tid_dest);
+  GNUNET_SERVER_notification_context_unicast (nc,
                                               c->handle,
                                               &msg.header,
                                               GNUNET_NO);
@@ -1969,6 +2051,48 @@ send_ack (struct MeshTunnel *t, GNUNET_PEER_Id peer,  uint32_t ack)
 }
 
 
+/**
+ * Send an end-to-end FWD ACK message for the most recent in-sequence payload.
+ * 
+ * @param t Tunnel this is about.
+ */
+static void
+tunnel_send_fwd_data_ack (struct MeshTunnel *t)
+{
+  struct GNUNET_MESH_DataACK msg;
+
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_UNICAST_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.tid = htonl (t->id.tid);
+  GNUNET_PEER_resolve (t->id.oid, &msg.oid);
+  msg.mid = GNUNET_htonll (t->bck_rel->mid_recv);
+  msg.futures = 0; // FIXME set bits of other newer messages received
+
+  send_prebuilt_message (&msg.header, t->prev_hop, t);
+}
+
+
+/**
+ * Send an end-to-end BCK ACK message for the most recent in-sequence payload.
+ * 
+ * @param t Tunnel this is about.
+ */
+static void
+tunnel_send_bck_data_ack (struct MeshTunnel *t)
+{
+  struct GNUNET_MESH_DataACK msg;
+
+  msg.header.type = htons (GNUNET_MESSAGE_TYPE_MESH_TO_ORIG_ACK);
+  msg.header.size = htons (sizeof (msg));
+  msg.tid = htonl (t->id.tid);
+  GNUNET_PEER_resolve (t->id.oid, &msg.oid);
+  msg.mid = GNUNET_htonll (t->fwd_rel->mid_recv);
+  msg.futures = 0; // FIXME set bits of other newer messages received
+
+  send_prebuilt_message (&msg.header, t->next_hop, t);
+}
+
+
 /**
  * Send an ACK informing the predecessor about the available buffer space.
  * In case there is no predecessor, inform the owning client.
@@ -1984,6 +2108,7 @@ static void
 tunnel_send_fwd_ack (struct MeshTunnel *t, uint16_t type)
 {
   uint32_t ack;
+  int delta;
 
   /* Is it after unicast retransmission? */
   switch (type)
@@ -1996,12 +2121,14 @@ tunnel_send_fwd_ack (struct MeshTunnel *t, uint16_t type)
         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Not sending ACK, nobuffer\n");
         return;
       }
+      if (GNUNET_YES == t->reliable && NULL != t->client)
+        tunnel_send_fwd_data_ack (t);
       break;
     case GNUNET_MESSAGE_TYPE_MESH_ACK:
     case GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK:
       break;
-    case GNUNET_MESSAGE_TYPE_MESH_PATH_ACK:
     case GNUNET_MESSAGE_TYPE_MESH_POLL:
+    case GNUNET_MESSAGE_TYPE_MESH_PATH_ACK:
       t->force_ack = GNUNET_YES;
       break;
     default:
@@ -2009,11 +2136,12 @@ tunnel_send_fwd_ack (struct MeshTunnel *t, uint16_t type)
   }
 
   /* Check if we need to transmit the ACK */
-  if (t->queue_max > t->next_fc.queue_n * 4 &&
+  if (NULL == t->owner && 
+      t->queue_max > t->next_fc.queue_n * 4 &&
       GMC_is_pid_bigger(t->prev_fc.last_ack_sent, t->prev_fc.last_pid_recv) &&
       GNUNET_NO == t->force_ack)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Not sending ACK, buffer free\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Not sending FWD ACK, buffer free\n");
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "  t->qmax: %u, t->qn: %u\n",
                 t->queue_max, t->next_fc.queue_n);
@@ -2024,7 +2152,14 @@ tunnel_send_fwd_ack (struct MeshTunnel *t, uint16_t type)
   }
 
   /* Ok, ACK might be necessary, what PID to ACK? */
-  ack = t->prev_fc.last_pid_recv + t->queue_max - t->next_fc.queue_n;
+  delta = t->queue_max - t->next_fc.queue_n;
+  if (0 > delta)
+    delta = 0;
+  ack = t->prev_fc.last_pid_recv + delta;
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " FWD ACK %u\n", ack);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              " last %u, qmax %u, q %u\n",
+              t->prev_fc.last_pid_recv, t->queue_max, t->next_fc.queue_n);
   if (ack == t->prev_fc.last_ack_sent && GNUNET_NO == t->force_ack)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Not sending FWD ACK, not needed\n");
@@ -2033,7 +2168,7 @@ tunnel_send_fwd_ack (struct MeshTunnel *t, uint16_t type)
 
   t->prev_fc.last_ack_sent = ack;
   if (NULL != t->owner)
-    send_local_ack (t, t->owner, ack);
+    send_local_ack (t, t->owner, GNUNET_YES);
   else if (0 != t->prev_hop)
     send_ack (t, t->prev_hop, ack);
   else
@@ -2071,6 +2206,8 @@ tunnel_send_bck_ack (struct MeshTunnel *t, uint16_t type)
                     "    Not sending ACK, nobuffer + traffic\n");
         return;
       }
+      if (GNUNET_YES == t->reliable && NULL != t->owner)
+        tunnel_send_bck_data_ack (t);
       break;
     case GNUNET_MESSAGE_TYPE_MESH_ACK:
     case GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK:
@@ -2100,7 +2237,7 @@ tunnel_send_bck_ack (struct MeshTunnel *t, uint16_t type)
   t->next_fc.last_ack_sent = ack;
 
   if (NULL != t->client)
-    send_local_ack (t, t->client, ack);
+    send_local_ack (t, t->client, GNUNET_NO);
   else if (0 != t->next_hop)
     send_ack (t, t->next_hop, ack);
   else
@@ -2110,37 +2247,54 @@ tunnel_send_bck_ack (struct MeshTunnel *t, uint16_t type)
 
 
 /**
- * Modify the unicast message TID from global to local and send to client.
+ * Modify the mesh message TID from global to local and send to client.
  * 
  * @param t Tunnel on which to send the message.
  * @param msg Message to modify and send.
+ * @param c Client to send to.
+ * @param tid Tunnel ID to use (c can be both owner and client).
  */
 static void
-tunnel_send_client_ucast (struct MeshTunnel *t,
-                          const struct GNUNET_MESH_Data *msg)
+tunnel_send_client_data (struct MeshTunnel *t,
+                         const struct GNUNET_MESH_Data *msg,
+                         struct MeshClient *c, MESH_TunnelNumber tid)
 {
-  struct GNUNET_MESH_Data *copy;
-  uint16_t size = ntohs (msg->header.size);
-  char cbuf[size];
+  struct GNUNET_MESH_LocalData *copy;
+  uint16_t size = ntohs (msg->header.size) - sizeof (struct GNUNET_MESH_Data);
+  char cbuf[size + sizeof (struct GNUNET_MESH_LocalData)];
 
-  if (size < sizeof (struct GNUNET_MESH_Data) +
-             sizeof (struct GNUNET_MessageHeader))
+  if (size < sizeof (struct GNUNET_MessageHeader))
   {
     GNUNET_break_op (0);
     return;
   }
-  if (NULL == t->client)
+  if (NULL == c)
   {
     GNUNET_break (0);
     return;
   }
-  copy = (struct GNUNET_MESH_Data *) cbuf;
-  memcpy (copy, msg, size);
-  copy->tid = htonl (t->local_tid_dest);
-  GNUNET_SERVER_notification_context_unicast (nc, t->client->handle,
+  copy = (struct GNUNET_MESH_LocalData *) cbuf;
+  memcpy (&copy[1], &msg[1], size);
+  copy->header.size = htons (sizeof (struct GNUNET_MESH_LocalData) + size);
+  copy->header.type = htons (GNUNET_MESSAGE_TYPE_MESH_LOCAL_DATA);
+  copy->tid = htonl (tid);
+  GNUNET_SERVER_notification_context_unicast (nc, c->handle,
                                               &copy->header, GNUNET_NO);
 }
 
+/**
+ * Modify the unicast message TID from global to local and send to client.
+ * 
+ * @param t Tunnel on which to send the message.
+ * @param msg Message to modify and send.
+ */
+static void
+tunnel_send_client_ucast (struct MeshTunnel *t,
+                          const struct GNUNET_MESH_Data *msg)
+{
+  tunnel_send_client_data (t, msg, t->client, t->local_tid_dest);
+}
+
 
 /**
  * Modify the to_origin  message TID from global to local and send to client.
@@ -2152,26 +2306,78 @@ static void
 tunnel_send_client_to_orig (struct MeshTunnel *t,
                             const struct GNUNET_MESH_Data *msg)
 {
-  struct GNUNET_MESH_Data *copy;
-  uint16_t size = ntohs (msg->header.size);
-  char cbuf[size];
+  tunnel_send_client_data (t, msg, t->owner, t->local_tid);
+}
 
-  if (size < sizeof (struct GNUNET_MESH_Data) +
-             sizeof (struct GNUNET_MessageHeader))
-  {
-    GNUNET_break_op (0);
+
+/**
+ * We haven't received an ACK after a certain time: restransmit the message.
+ *
+ * @param cls Closure (MeshReliableMessage with the message to restransmit)
+ * @param tc TaskContext.
+ */
+static void
+tunnel_retransmit_message (void *cls,
+                           const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct MeshTunnelReliability *rel = cls;
+  struct MeshReliableMessage *copy;
+  struct MeshFlowControl *fc;
+  struct MeshPeerQueue *q;
+  struct MeshPeerInfo *pi;
+  struct MeshTunnel *t;
+  struct GNUNET_MESH_Data *payload;
+  GNUNET_PEER_Id hop;
+
+  rel->retry_task = GNUNET_SCHEDULER_NO_TASK;
+  if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     return;
-  }
-  if (NULL == t->owner)
+
+  t = rel->t;
+  copy = rel->head_sent;
+  if (NULL == copy)
   {
     GNUNET_break (0);
     return;
   }
-  copy = (struct GNUNET_MESH_Data *) cbuf;
-  memcpy (cbuf, msg, size);
-  copy->tid = htonl (t->local_tid);
-  GNUNET_SERVER_notification_context_unicast (nc, t->owner->handle,
-                                              &copy->header, GNUNET_NO);
+
+  /* Search the message to be retransmitted in the outgoing queue */
+  payload = (struct GNUNET_MESH_Data *) &copy[1];
+  hop = rel == t->fwd_rel ? t->next_hop : t->prev_hop;
+  fc = rel == t->fwd_rel ? &t->next_fc : &t->prev_fc;
+  pi = peer_get_short (hop);
+  for (q = pi->queue_head; NULL != q; q = q->next)
+  {
+    if (ntohs (payload->header.type) == q->type)
+    {
+      struct GNUNET_MESH_Data *queued_data = q->cls;
+
+      if (queued_data->mid == payload->mid)
+        break;
+    }
+  }
+
+  /* Message not found in the queue */
+  if (NULL == q)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! RETRANSMIT %llu\n", copy->mid);
+    copy->timestamp = GNUNET_TIME_absolute_get ();
+
+    fc->last_ack_sent++;
+    payload->pid = htonl (fc->last_pid_recv + 1);
+    fc->last_pid_recv++;
+    send_prebuilt_message (&payload->header, hop, t);
+    GNUNET_STATISTICS_update (stats, "# data retransmitted", 1, GNUNET_NO);
+  }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! STILL IN QUEUE %llu\n", copy->mid);
+  }
+
+  rel->retry_timer = GNUNET_TIME_STD_BACKOFF (rel->retry_timer);
+  rel->retry_task = GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
+                                                  &tunnel_retransmit_message,
+                                                  cls);
 }
 
 
@@ -2356,9 +2562,8 @@ tunnel_destroy (struct MeshTunnel *t)
 
   if (NULL != c)
   {
-    GMC_hash32 (t->local_tid, &hash);
     if (GNUNET_YES !=
-        GNUNET_CONTAINER_multihashmap_remove (c->own_tunnels, &hash, t))
+      GNUNET_CONTAINER_multihashmap32_remove (c->own_tunnels, t->local_tid, t))
     {
       GNUNET_break (0);
       r = GNUNET_SYSERR;
@@ -2368,15 +2573,16 @@ tunnel_destroy (struct MeshTunnel *t)
   if (NULL != t->client)
   {
     c = t->client;
-    GMC_hash32 (t->local_tid_dest, &hash);
     if (GNUNET_YES !=
-          GNUNET_CONTAINER_multihashmap_remove (c->incoming_tunnels, &hash, t))
+        GNUNET_CONTAINER_multihashmap32_remove (c->incoming_tunnels,
+                                                t->local_tid_dest, t))
     {
       GNUNET_break (0);
       r = GNUNET_SYSERR;
     }
     if (GNUNET_YES != 
-      GNUNET_CONTAINER_multihashmap_remove (incoming_tunnels, &hash, t))
+        GNUNET_CONTAINER_multihashmap32_remove (incoming_tunnels,
+                                                t->local_tid_dest, t))
     {
       GNUNET_break (0);
       r = GNUNET_SYSERR;
@@ -2503,10 +2709,10 @@ tunnel_new (GNUNET_PEER_Id owner,
 
   if (NULL != client)
   {
-    GMC_hash32 (t->local_tid, &hash);
     if (GNUNET_OK !=
-        GNUNET_CONTAINER_multihashmap_put (client->own_tunnels, &hash, t,
-                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
+        GNUNET_CONTAINER_multihashmap32_put (client->own_tunnels,
+                                             t->local_tid, t,
+                                             GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
     {
       tunnel_destroy (t);
       GNUNET_break (0);
@@ -2519,19 +2725,34 @@ tunnel_new (GNUNET_PEER_Id owner,
 }
 
 
+/**
+ * Set options in a tunnel, extracted from a bit flag field
+ * 
+ * @param t Tunnel to set options to.
+ * @param options Bit array in host byte order.
+ */
+static void
+tunnel_set_options (struct MeshTunnel *t, uint32_t options)
+{
+  t->nobuffer = (options & GNUNET_MESH_OPTION_NOBUFFER) != 0 ?
+                 GNUNET_YES : GNUNET_NO;
+  t->reliable = (options & GNUNET_MESH_OPTION_RELIABLE) != 0 ?
+                 GNUNET_YES : GNUNET_NO;
+}
+
 
 /**
  * Iterator for deleting each tunnel whose client endpoint disconnected.
  *
  * @param cls Closure (client that has disconnected).
- * @param key The hash of the local tunnel id (used to access the hashmap).
+ * @param key The local tunnel id (used to access the hashmap).
  * @param value The value stored at the key (tunnel to destroy).
  *
  * @return GNUNET_OK, keep iterating.
  */
 static int
 tunnel_destroy_iterator (void *cls,
-                         const struct GNUNET_HashCode * key,
+                         uint32_t key,
                          void *value)
 {
   struct MeshTunnel *t = value;
@@ -2550,7 +2771,7 @@ tunnel_destroy_iterator (void *cls,
         t->next_hop = 0;
     }
   }
-  else if (c == t->owner)
+  if (c == t->owner)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " Client %u is owner.\n", c->id);
     t->owner = NULL;
@@ -2559,10 +2780,7 @@ tunnel_destroy_iterator (void *cls,
         t->prev_hop = 0;
     }
   }
-  else
-  {
-    GNUNET_break (0);
-  }
+
   tunnel_destroy_empty (t);
 
   return GNUNET_OK;
@@ -2654,7 +2872,9 @@ send_core_path_create (void *cls, size_t size, void *buf)
 
   opt = 0;
   if (GNUNET_YES == t->nobuffer)
-    opt |= MESH_TUNNEL_OPT_NOBUFFER;
+    opt |= GNUNET_MESH_OPTION_NOBUFFER;
+  if (GNUNET_YES == t->reliable)
+    opt |= GNUNET_MESH_OPTION_RELIABLE;
   msg->opt = htonl (opt);
   msg->port = htonl (t->port);
 
@@ -2762,6 +2982,7 @@ queue_destroy (struct MeshPeerQueue *queue, int clear_cls)
     else
     {
       GNUNET_break (0);
+      GNUNET_free (queue);
       return;
     }
     fc->queue_n--;
@@ -2894,6 +3115,8 @@ queue_send (void *cls, size_t size, void *buf)
   {
     case 0:
     case GNUNET_MESSAGE_TYPE_MESH_ACK:
+    case GNUNET_MESSAGE_TYPE_MESH_UNICAST_ACK:
+    case GNUNET_MESSAGE_TYPE_MESH_TO_ORIG_ACK:
     case GNUNET_MESSAGE_TYPE_MESH_POLL:
     case GNUNET_MESSAGE_TYPE_MESH_PATH_BROKEN:
     case GNUNET_MESSAGE_TYPE_MESH_PATH_DESTROY:
@@ -2938,6 +3161,7 @@ queue_send (void *cls, size_t size, void *buf)
 
   /* Send ACK if needed, after accounting for sent ID in fc->queue_n */
   pid = ((struct GNUNET_MESH_Data *) buf)->pid;
+  pid = ntohl (pid);
   switch (type)
   {
     case GNUNET_MESSAGE_TYPE_MESH_UNICAST:
@@ -2988,7 +3212,7 @@ queue_send (void *cls, size_t size, void *buf)
     else
     {
       GNUNET_break (0);
-      fc = NULL;
+      return data_size;
     }
     if (NULL != fc && GNUNET_SCHEDULER_NO_TASK == fc->poll_task)
     {
@@ -3025,31 +3249,40 @@ queue_add (void *cls, uint16_t type, size_t size,
 {
   struct MeshPeerQueue *queue;
   struct GNUNET_PeerIdentity id;
-  unsigned int *n;
+  struct MeshFlowControl *fc;
+  int priority;
 
-  n = NULL;
+  fc = NULL;
+  priority = GNUNET_NO;
   if (GNUNET_MESSAGE_TYPE_MESH_UNICAST == type)
   {
-    n = &t->next_fc.queue_n;
+    fc = &t->next_fc;
   }
   else if (GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN == type)
   {
-    n = &t->prev_fc.queue_n;
+    fc = &t->prev_fc;
   }
-  if (NULL != n)
+  if (NULL != fc)
   {
-    if (*n >= t->queue_max)
+    if (fc->queue_n >= t->queue_max)
     {
-      GNUNET_break(0);
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+      GNUNET_break (0);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "queue full: %u/%u\n",
-                  *n, t->queue_max);
-      GNUNET_STATISTICS_update(stats,
-                               "# messages dropped (buffer full)",
-                               1, GNUNET_NO);
-      return; /* Drop message */
+                  fc->queue_n, t->queue_max);
+
+      /* If this isn't a retransmission, drop the message */
+      if (GNUNET_NO == t->reliable ||
+          (NULL == t->owner && GNUNET_MESSAGE_TYPE_MESH_UNICAST == type) ||
+          (NULL == t->client && GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN == type))
+      {
+        GNUNET_STATISTICS_update (stats, "# messages dropped (buffer full)",
+                                  1, GNUNET_NO);
+        return; /* Drop this message */
+      }
+      priority = GNUNET_YES;
     }
-    (*n)++;
+    fc->queue_n++;
   }
   queue = GNUNET_malloc (sizeof (struct MeshPeerQueue));
   queue->cls = cls;
@@ -3057,7 +3290,10 @@ queue_add (void *cls, uint16_t type, size_t size,
   queue->size = size;
   queue->peer = dst;
   queue->tunnel = t;
-  GNUNET_CONTAINER_DLL_insert_tail (dst->queue_head, dst->queue_tail, queue);
+  if (GNUNET_YES == priority)
+    GNUNET_CONTAINER_DLL_insert (dst->queue_head, dst->queue_tail, queue);
+  else
+    GNUNET_CONTAINER_DLL_insert_tail (dst->queue_head, dst->queue_tail, queue);
   if (NULL == dst->core_transmit)
   {
     GNUNET_PEER_resolve (dst->id, &id);
@@ -3148,11 +3384,15 @@ handle_mesh_path_create (void *cls, const struct GNUNET_PeerIdentity *peer,
     }
     t->port = ntohl (msg->port);
     opt = ntohl (msg->opt);
-    if (0 != (opt & MESH_TUNNEL_OPT_NOBUFFER))
+    if (0 != (opt & GNUNET_MESH_OPTION_NOBUFFER))
     {
       t->nobuffer = GNUNET_YES;
       t->queue_max = 1;
     }
+    if (0 != (opt & GNUNET_MESH_OPTION_RELIABLE))
+    {
+      t->reliable = GNUNET_YES;
+    }
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  nobuffer:%d\n", t->nobuffer);
 
     tunnel_reset_timeout (t);
@@ -3208,11 +3448,9 @@ handle_mesh_path_create (void *cls, const struct GNUNET_PeerIdentity *peer,
   if (own_pos == size - 1)
   {
     struct MeshClient *c;
-    struct GNUNET_HashCode hc;
 
     /* Find target client */
-    GMC_hash32 (t->port, &hc);
-    c = GNUNET_CONTAINER_multihashmap_get (ports, &hc);
+    c = GNUNET_CONTAINER_multihashmap32_get (ports, t->port);
     if (NULL == c)
     {
       /* TODO send reject */
@@ -3228,6 +3466,14 @@ handle_mesh_path_create (void *cls, const struct GNUNET_PeerIdentity *peer,
     t->local_tid_dest = next_local_tid++;
     next_local_tid = next_local_tid | GNUNET_MESH_LOCAL_TUNNEL_ID_SERV;
 
+    if (GNUNET_YES == t->reliable)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! Reliable\n");
+      t->bck_rel = GNUNET_malloc (sizeof (struct MeshTunnelReliability));
+      t->bck_rel->t = t;
+      t->bck_rel->expected_delay = MESH_RETRANSMIT_TIME;
+    }
+
     tunnel_add_client (t, c);
     send_client_tunnel_create (t);
     send_path_ack (t);
@@ -3482,19 +3728,6 @@ handle_mesh_unicast (void *cls, const struct GNUNET_PeerIdentity *peer,
     return GNUNET_OK;
   }
   pid = ntohl (msg->pid);
-  if (t->prev_fc.last_pid_recv == pid)
-  {
-    GNUNET_STATISTICS_update (stats, "# duplicate PID drops", 1, GNUNET_NO);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                " Already seen pid %u, DROPPING!\n", pid);
-    return GNUNET_OK;
-  }
-  else
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                " pid %u not seen yet, forwarding\n", pid);
-  }
-
   if (GMC_is_pid_bigger (pid, t->prev_fc.last_ack_sent))
   {
     GNUNET_STATISTICS_update (stats, "# unsolicited unicast", 1, GNUNET_NO);
@@ -3505,7 +3738,6 @@ handle_mesh_unicast (void *cls, const struct GNUNET_PeerIdentity *peer,
     tunnel_send_fwd_ack(t, GNUNET_MESSAGE_TYPE_MESH_POLL);
     return GNUNET_OK;
   }
-  t->prev_fc.last_pid_recv = pid;
 
   tunnel_reset_timeout (t);
   if (t->dest == myid)
@@ -3514,10 +3746,30 @@ handle_mesh_unicast (void *cls, const struct GNUNET_PeerIdentity *peer,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "  it's for us! sending to clients...\n");
     GNUNET_STATISTICS_update (stats, "# unicast received", 1, GNUNET_NO);
-    tunnel_send_client_ucast (t, msg);
+//     if (GMC_is_pid_bigger(pid, t->prev_fc.last_pid_recv)) FIXME use
+    if (GMC_is_pid_bigger (pid, t->prev_fc.last_pid_recv)
+        &&
+          (GNUNET_NO == t->reliable ||
+           GNUNET_ntohll (msg->mid) == (t->bck_rel->mid_recv + 1)) )
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  " pid %u not seen yet, forwarding\n", pid);
+      t->prev_fc.last_pid_recv = pid;
+      t->bck_rel->mid_recv++;
+      tunnel_send_client_ucast (t, msg);
+    }
+    else
+    {
+//       GNUNET_STATISTICS_update (stats, "# duplicate PID", 1, GNUNET_NO);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  " Pid %u not expected (%u), sending FWD ACK!\n",
+                  pid, t->prev_fc.last_pid_recv + 1);
+    }
     tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_UNICAST);
     return GNUNET_OK;
   }
+  if (GMC_is_pid_bigger(pid, t->prev_fc.last_pid_recv))
+    t->prev_fc.last_pid_recv = pid;
   if (0 == t->next_hop)
   {
     GNUNET_break (0);
@@ -3575,7 +3827,6 @@ handle_mesh_to_orig (void *cls, const struct GNUNET_PeerIdentity *peer,
               GNUNET_MESH_DEBUG_M2S (ntohs (msg[1].header.type)));
   t = tunnel_get (&msg->oid, ntohl (msg->tid));
   pid = ntohl (msg->pid);
-
   if (NULL == t)
   {
     /* TODO notify that we dont know this tunnel (whom)? */
@@ -3586,16 +3837,6 @@ handle_mesh_to_orig (void *cls, const struct GNUNET_PeerIdentity *peer,
     return GNUNET_OK;
   }
 
-  if (t->next_fc.last_pid_recv == pid)
-  {
-    /* already seen this packet, drop */
-    GNUNET_STATISTICS_update (stats, "# duplicate PID drops BCK", 1, GNUNET_NO);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                " Already seen pid %u, DROPPING!\n", pid);
-    tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_ACK);
-    return GNUNET_OK;
-  }
-
   if (GMC_is_pid_bigger (pid, t->next_fc.last_ack_sent))
   {
     GNUNET_STATISTICS_update (stats, "# unsolicited to_orig", 1, GNUNET_NO);
@@ -3606,9 +3847,6 @@ handle_mesh_to_orig (void *cls, const struct GNUNET_PeerIdentity *peer,
     tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_POLL);
     return GNUNET_OK;
   }
-  t->next_fc.last_pid_recv = pid;
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              " pid %u not seen yet, forwarding\n", pid);
 
   if (myid == t->id.oid)
   {
@@ -3616,13 +3854,28 @@ handle_mesh_to_orig (void *cls, const struct GNUNET_PeerIdentity *peer,
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "  it's for us! sending to clients...\n");
     GNUNET_STATISTICS_update (stats, "# to origin received", 1, GNUNET_NO);
-    tunnel_send_client_to_orig (t, msg);
+    if ( (GNUNET_NO == t->reliable &&
+          GMC_is_pid_bigger(pid, t->next_fc.last_pid_recv))
+        ||
+          (GNUNET_YES == t->reliable &&
+           pid == t->next_fc.last_pid_recv + 1) ) // FIXME use "futures" as accepting
+    {
+      t->next_fc.last_pid_recv = pid;
+      tunnel_send_client_to_orig (t, msg);
+    }
+    else
+    {
+//       GNUNET_STATISTICS_update (stats, "# duplicate PID drops BCK", 1, GNUNET_NO);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  " Pid %u not expected, sending FWD ACK!\n", pid);
+    }
     tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN);
     return GNUNET_OK;
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "  not for us, retransmitting...\n");
-
+  if (GMC_is_pid_bigger (pid, t->next_fc.last_pid_recv))
+    t->next_fc.last_pid_recv = pid;
   if (0 == t->prev_hop) /* No owner AND no prev hop */
   {
     if (GNUNET_YES == t->destroy)
@@ -3660,76 +3913,201 @@ handle_mesh_to_orig (void *cls, const struct GNUNET_PeerIdentity *peer,
 
 
 /**
- * Core handler for mesh network traffic point-to-point acks.
+ * Core handler for mesh network traffic end-to-end ACKs.
  *
- * @param cls closure
- * @param message message
- * @param peer peer identity this notification is about
+ * @param cls Closure.
+ * @param message Message.
+ * @param peer Peer identity this notification is about.
  *
  * @return GNUNET_OK to keep the connection open,
  *         GNUNET_SYSERR to close it (signal serious error)
  */
 static int
-handle_mesh_ack (void *cls, const struct GNUNET_PeerIdentity *peer,
-                 const struct GNUNET_MessageHeader *message)
+handle_mesh_data_ack (void *cls, const struct GNUNET_PeerIdentity *peer,
+                      const struct GNUNET_MessageHeader *message)
 {
-  struct GNUNET_MESH_ACK *msg;
+  struct GNUNET_MESH_DataACK *msg;
+  struct MeshTunnelReliability *rel;
+  struct MeshReliableMessage *copy;
+  struct MeshReliableMessage *next;
   struct MeshTunnel *t;
   GNUNET_PEER_Id id;
-  uint32_t ack;
+  uint64_t ack;
+  uint16_t type;
+  int work;
 
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got an ACK packet from %s!\n",
-              GNUNET_i2s (peer));
-  msg = (struct GNUNET_MESH_ACK *) message;
+  type = ntohs (message->type);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got a %s message from %s!\n",
+              GNUNET_MESH_DEBUG_M2S (type), GNUNET_i2s (peer));
+  msg = (struct GNUNET_MESH_DataACK *) message;
 
   t = tunnel_get (&msg->oid, ntohl (msg->tid));
-
   if (NULL == t)
   {
     /* TODO notify that we dont know this tunnel (whom)? */
     GNUNET_STATISTICS_update (stats, "# ack on unknown tunnel", 1, GNUNET_NO);
     return GNUNET_OK;
   }
-  ack = ntohl (msg->pid);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  ACK %u\n", ack);
+  ack = GNUNET_ntohll (msg->mid);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  ACK %llu\n", ack);
 
   /* Is this a forward or backward ACK? */
   id = GNUNET_PEER_search (peer);
-  if (t->next_hop == id)
+  if (t->next_hop == id && GNUNET_MESSAGE_TYPE_MESH_UNICAST_ACK == type)
   {
-    debug_fwd_ack++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  FWD ACK\n");
-    if (GNUNET_SCHEDULER_NO_TASK != t->next_fc.poll_task &&
-        GMC_is_pid_bigger (ack, t->next_fc.last_ack_recv))
+    if (NULL == t->owner)
     {
-      GNUNET_SCHEDULER_cancel (t->next_fc.poll_task);
-      t->next_fc.poll_task = GNUNET_SCHEDULER_NO_TASK;
-      t->next_fc.poll_time = GNUNET_TIME_UNIT_SECONDS;
+      send_prebuilt_message (message, t->prev_hop, t);
+      return GNUNET_OK;
     }
-    t->next_fc.last_ack_recv = ack;
-    peer_unlock_queue (t->next_hop);
-    tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_ACK);
+    rel = t->fwd_rel;
+    tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_UNICAST);
   }
-  else if (t->prev_hop == id)
+  else if (t->prev_hop == id && GNUNET_MESSAGE_TYPE_MESH_TO_ORIG_ACK == type)
   {
-    debug_bck_ack++;
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  BCK ACK\n");
-    if (GNUNET_SCHEDULER_NO_TASK != t->prev_fc.poll_task &&
-        GMC_is_pid_bigger (ack, t->prev_fc.last_ack_recv))
+    if (NULL == t->client)
     {
-      GNUNET_SCHEDULER_cancel (t->prev_fc.poll_task);
-      t->prev_fc.poll_task = GNUNET_SCHEDULER_NO_TASK;
-      t->prev_fc.poll_time = GNUNET_TIME_UNIT_SECONDS;
+      send_prebuilt_message (message, t->next_hop, t);
+      return GNUNET_OK;
     }
-    t->prev_fc.last_ack_recv = ack;
-    peer_unlock_queue (t->prev_hop);
-    tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_ACK);
+    rel = t->bck_rel;
+    tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN);
   }
   else
+  {
     GNUNET_break_op (0);
-  return GNUNET_OK;
-}
-
+    return GNUNET_OK;
+  }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! ACK %llu\n", ack);
+  for (work = GNUNET_NO, copy = rel->head_sent; copy != NULL; copy = next)
+  {
+    struct GNUNET_TIME_Relative time;
+
+    if (copy->mid > ack)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!!  head %llu, out!\n", copy->mid);
+      break;
+    }
+    work = GNUNET_YES;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!!  id %llu\n", copy->mid);
+    next = copy->next;
+    time = GNUNET_TIME_absolute_get_duration (copy->timestamp);
+    rel->expected_delay.rel_value += time.rel_value;
+    rel->expected_delay.rel_value /= 2;
+    rel->n_sent--;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!!  new expected delay %s\n",
+                GNUNET_STRINGS_relative_time_to_string (rel->expected_delay,
+                                                        GNUNET_NO));
+    rel->retry_timer = rel->expected_delay;
+    GNUNET_CONTAINER_DLL_remove (rel->head_sent, rel->tail_sent, copy);
+    GNUNET_free (copy);
+  }
+
+  if (GNUNET_YES == work)
+  {
+    if (GNUNET_SCHEDULER_NO_TASK != rel->retry_task)
+    {
+      GNUNET_SCHEDULER_cancel (rel->retry_task);
+      if (NULL == rel->head_sent)
+      {
+        rel->retry_task = GNUNET_SCHEDULER_NO_TASK;
+      }
+      else
+      {
+        struct GNUNET_TIME_Absolute new_target;
+        struct GNUNET_TIME_Relative delay;
+
+        new_target = GNUNET_TIME_absolute_add (rel->head_sent->timestamp,
+                                               rel->retry_timer);
+        delay = GNUNET_TIME_absolute_get_remaining (new_target);
+        rel->retry_task =
+            GNUNET_SCHEDULER_add_delayed (delay,
+                                          &tunnel_retransmit_message,
+                                          rel);
+      }
+    }
+    else
+      GNUNET_break (0);
+  }
+  return GNUNET_OK;
+}
+
+/**
+ * Core handler for mesh network traffic point-to-point acks.
+ *
+ * @param cls closure
+ * @param message message
+ * @param peer peer identity this notification is about
+ *
+ * @return GNUNET_OK to keep the connection open,
+ *         GNUNET_SYSERR to close it (signal serious error)
+ */
+static int
+handle_mesh_ack (void *cls, const struct GNUNET_PeerIdentity *peer,
+                 const struct GNUNET_MessageHeader *message)
+{
+  struct GNUNET_MESH_ACK *msg;
+  struct MeshTunnel *t;
+  struct MeshFlowControl *fc;
+  GNUNET_PEER_Id id;
+  uint32_t ack;
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got an ACK packet from %s!\n",
+              GNUNET_i2s (peer));
+  msg = (struct GNUNET_MESH_ACK *) message;
+
+  t = tunnel_get (&msg->oid, ntohl (msg->tid));
+
+  if (NULL == t)
+  {
+    /* TODO notify that we dont know this tunnel (whom)? */
+    GNUNET_STATISTICS_update (stats, "# ack on unknown tunnel", 1, GNUNET_NO);
+    return GNUNET_OK;
+  }
+  ack = ntohl (msg->pid);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  ACK %u\n", ack);
+
+  /* Is this a forward or backward ACK? */
+  id = GNUNET_PEER_search (peer);
+  if (t->next_hop == id)
+  {
+    debug_fwd_ack++;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  FWD ACK\n");
+    fc = &t->next_fc;
+  }
+  else if (t->prev_hop == id)
+  {
+    debug_bck_ack++;
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  BCK ACK\n");
+    fc = &t->prev_fc;
+  }
+  else
+  {
+    GNUNET_break_op (0);
+    return GNUNET_OK;
+  }
+
+  if (GNUNET_SCHEDULER_NO_TASK != fc->poll_task &&
+      GMC_is_pid_bigger (ack, fc->last_ack_recv))
+  {
+    GNUNET_SCHEDULER_cancel (fc->poll_task);
+    fc->poll_task = GNUNET_SCHEDULER_NO_TASK;
+    fc->poll_time = GNUNET_TIME_UNIT_SECONDS;
+  }
+  fc->last_ack_recv = ack;
+  peer_unlock_queue (id);
+
+  if (t->next_hop == id)
+    tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_ACK);
+  else
+    tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_ACK);
+
+  return GNUNET_OK;
+}
+
 
 /**
  * Core handler for mesh network traffic point-to-point ack polls.
@@ -3747,7 +4125,10 @@ handle_mesh_poll (void *cls, const struct GNUNET_PeerIdentity *peer,
 {
   struct GNUNET_MESH_Poll *msg;
   struct MeshTunnel *t;
+  struct MeshFlowControl *fc;
   GNUNET_PEER_Id id;
+  uint32_t pid;
+  uint32_t old;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got an POLL packet from %s!\n",
               GNUNET_i2s (peer));
@@ -3766,18 +4147,31 @@ handle_mesh_poll (void *cls, const struct GNUNET_PeerIdentity *peer,
 
   /* Is this a forward or backward ACK? */
   id = GNUNET_PEER_search(peer);
+  pid = ntohl (msg->pid);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  PID %u\n", pid);
   if (t->next_hop == id)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  from FWD\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  was %u\n", t->next_fc.last_pid_recv);
+    fc = &t->next_fc;
+    old = fc->last_pid_recv;
+    fc->last_pid_recv = pid;
     tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_POLL);
   }
   else if (t->prev_hop == id)
   {
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  from BCK\n");
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  was %u\n", t->prev_fc.last_pid_recv);
+    fc = &t->prev_fc;
+    old = fc->last_pid_recv;
+    fc->last_pid_recv = pid;
     tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_POLL);
   }
   else
     GNUNET_break (0);
+  
+  if (GNUNET_YES == t->reliable)
+    fc->last_pid_recv = old;
 
   return GNUNET_OK;
 }
@@ -3816,6 +4210,8 @@ handle_mesh_keepalive (void *cls, const struct GNUNET_PeerIdentity *peer,
   }
 
   tunnel_reset_timeout (t);
+  if (NULL != t->client || 0 == t->next_hop || myid == t->next_hop)
+    return GNUNET_OK;
 
   GNUNET_STATISTICS_update (stats, "# keepalives forwarded", 1, GNUNET_NO);
   send_prebuilt_message (message, t->next_hop, t);
@@ -3834,9 +4230,13 @@ static struct GNUNET_CORE_MessageHandler core_handlers[] = {
   {&handle_mesh_tunnel_destroy, GNUNET_MESSAGE_TYPE_MESH_TUNNEL_DESTROY,
    sizeof (struct GNUNET_MESH_TunnelDestroy)},
   {&handle_mesh_unicast, GNUNET_MESSAGE_TYPE_MESH_UNICAST, 0},
+  {&handle_mesh_to_orig, GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN, 0},
+  {&handle_mesh_data_ack, GNUNET_MESSAGE_TYPE_MESH_UNICAST_ACK,
+    sizeof (struct GNUNET_MESH_DataACK)},
+  {&handle_mesh_data_ack, GNUNET_MESSAGE_TYPE_MESH_TO_ORIG_ACK,
+    sizeof (struct GNUNET_MESH_DataACK)},
   {&handle_mesh_keepalive, GNUNET_MESSAGE_TYPE_MESH_PATH_KEEPALIVE,
     sizeof (struct GNUNET_MESH_TunnelKeepAlive)},
-  {&handle_mesh_to_orig, GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN, 0},
   {&handle_mesh_ack, GNUNET_MESSAGE_TYPE_MESH_ACK,
     sizeof (struct GNUNET_MESH_ACK)},
   {&handle_mesh_poll, GNUNET_MESSAGE_TYPE_MESH_POLL,
@@ -4014,30 +4414,22 @@ handle_local_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
     return;
   }
 
-  c = clients_head;
-  while (NULL != c)
+  c = client_get (client);
+  if (NULL != c)
   {
-    if (c->handle != client)
-    {
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "   ... searching %p (%u)\n",
-                  c->handle, c->id);
-      c = c->next;
-      continue;
-    }
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "matching client found (%u)\n",
                 c->id);
     GNUNET_SERVER_client_drop (c->handle);
     c->shutting_down = GNUNET_YES;
-    GNUNET_CONTAINER_multihashmap_iterate (c->own_tunnels,
-                                           &tunnel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap_iterate (c->incoming_tunnels,
-                                           &tunnel_destroy_iterator, c);
-    GNUNET_CONTAINER_multihashmap_destroy (c->own_tunnels);
-    GNUNET_CONTAINER_multihashmap_destroy (c->incoming_tunnels);
+    GNUNET_CONTAINER_multihashmap32_iterate (c->own_tunnels,
+                                             &tunnel_destroy_iterator, c);
+    GNUNET_CONTAINER_multihashmap32_iterate (c->incoming_tunnels,
+                                             &tunnel_destroy_iterator, c);
+    GNUNET_CONTAINER_multihashmap32_destroy (c->own_tunnels);
+    GNUNET_CONTAINER_multihashmap32_destroy (c->incoming_tunnels);
 
     if (NULL != c->ports)
-      GNUNET_CONTAINER_multihashmap_destroy (c->ports);
+      GNUNET_CONTAINER_multihashmap32_destroy (c->ports);
     next = c->next;
     GNUNET_CONTAINER_DLL_remove (clients_head, clients_tail, c);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  CLIENT FREE at %p\n", c);
@@ -4045,6 +4437,10 @@ handle_local_client_disconnect (void *cls, struct GNUNET_SERVER_Client *client)
     GNUNET_STATISTICS_update (stats, "# clients", -1, GNUNET_NO);
     c = next;
   }
+  else
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, " context NULL!\n");
+  }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "done!\n");
   return;
 }
@@ -4087,33 +4483,32 @@ handle_local_new_client (void *cls, struct GNUNET_SERVER_Client *client,
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  client has %u ports\n", size);
   c->handle = client;
   GNUNET_SERVER_client_keep (client);
+  GNUNET_SERVER_client_set_user_context (client, c);
   if (size > 0)
   {
     uint32_t u32;
-    struct GNUNET_HashCode hc;
 
     p = (uint32_t *) &cc_msg[1];
-    c->ports = GNUNET_CONTAINER_multihashmap_create (size, GNUNET_NO);
+    c->ports = GNUNET_CONTAINER_multihashmap32_create (size);
     for (i = 0; i < size; i++)
     {
       u32 = ntohl (p[i]);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "    port: %u\n", u32);
-      GMC_hash32 (u32, &hc);
 
       /* store in client's hashmap */
-      GNUNET_CONTAINER_multihashmap_put (c->ports, &hc, c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
+      GNUNET_CONTAINER_multihashmap32_put (c->ports, u32, c,
+                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_FAST);
       /* store in global hashmap */
       /* FIXME only allow one client to have the port open,
        *       have a backup hashmap with waiting clients */
-      GNUNET_CONTAINER_multihashmap_put (ports, &hc, c,
-                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+      GNUNET_CONTAINER_multihashmap32_put (ports, u32, c,
+                                           GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
     }
   }
 
   GNUNET_CONTAINER_DLL_insert (clients_head, clients_tail, c);
-  c->own_tunnels = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
-  c->incoming_tunnels = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
+  c->own_tunnels = GNUNET_CONTAINER_multihashmap32_create (32);
+  c->incoming_tunnels = GNUNET_CONTAINER_multihashmap32_create (32);
   GNUNET_SERVER_notification_context_add (nc, client);
   GNUNET_STATISTICS_update (stats, "# clients", 1, GNUNET_NO);
 
@@ -4189,6 +4584,15 @@ handle_local_tunnel_create (void *cls, struct GNUNET_SERVER_Client *client,
     return;
   }
   t->port = ntohl (t_msg->port);
+  tunnel_set_options (t, ntohl (t_msg->options));
+  if (GNUNET_YES == t->reliable)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! Reliable\n");
+    t->fwd_rel = GNUNET_malloc (sizeof (struct MeshTunnelReliability));
+    t->fwd_rel->t = t;
+    t->fwd_rel->expected_delay = MESH_RETRANSMIT_TIME;
+  }
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "CREATED TUNNEL %s[%x]:%u (%x)\n",
               GNUNET_i2s (&my_full_id), t->id.tid, t->port, t->local_tid);
 
@@ -4255,7 +4659,7 @@ handle_local_tunnel_destroy (void *cls, struct GNUNET_SERVER_Client *client,
   {
     t->client = NULL;
   }
-  else if (c == t->owner)
+  if (c == t->owner)
   {
     peer_info_remove_tunnel (peer_get_short (t->dest), t);
     t->owner = NULL;
@@ -4270,233 +4674,26 @@ handle_local_tunnel_destroy (void *cls, struct GNUNET_SERVER_Client *client,
 
 
 /**
- * Handler for requests of seeting tunnel's buffering policy.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_local_tunnel_buffer (void *cls, struct GNUNET_SERVER_Client *client,
-                            const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_MESH_TunnelMessage *tunnel_msg;
-  struct MeshClient *c;
-  struct MeshTunnel *t;
-  MESH_TunnelNumber tid;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got a BUFFER request from client!\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  tunnel_msg = (struct GNUNET_MESH_TunnelMessage *) message;
-
-  /* Retrieve tunnel */
-  tid = ntohl (tunnel_msg->tunnel_id);
-  t = tunnel_get_by_local_id(c, tid);
-  if (NULL == t)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  tunnel %X not found\n", tid);
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  switch (ntohs(message->type))
-  {
-      case GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_BUFFER:
-          t->nobuffer = GNUNET_NO;
-          break;
-      case GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_NOBUFFER:
-          t->nobuffer = GNUNET_YES;
-          break;
-      default:
-          GNUNET_break (0);
-  }
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for requests of seeting tunnel's reliability policy.
- *
- * @param cls Closure (unused).
- * @param client Identification of the client.
- * @param message The actual message.
- */
-static void
-handle_local_tunnel_reliability (void *cls, struct GNUNET_SERVER_Client *client,
-                                 const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_MESH_TunnelMessage *tunnel_msg;
-  struct MeshClient *c;
-  struct MeshTunnel *t;
-  MESH_TunnelNumber tid;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got a BUFFER request from client!\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  tunnel_msg = (struct GNUNET_MESH_TunnelMessage *) message;
-
-  /* Retrieve tunnel */
-  tid = ntohl (tunnel_msg->tunnel_id);
-  t = tunnel_get_by_local_id(c, tid);
-  if (NULL == t)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "  tunnel %X not found\n", tid);
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  switch (ntohs(message->type))
-  {
-      case GNUNET_MESSAGE_TYPE_MESH_LOCAL_RELIABLE:
-          t->reliable = GNUNET_YES;
-          break;
-      case GNUNET_MESSAGE_TYPE_MESH_LOCAL_UNRELIABLE:
-          t->reliable = GNUNET_NO;
-          break;
-      default:
-          GNUNET_break (0);
-  }
-
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-}
-
-
-/**
- * Handler for client traffic directed to one peer
+ * Handler for client traffic
  *
  * @param cls closure
  * @param client identification of the client
  * @param message the actual message
  */
 static void
-handle_local_unicast (void *cls, struct GNUNET_SERVER_Client *client,
-                      const struct GNUNET_MessageHeader *message)
+handle_local_data (void *cls, struct GNUNET_SERVER_Client *client,
+                   const struct GNUNET_MessageHeader *message)
 {
+  struct GNUNET_MESH_LocalData *data_msg;
   struct MeshClient *c;
   struct MeshTunnel *t;
-  struct GNUNET_MESH_Data *data_msg;
-  MESH_TunnelNumber tid;
-  size_t size;
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got a unicast request from a client!\n");
-
-  /* Sanity check for client registration */
-  if (NULL == (c = client_get (client)))
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
-
-  data_msg = (struct GNUNET_MESH_Data *) message;
-
-  /* Sanity check for message size */
-  size = ntohs (message->size);
-  if (sizeof (struct GNUNET_MESH_Data) +
-      sizeof (struct GNUNET_MessageHeader) > size)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Tunnel exists? */
-  tid = ntohl (data_msg->tid);
-  t = tunnel_get_by_local_id (c, tid);
-  if (NULL == t)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /*  Is it a local tunnel? Then, does client own the tunnel? */
-  if (t->owner->handle != client)
-  {
-    GNUNET_break (0);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* PID should be as expected: client<->service communication */
-  if (ntohl (data_msg->pid) != t->prev_fc.last_pid_recv + 1)
-  {
-    GNUNET_break (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              "Unicast PID, expected %u, got %u\n",
-              t->prev_fc.last_pid_recv + 1, ntohl (data_msg->pid));
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
-  /* Ok, everything is correct, send the message
-   * (pretend we got it from a mesh peer)
-   */
-  {
-    /* Work around const limitation */
-    char buf[ntohs (message->size)] GNUNET_ALIGN;
-    struct GNUNET_MESH_Data *copy;
-
-    copy = (struct GNUNET_MESH_Data *) buf;
-    memcpy (buf, data_msg, size);
-    copy->oid = my_full_id;
-    copy->tid = htonl (t->id.tid);
-    copy->ttl = htonl (default_ttl);
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                "  calling generic handler...\n");
-        handle_mesh_unicast (NULL, &my_full_id, &copy->header);
-  }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "receive done OK\n");
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
-
-  return;
-}
-
-
-/**
- * Handler for client traffic directed to the origin
- *
- * @param cls closure
- * @param client identification of the client
- * @param message the actual message
- */
-static void
-handle_local_to_origin (void *cls, struct GNUNET_SERVER_Client *client,
-                        const struct GNUNET_MessageHeader *message)
-{
-  struct GNUNET_MESH_Data *data_msg;
   struct MeshFlowControl *fc;
-  struct MeshClient *c;
-  struct MeshTunnel *t;
   MESH_TunnelNumber tid;
   size_t size;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Got a ToOrigin request from a client!\n");
+              "Got data from a client!\n");
+
   /* Sanity check for client registration */
   if (NULL == (c = client_get (client)))
   {
@@ -4506,12 +4703,11 @@ handle_local_to_origin (void *cls, struct GNUNET_SERVER_Client *client,
   }
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  by client %u\n", c->id);
 
-  data_msg = (struct GNUNET_MESH_Data *) message;
+  data_msg = (struct GNUNET_MESH_LocalData *) message;
 
   /* Sanity check for message size */
-  size = ntohs (message->size);
-  if (sizeof (struct GNUNET_MESH_Data) +
-      sizeof (struct GNUNET_MessageHeader) > size)
+  size = ntohs (message->size) - sizeof (struct GNUNET_MESH_LocalData);
+  if (size < sizeof (struct GNUNET_MessageHeader))
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
@@ -4520,8 +4716,7 @@ handle_local_to_origin (void *cls, struct GNUNET_SERVER_Client *client,
 
   /* Tunnel exists? */
   tid = ntohl (data_msg->tid);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  on tunnel %X\n", tid);
-  if (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV)
+  if (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_CLI)
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
@@ -4530,52 +4725,85 @@ handle_local_to_origin (void *cls, struct GNUNET_SERVER_Client *client,
   t = tunnel_get_by_local_id (c, tid);
   if (NULL == t)
   {
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Tunnel %X unknown.\n", tid);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "  for client %u.\n", c->id);
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
 
-  /*  It should be sent by someone who has this as incoming tunnel. */
-  if (t->client != c)
+  /* Is the client in the tunnel? */
+  if ( !( (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV &&
+           t->owner &&
+           t->owner->handle == client)
+         ||
+          (tid >= GNUNET_MESH_LOCAL_TUNNEL_ID_SERV &&
+           t->client && 
+           t->client->handle == client) ) )
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
 
-  /* PID should be as expected */
-  fc = &t->next_fc;
-  if (ntohl (data_msg->pid) != fc->last_pid_recv + 1)
-  {
-    GNUNET_break (0);
-    GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                "To Origin PID, expected %u, got %u\n",
-                fc->last_pid_recv + 1,
-                ntohl (data_msg->pid));
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-
   /* Ok, everything is correct, send the message
    * (pretend we got it from a mesh peer)
    */
   {
-    char buf[ntohs (message->size)] GNUNET_ALIGN;
-    struct GNUNET_MESH_Data *copy;
-
-    /* Work around 'const' limitation */
-    memcpy (buf, data_msg, size);
-    copy = (struct GNUNET_MESH_Data *) buf;
-    GNUNET_PEER_resolve (t->id.oid, &copy->oid);
-    copy->tid = htonl (t->id.tid);
-    copy->ttl = htonl (default_ttl);
+    struct GNUNET_MESH_Data *payload;
+    char cbuf[sizeof(struct GNUNET_MESH_Data) + size];
 
+    fc = tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV ? &t->prev_fc : &t->next_fc;
+    if (GNUNET_YES == t->reliable)
+    {
+      struct MeshTunnelReliability *rel;
+      struct MeshReliableMessage *copy;
+
+      rel = (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV) ? t->fwd_rel : t->bck_rel;
+      copy = GNUNET_malloc (sizeof (struct MeshReliableMessage)
+                            + sizeof(struct GNUNET_MESH_Data)
+                            + size);
+      copy->mid = rel->mid_sent++;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "!!! DATA %llu\n", copy->mid);
+      copy->timestamp = GNUNET_TIME_absolute_get ();
+      copy->rel = rel;
+      rel->n_sent++;
+      GNUNET_CONTAINER_DLL_insert_tail (rel->head_sent, rel->tail_sent, copy);
+      if (GNUNET_SCHEDULER_NO_TASK == rel->retry_task)
+      {
+        rel->retry_timer =
+            GNUNET_TIME_relative_multiply (rel->expected_delay,
+                                           MESH_RETRANSMIT_MARGIN);
+        rel->retry_task =
+            GNUNET_SCHEDULER_add_delayed (rel->retry_timer,
+                                          &tunnel_retransmit_message,
+                                          rel);
+      }
+      payload = (struct GNUNET_MESH_Data *) &copy[1];
+      payload->mid = GNUNET_htonll (copy->mid);
+    }
+    else
+    {
+      payload = (struct GNUNET_MESH_Data *) cbuf;
+      payload->mid = 0;
+      // FIXME FIXME FIXME FIXME FIXME FIXME FIXME FIXME FIXME FIXME 
+      // use different struct for unreliable traffic, save 8 bytes
+    }
+    memcpy (&payload[1], &data_msg[1], size);
+    payload->header.size = htons (sizeof (struct GNUNET_MESH_Data) + size);
+    payload->header.type = htons (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV ?
+                                  GNUNET_MESSAGE_TYPE_MESH_UNICAST :
+                                  GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN);
+    GNUNET_PEER_resolve(t->id.oid, &payload->oid);;
+    payload->tid = htonl (t->id.tid);
+    payload->ttl = htonl (default_ttl);
+    payload->pid = htonl (fc->last_pid_recv + 1);
     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                 "  calling generic handler...\n");
-        handle_mesh_to_orig (NULL, &my_full_id, &copy->header);
+    if (tid < GNUNET_MESH_LOCAL_TUNNEL_ID_SERV)
+      handle_mesh_unicast (NULL, &my_full_id, &payload->header);
+    else
+      handle_mesh_to_orig (NULL, &my_full_id, &payload->header);
   }
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "receive done OK\n");
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 
   return;
@@ -4597,7 +4825,6 @@ handle_local_ack (void *cls, struct GNUNET_SERVER_Client *client,
   struct MeshTunnel *t;
   struct MeshClient *c;
   MESH_TunnelNumber tid;
-  uint32_t ack;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Got a local ACK\n");
   /* Sanity check for client registration */
@@ -4624,20 +4851,17 @@ handle_local_ack (void *cls, struct GNUNET_SERVER_Client *client,
     return;
   }
 
-  ack = ntohl (msg->ack);
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "  ack %u\n", ack);
-
   /* Does client own tunnel? I.E: Is this an ACK for BCK traffic? */
   if (t->owner == c)
   {
     /* The client owns the tunnel, ACK is for data to_origin, send BCK ACK. */
-    t->prev_fc.last_ack_recv = ack;
+    t->prev_fc.last_ack_recv++;
     tunnel_send_bck_ack (t, GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK);
   }
   else
   {
     /* The client doesn't own the tunnel, this ACK is for FWD traffic. */
-    t->next_fc.last_ack_recv = ack;
+    t->next_fc.last_ack_recv++;
     tunnel_send_fwd_ack (t, GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK);
   }
 
@@ -4789,22 +5013,8 @@ static struct GNUNET_SERVER_MessageHandler client_handlers[] = {
   {&handle_local_tunnel_destroy, NULL,
    GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_DESTROY,
    sizeof (struct GNUNET_MESH_TunnelMessage)},
-  {&handle_local_tunnel_buffer, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_BUFFER,
-   sizeof (struct GNUNET_MESH_TunnelMessage)},
-  {&handle_local_tunnel_buffer, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_NOBUFFER,
-   sizeof (struct GNUNET_MESH_TunnelMessage)},
-  {&handle_local_tunnel_reliability, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_BUFFER,
-   sizeof (struct GNUNET_MESH_TunnelMessage)},
-  {&handle_local_tunnel_reliability, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_LOCAL_TUNNEL_NOBUFFER,
-   sizeof (struct GNUNET_MESH_TunnelMessage)},
-  {&handle_local_unicast, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_UNICAST, 0},
-  {&handle_local_to_origin, NULL,
-   GNUNET_MESSAGE_TYPE_MESH_TO_ORIGIN, 0},
+  {&handle_local_data, NULL,
+   GNUNET_MESSAGE_TYPE_MESH_LOCAL_DATA, 0},
   {&handle_local_ack, NULL,
    GNUNET_MESSAGE_TYPE_MESH_LOCAL_ACK,
    sizeof (struct GNUNET_MESH_LocalAck)},
@@ -5028,11 +5238,6 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_CORE_disconnect (core_handle);
     core_handle = NULL;
   }
-  if (NULL != keygen)
-  {
-    GNUNET_CRYPTO_ecc_key_create_stop (keygen);
-    keygen = NULL;
-  }
   GNUNET_CONTAINER_multihashmap_iterate (tunnels, &shutdown_tunnel, NULL);
   GNUNET_CONTAINER_multihashmap_iterate (peers, &shutdown_peer, NULL);
   if (dht_handle != NULL)
@@ -5054,64 +5259,6 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 }
 
 
-/**
- * Callback for hostkey read/generation.
- *
- * @param cls Closure (Configuration handle).
- * @param pk The ECC private key.
- * @param emsg Error message, if any.
- */
-static void
-key_generation_cb (void *cls,
-                   struct GNUNET_CRYPTO_EccPrivateKey *pk,
-                   const char *emsg)
-{
-  const struct GNUNET_CONFIGURATION_Handle *c = cls;
-
-  keygen = NULL;
-  if (NULL == pk)
-  {
-    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                _("Could not access hostkey: %s. Exiting.\n"),
-                emsg);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-  my_private_key = pk;
-  GNUNET_CRYPTO_ecc_key_get_public (my_private_key, &my_public_key);
-  GNUNET_CRYPTO_hash (&my_public_key, sizeof (my_public_key),
-                      &my_full_id.hashPubKey);
-  myid = GNUNET_PEER_intern (&my_full_id);
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-              "Mesh for peer [%s] starting\n",
-              GNUNET_i2s(&my_full_id));
-
-  core_handle = GNUNET_CORE_connect (c, /* Main configuration */
-                                     NULL,      /* Closure passed to MESH functions */
-                                     &core_init,        /* Call core_init once connected */
-                                     &core_connect,     /* Handle connects */
-                                     &core_disconnect,  /* remove peers on disconnects */
-                                     NULL,      /* Don't notify about all incoming messages */
-                                     GNUNET_NO, /* For header only in notification */
-                                     NULL,      /* Don't notify about all outbound messages */
-                                     GNUNET_NO, /* For header-only out notification */
-                                     core_handlers);    /* Register these handlers */
-  if (core_handle == NULL)
-  {
-    GNUNET_break (0);
-    GNUNET_SCHEDULER_shutdown ();
-    return;
-  }
-
-  next_tid = 0;
-  next_local_tid = GNUNET_MESH_LOCAL_TUNNEL_ID_SERV;
-
-  announce_id_task = GNUNET_SCHEDULER_add_now (&announce_id, cls);
-
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Mesh service running\n");
-}
-
-
 /**
  * Process mesh requests.
  *
@@ -5124,6 +5271,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
      const struct GNUNET_CONFIGURATION_Handle *c)
 {
   char *keyfile;
+  struct GNUNET_CRYPTO_EccPrivateKey *pk;
 
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "starting to run\n");
   server_handle = server;
@@ -5223,8 +5371,8 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   }
 
   if (GNUNET_OK !=
-    GNUNET_CONFIGURATION_get_value_number (c, "MESH", "DROP_PERCENT",
-                                           &drop_percent))
+      GNUNET_CONFIGURATION_get_value_number (c, "MESH", "DROP_PERCENT",
+                                             &drop_percent))
   {
     drop_percent = 0;
   }
@@ -5248,9 +5396,9 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   }
 
   tunnels = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
-  incoming_tunnels = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
+  incoming_tunnels = GNUNET_CONTAINER_multihashmap32_create (32);
   peers = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
-  ports = GNUNET_CONTAINER_multihashmap_create (32, GNUNET_NO);
+  ports = GNUNET_CONTAINER_multihashmap32_create (32);
 
   dht_handle = GNUNET_DHT_connect (c, 64);
   if (NULL == dht_handle)
@@ -5262,10 +5410,38 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   /* Scheduled the task to clean up when shutdown is called */
   GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task,
                                 NULL);
-  keygen = GNUNET_CRYPTO_ecc_key_create_start (keyfile,
-                                               &key_generation_cb,
-                                               (void *) c);
+  pk = GNUNET_CRYPTO_ecc_key_create_from_file (keyfile);
   GNUNET_free (keyfile);
+  GNUNET_assert (NULL != pk);
+  my_private_key = pk;
+  GNUNET_CRYPTO_ecc_key_get_public (my_private_key, &my_public_key);
+  GNUNET_CRYPTO_hash (&my_public_key, sizeof (my_public_key),
+                      &my_full_id.hashPubKey);
+  myid = GNUNET_PEER_intern (&my_full_id);
+  GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+              "Mesh for peer [%s] starting\n",
+              GNUNET_i2s(&my_full_id));
+
+  core_handle = GNUNET_CORE_connect (c, /* Main configuration */
+                                     NULL,      /* Closure passed to MESH functions */
+                                     &core_init,        /* Call core_init once connected */
+                                     &core_connect,     /* Handle connects */
+                                     &core_disconnect,  /* remove peers on disconnects */
+                                     NULL,      /* Don't notify about all incoming messages */
+                                     GNUNET_NO, /* For header only in notification */
+                                     NULL,      /* Don't notify about all outbound messages */
+                                     GNUNET_NO, /* For header-only out notification */
+                                     core_handlers);    /* Register these handlers */
+  if (NULL == core_handle)
+  {
+    GNUNET_break (0);
+    GNUNET_SCHEDULER_shutdown ();
+    return;
+  }
+  next_tid = 0;
+  next_local_tid = GNUNET_MESH_LOCAL_TUNNEL_ID_SERV;
+  announce_id_task = GNUNET_SCHEDULER_add_now (&announce_id, cls);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Mesh service running\n");
 }