fix client-client loopback flow control
[oweals/gnunet.git] / src / cadet / cadet_api.c
index 1d4530b6f867349aee794c1b559f75eb22748a74..89d9daedab78798a9a2c8fa53c6babf44b8d3e83 100644 (file)
@@ -1,6 +1,6 @@
 /*
      This file is part of GNUnet.
-     Copyright (C) 2011 GNUnet e.V.
+     Copyright (C) 2011, 2017 GNUnet e.V.
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
@@ -19,7 +19,7 @@
 */
 /**
  * @file cadet/cadet_api.c
- * @brief cadet api: client implementation of new cadet service
+ * @brief cadet api: client implementation of cadet service
  * @author Bartlomiej Polot
  */
 
  */
 struct GNUNET_CADET_TransmitHandle
 {
-    /**
-     * Double Linked list
-     */
+  /**
+   * Double Linked list
+   */
   struct GNUNET_CADET_TransmitHandle *next;
 
-    /**
-     * Double Linked list
-     */
+  /**
+   * Double Linked list
+   */
   struct GNUNET_CADET_TransmitHandle *prev;
 
-    /**
-     * Channel this message is sent on / for (may be NULL for control messages).
-     */
+  /**
+   * Channel this message is sent on / for (may be NULL for control messages).
+   */
   struct GNUNET_CADET_Channel *channel;
 
-    /**
-     * Request data task.
-     */
+  /**
+   * Request data task.
+   */
   struct GNUNET_SCHEDULER_Task *request_data_task;
 
-    /**
-     * Callback to obtain the message to transmit, or NULL if we
-     * got the message in 'data'.  Notice that messages built
-     * by 'notify' need to be encapsulated with information about
-     * the 'target'.
-     */
+  /**
+   * Callback to obtain the message to transmit, or NULL if we
+   * got the message in 'data'.  Notice that messages built
+   * by 'notify' need to be encapsulated with information about
+   * the 'target'.
+   */
   GNUNET_CONNECTION_TransmitReadyNotify notify;
 
-    /**
-     * Closure for 'notify'
-     */
+  /**
+   * Closure for 'notify'
+   */
   void *notify_cls;
 
-    /**
-     * Size of the payload.
-     */
+  /**
+   * Size of the payload.
+   */
   size_t size;
 };
 
-union CadetInfoCB {
+
+union CadetInfoCB
+{
 
   /**
    * Channel callback.
@@ -114,14 +116,14 @@ union CadetInfoCB {
  */
 struct GNUNET_CADET_Handle
 {
-    /**
-     * Message queue (if available).
-     */
+  /**
+   * Message queue (if available).
+   */
   struct GNUNET_MQ_Handle *mq;
 
-    /**
-     * Set of handlers used for processing incoming messages in the channels
-     */
+  /**
+   * Set of handlers used for processing incoming messages in the channels
+   */
   const struct GNUNET_CADET_MessageHandler *message_handlers;
 
   /**
@@ -134,40 +136,40 @@ struct GNUNET_CADET_Handle
    */
   struct GNUNET_CONTAINER_MultiHashMap *ports;
 
-    /**
-     * Double linked list of the channels this client is connected to, head.
-     */
+  /**
+   * Double linked list of the channels this client is connected to, head.
+   */
   struct GNUNET_CADET_Channel *channels_head;
 
-    /**
-     * Double linked list of the channels this client is connected to, tail.
-     */
+  /**
+   * Double linked list of the channels this client is connected to, tail.
+   */
   struct GNUNET_CADET_Channel *channels_tail;
 
-    /**
-     * Callback for inbound channel disconnection
-     */
+  /**
+   * Callback for inbound channel disconnection
+   */
   GNUNET_CADET_ChannelEndHandler *cleaner;
 
-    /**
-     * Closure for all the handlers given by the client
-     */
+  /**
+   * Closure for all the handlers given by the client
+   */
   void *cls;
 
-    /**
-     * Messages to send to the service, head.
-     */
+  /**
+   * Messages to send to the service, head.
+   */
   struct GNUNET_CADET_TransmitHandle *th_head;
 
-    /**
-     * Messages to send to the service, tail.
-     */
+  /**
+   * Messages to send to the service, tail.
+   */
   struct GNUNET_CADET_TransmitHandle *th_tail;
 
-    /**
-     * chid of the next channel to create (to avoid reusing IDs often)
-     */
-  CADET_ChannelNumber next_chid;
+  /**
+   * child of the next channel to create (to avoid reusing IDs often)
+   */
+  struct GNUNET_CADET_ClientChannelNumber next_ccn;
 
   /**
    * Configuration given by the client, in case of reconnection
@@ -201,9 +203,9 @@ struct GNUNET_CADET_Handle
  */
 struct GNUNET_CADET_Peer
 {
-    /**
-     * ID of the peer in short form
-     */
+  /**
+   * ID of the peer in short form
+   */
   GNUNET_PEER_Id id;
 
   /**
@@ -218,34 +220,34 @@ struct GNUNET_CADET_Peer
  */
 struct GNUNET_CADET_Channel
 {
-    /**
-     * DLL next
-     */
+  /**
+   * DLL next
+   */
   struct GNUNET_CADET_Channel *next;
 
-    /**
-     * DLL prev
-     */
+  /**
+   * DLL prev
+   */
   struct GNUNET_CADET_Channel *prev;
 
-    /**
-     * Handle to the cadet this channel belongs to
-     */
+  /**
+   * Handle to the cadet this channel belongs to
+   */
   struct GNUNET_CADET_Handle *cadet;
 
-    /**
-     * Local ID of the channel
-     */
-  CADET_ChannelNumber chid;
+  /**
+   * Local ID of the channel
+   */
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
-    /**
-     * Channel's port, if any.
-     */
+  /**
+   * Channel's port, if any.
+   */
   struct GNUNET_CADET_Port *port;
 
-    /**
-     * Other end of the channel.
-     */
+  /**
+   * Other end of the channel.
+   */
   GNUNET_PEER_Id peer;
 
   /**
@@ -253,46 +255,42 @@ struct GNUNET_CADET_Channel
    */
   void *ctx;
 
-    /**
-     * Size of packet queued in this channel
-     */
-  unsigned int packet_size;
-
-    /**
-     * Channel options: reliability, etc.
-     */
+  /**
+   * Channel options: reliability, etc.
+   */
   enum GNUNET_CADET_ChannelOption options;
 
-    /**
-     * Are we allowed to send to the service?
-     */
-  int allow_send;
+  /**
+   * Are we allowed to send to the service?
+   */
+  unsigned int allow_send;
 
 };
 
+
 /**
  * Opaque handle to a port.
  */
 struct GNUNET_CADET_Port
 {
-    /**
-     * Handle to the CADET session this port belongs to.
-     */
+  /**
+   * Handle to the CADET session this port belongs to.
+   */
   struct GNUNET_CADET_Handle *cadet;
 
-    /**
-     * Port ID.
-     */
+  /**
+   * Port ID.
+   */
   struct GNUNET_HashCode *hash;
 
-    /**
-     * Callback handler for incoming channels on this port.
-     */
+  /**
+   * Callback handler for incoming channels on this port.
+   */
   GNUNET_CADET_InboundChannelNotificationHandler *handler;
 
-    /**
-     * Closure for @a handler.
-     */
+  /**
+   * Closure for @a handler.
+   */
   void *cls;
 };
 
@@ -356,22 +354,20 @@ find_port (const struct GNUNET_CADET_Handle *h,
 
 /**
  * Get the channel handler for the channel specified by id from the given handle
+ *
  * @param h Cadet handle
- * @param chid ID of the wanted channel
+ * @param ccn ID of the wanted channel
  * @return handle to the required channel or NULL if not found
  */
 static struct GNUNET_CADET_Channel *
-retrieve_channel (struct GNUNET_CADET_Handle *h, CADET_ChannelNumber chid)
+retrieve_channel (struct GNUNET_CADET_Handle *h,
+                  struct GNUNET_CADET_ClientChannelNumber ccn)
 {
   struct GNUNET_CADET_Channel *ch;
 
-  ch = h->channels_head;
-  while (ch != NULL)
-  {
-    if (ch->chid == chid)
+  for (ch = h->channels_head; NULL != ch; ch = ch->next)
+    if (ch->ccn.channel_of_client == ccn.channel_of_client)
       return ch;
-    ch = ch->next;
-  }
   return NULL;
 }
 
@@ -380,33 +376,38 @@ retrieve_channel (struct GNUNET_CADET_Handle *h, CADET_ChannelNumber chid)
  * Create a new channel and insert it in the channel list of the cadet handle
  *
  * @param h Cadet handle
- * @param chid Desired chid of the channel, 0 to assign one automatically.
+ * @param ccn Desired ccn of the channel, 0 to assign one automatically.
  *
  * @return Handle to the created channel.
  */
 static struct GNUNET_CADET_Channel *
-create_channel (struct GNUNET_CADET_Handle *h, CADET_ChannelNumber chid)
+create_channel (struct GNUNET_CADET_Handle *h,
+                struct GNUNET_CADET_ClientChannelNumber ccn)
 {
   struct GNUNET_CADET_Channel *ch;
 
   ch = GNUNET_new (struct GNUNET_CADET_Channel);
-  GNUNET_CONTAINER_DLL_insert (h->channels_head, h->channels_tail, ch);
+  GNUNET_CONTAINER_DLL_insert (h->channels_head,
+                               h->channels_tail,
+                               ch);
   ch->cadet = h;
-  if (0 == chid)
+  if (0 == ccn.channel_of_client)
   {
-    ch->chid = h->next_chid;
-    while (NULL != retrieve_channel (h, h->next_chid))
+    ch->ccn = h->next_ccn;
+    while (NULL != retrieve_channel (h,
+                                     h->next_ccn))
     {
-      h->next_chid++;
-      h->next_chid &= ~GNUNET_CADET_LOCAL_CHANNEL_ID_SERV;
-      h->next_chid |= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
+      h->next_ccn.channel_of_client
+        = htonl (1 + ntohl (h->next_ccn.channel_of_client));
+      if (0 == ntohl (h->next_ccn.channel_of_client))
+        h->next_ccn.channel_of_client
+          = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
     }
   }
   else
   {
-    ch->chid = chid;
+    ch->ccn = ccn;
   }
-  ch->allow_send = GNUNET_NO;
   return ch;
 }
 
@@ -425,27 +426,35 @@ create_channel (struct GNUNET_CADET_Handle *h, CADET_ChannelNumber chid)
  */
 // FIXME: simplify: call_cleaner is always #GNUNET_YES!!!
 static void
-destroy_channel (struct GNUNET_CADET_Channel *ch, int call_cleaner)
+destroy_channel (struct GNUNET_CADET_Channel *ch,
+                 int call_cleaner)
 {
   struct GNUNET_CADET_Handle *h;
   struct GNUNET_CADET_TransmitHandle *th;
   struct GNUNET_CADET_TransmitHandle *next;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " destroy_channel %X\n", ch->chid);
-
   if (NULL == ch)
   {
     GNUNET_break (0);
     return;
   }
   h = ch->cadet;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       " destroy_channel %X of %p\n",
+       ch->ccn,
+       h);
 
-  GNUNET_CONTAINER_DLL_remove (h->channels_head, h->channels_tail, ch);
+  GNUNET_CONTAINER_DLL_remove (h->channels_head,
+                               h->channels_tail,
+                               ch);
 
   /* signal channel destruction */
-  if ( (NULL != h->cleaner) && (0 != ch->peer) && (GNUNET_YES == call_cleaner) )
+  if ( (NULL != h->cleaner) &&
+       (0 != ch->peer) &&
+       (GNUNET_YES == call_cleaner) )
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, " calling cleaner\n");
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         " calling cleaner\n");
     h->cleaner (h->cls, ch, ch->ctx);
   }
 
@@ -466,7 +475,6 @@ destroy_channel (struct GNUNET_CADET_Channel *ch, int call_cleaner)
   if (0 != ch->peer)
     GNUNET_PEER_change_rc (ch->peer, -1);
   GNUNET_free (ch);
-  return;
 }
 
 
@@ -516,13 +524,15 @@ send_ack (struct GNUNET_CADET_Channel *ch)
   struct GNUNET_CADET_LocalAck *msg;
   struct GNUNET_MQ_Envelope *env;
 
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK);
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Sending ACK on channel %X\n", ch->chid);
-  msg->channel_id = htonl (ch->chid);
-  GNUNET_MQ_send (ch->cadet->mq, env);
-
-  return;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Sending ACK on channel %X\n",
+       ch->ccn.channel_of_client);
+  msg->ccn = ch->ccn;
+  GNUNET_MQ_send (ch->cadet->mq,
+                  env);
 }
 
 
@@ -547,21 +557,28 @@ request_data (void *cls)
   struct GNUNET_MQ_Envelope *env;
   size_t osize;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Requesting Data: %u bytes\n", th->size);
-
-  GNUNET_assert (GNUNET_YES == th->channel->allow_send);
-  th->channel->allow_send = GNUNET_NO;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Requesting Data: %u bytes (allow send is %u)\n",
+       th->size,
+       th->channel->allow_send);
+
+  GNUNET_assert (0 < th->channel->allow_send);
+  th->channel->allow_send--;
+  /* NOTE: we may be allowed to send another packet immediately,
+     albeit the current logic waits for the ACK. */
   th->request_data_task = NULL;
-  th->channel->packet_size = 0;
   remove_from_queue (th);
 
-  env = GNUNET_MQ_msg_extra (msg, th->size,
+  env = GNUNET_MQ_msg_extra (msg,
+                             th->size,
                              GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA);
-  msg->id = htonl (th->channel->chid);
-  osize = th->notify (th->notify_cls, th->size, &msg[1]);
+  msg->ccn = th->channel->ccn;
+  osize = th->notify (th->notify_cls,
+                      th->size,
+                      &msg[1]);
   GNUNET_assert (osize == th->size);
-  GNUNET_MQ_send (th->channel->cadet->mq, env);
-
+  GNUNET_MQ_send (th->channel->cadet->mq,
+                  env);
   GNUNET_free (th);
 }
 
@@ -574,53 +591,59 @@ request_data (void *cls)
  */
 static void
 handle_channel_created (void *cls,
-                        const struct GNUNET_CADET_ChannelCreateMessage *msg)
+                        const struct GNUNET_CADET_LocalChannelCreateMessage *msg)
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
   struct GNUNET_CADET_Port *port;
   const struct GNUNET_HashCode *port_number;
-  CADET_ChannelNumber chid;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
-  chid = ntohl (msg->channel_id);
+  ccn = msg->ccn;
   port_number = &msg->port;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Creating incoming channel %X [%s]\n",
-       chid, GNUNET_h2s (port_number));
-  if (chid < GNUNET_CADET_LOCAL_CHANNEL_ID_SERV)
+  if (ntohl (ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
   {
     GNUNET_break (0);
     return;
   }
   port = find_port (h, port_number);
-  if (NULL != port)
-  {
-    void *ctx;
-
-    ch = create_channel (h, chid);
-    ch->allow_send = GNUNET_NO;
-    ch->peer = GNUNET_PEER_intern (&msg->peer);
-    ch->cadet = h;
-    ch->chid = chid;
-    ch->port = port;
-    ch->options = ntohl (msg->opt);
-
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "  created channel %p\n", ch);
-    ctx = port->handler (port->cls, ch, &msg->peer, port->hash, ch->options);
-    if (NULL != ctx)
-      ch->ctx = ctx;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "User notified\n");
-  }
-  else
+  if (NULL == port)
   {
-    struct GNUNET_CADET_ChannelDestroyMessage *d_msg;
+    struct GNUNET_CADET_LocalChannelDestroyMessage *d_msg;
     struct GNUNET_MQ_Envelope *env;
 
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "No handler for incoming channels\n");
-    env = GNUNET_MQ_msg (d_msg, GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-    d_msg->channel_id = msg->channel_id;
-    GNUNET_MQ_send (h->mq, env);
+    GNUNET_break (0);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "No handler for incoming channel %X [%s]\n",
+         ntohl (ccn.channel_of_client),
+         GNUNET_h2s (port_number));
+    /* FIXME: should disconnect instead, this is a serious error! */
+    env = GNUNET_MQ_msg (d_msg,
+                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+    d_msg->ccn = msg->ccn;
+    GNUNET_MQ_send (h->mq,
+                    env);
+    return;
   }
-  return;
+
+  ch = create_channel (h,
+                       ccn);
+  ch->peer = GNUNET_PEER_intern (&msg->peer);
+  ch->cadet = h;
+  ch->ccn = ccn;
+  ch->port = port;
+  ch->options = ntohl (msg->opt);
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating incoming channel %X [%s] %p\n",
+       ntohl (ccn.channel_of_client),
+       GNUNET_h2s (port_number),
+       ch);
+  ch->ctx = port->handler (port->cls,
+                           ch,
+                           &msg->peer,
+                           port->hash,
+                           ch->options);
 }
 
 
@@ -632,22 +655,28 @@ handle_channel_created (void *cls,
  */
 static void
 handle_channel_destroy (void *cls,
-                        const struct GNUNET_CADET_ChannelDestroyMessage *msg)
+                        const struct GNUNET_CADET_LocalChannelDestroyMessage *msg)
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  CADET_ChannelNumber chid;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
-  chid = ntohl (msg->channel_id);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Channel %X Destroy from service\n", chid);
-  ch = retrieve_channel (h, chid);
+  ccn = msg->ccn;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Channel %X Destroy from service\n",
+       ntohl (ccn.channel_of_client));
+  ch = retrieve_channel (h,
+                         ccn);
 
   if (NULL == ch)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "channel %X unknown\n", chid);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "channel %X unknown\n",
+         ntohl (ccn.channel_of_client));
     return;
   }
-  destroy_channel (ch, GNUNET_YES);
+  destroy_channel (ch,
+                   GNUNET_YES);
 }
 
 
@@ -674,7 +703,8 @@ check_local_data (void *cls,
     return GNUNET_SYSERR;
   }
 
-  ch = retrieve_channel (h, ntohl (message->id));
+  ch = retrieve_channel (h,
+                         message->ccn);
   if (NULL == ch)
   {
     GNUNET_break_op (0);
@@ -702,37 +732,38 @@ handle_local_data (void *cls,
   unsigned int i;
   uint16_t type;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got a data message!\n");
-  ch = retrieve_channel (h, ntohl (message->id));
+  ch = retrieve_channel (h,
+                         message->ccn);
   GNUNET_assert (NULL != ch);
 
   payload = (struct GNUNET_MessageHeader *) &message[1];
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  %s data on channel %s [%X]\n",
-       GC_f2s (ch->chid >= GNUNET_CADET_LOCAL_CHANNEL_ID_SERV),
-       GNUNET_i2s (GNUNET_PEER_resolve2 (ch->peer)), ntohl (message->id));
-
   type = ntohs (payload->type);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  payload type %s\n", GC_m2s (type));
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got a %s data on channel %s [%X] of type %s (%u)\n",
+       GC_f2s (ntohl (ch->ccn.channel_of_client) >=
+               GNUNET_CADET_LOCAL_CHANNEL_ID_CLI),
+       GNUNET_i2s (GNUNET_PEER_resolve2 (ch->peer)),
+       ntohl (message->ccn.channel_of_client),
+       GC_m2s (type),
+       type);
+
   for (i = 0; i < h->n_handlers; i++)
   {
     handler = &h->message_handlers[i];
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "    checking handler for type %u\n",
-         handler->type);
     if (handler->type == type)
     {
       if (GNUNET_OK !=
-          handler->callback (h->cls, ch, &ch->ctx, payload))
-      {
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "callback caused disconnection\n");
-        GNUNET_CADET_channel_destroy (ch);
-        break;
-      }
-      else
+          handler->callback (h->cls,
+                             ch,
+                             &ch->ctx,
+                             payload))
       {
         LOG (GNUNET_ERROR_TYPE_DEBUG,
-             "callback completed successfully\n");
+             "callback caused disconnection\n");
+        GNUNET_CADET_channel_destroy (ch);
         break;
       }
+      break;
     }
   }
 }
@@ -751,37 +782,33 @@ handle_local_ack (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
   struct GNUNET_CADET_Channel *ch;
-  CADET_ChannelNumber chid;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
+  struct GNUNET_CADET_TransmitHandle *th;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Got an ACK!\n");
-  chid = ntohl (message->channel_id);
-  ch = retrieve_channel (h, chid);
+  ccn = message->ccn;
+  ch = retrieve_channel (h, ccn);
   if (NULL == ch)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "ACK on unknown channel %X\n", chid);
+    LOG (GNUNET_ERROR_TYPE_DEBUG,
+         "ACK on unknown channel %X\n",
+         ntohl (ccn.channel_of_client));
     return;
   }
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  on channel %X!\n", ch->chid);
-  ch->allow_send = GNUNET_YES;
-  if (0 < ch->packet_size)
+  ch->allow_send++;
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Got an ACK on channel %X, allow send now %u!\n",
+       ntohl (ch->ccn.channel_of_client),
+       ch->allow_send);
+  for (th = h->th_head; NULL != th; th = th->next)
   {
-    struct GNUNET_CADET_TransmitHandle *th;
-    struct GNUNET_CADET_TransmitHandle *next;
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "  pending data, sending %u bytes!\n",
-         ch->packet_size);
-    for (th = h->th_head; NULL != th; th = next)
+    if ( (th->channel == ch) &&
+         (NULL == th->request_data_task) )
     {
-      next = th->next;
-      if (th->channel == ch)
-      {
-        GNUNET_assert (NULL == th->request_data_task);
-        th->request_data_task = GNUNET_SCHEDULER_add_now (&request_data, th);
-        break;
-      }
+      th->request_data_task
+        = GNUNET_SCHEDULER_add_now (&request_data,
+                                    th);
+      break;
     }
-    /* Complain if we got thru all th without sending anything, ch was wrong */
-    GNUNET_break (NULL != th);
   }
 }
 
@@ -1132,9 +1159,12 @@ handle_get_tunnels (void *cls,
 {
   struct GNUNET_CADET_Handle *h = cls;
 
-  h->info_cb.tunnels_cb (h->info_cls, &msg->destination,
-                         ntohl (msg->channels), ntohl (msg->connections),
-                         ntohs (msg->estate), ntohs (msg->cstate));
+  h->info_cb.tunnels_cb (h->info_cls,
+                         &msg->destination,
+                         ntohl (msg->channels),
+                         ntohl (msg->connections),
+                         ntohs (msg->estate),
+                         ntohs (msg->cstate));
 
 }
 
@@ -1170,13 +1200,14 @@ check_get_tunnel (void *cls,
   if (esize > msize)
   {
     GNUNET_break_op (0);
-    h->info_cb.tunnel_cb (h->info_cls, NULL, 0, 0, NULL, NULL, 0, 0);
+    h->info_cb.tunnel_cb (h->info_cls,
+                          NULL, 0, 0, NULL, NULL, 0, 0);
     goto clean_cls;
   }
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
-  esize += ch_n * sizeof (CADET_ChannelNumber);
-  esize += c_n * sizeof (struct GNUNET_CADET_Hash);
+  esize += ch_n * sizeof (struct GNUNET_CADET_ChannelTunnelNumber);
+  esize += c_n * sizeof (struct GNUNET_CADET_ConnectionTunnelIdentifier);
   if (msize != esize)
   {
     GNUNET_break_op (0);
@@ -1186,7 +1217,8 @@ check_get_tunnel (void *cls,
                 (unsigned int) esize,
                 ch_n,
                 c_n);
-    h->info_cb.tunnel_cb (h->info_cls, NULL, 0, 0, NULL, NULL, 0, 0);
+    h->info_cb.tunnel_cb (h->info_cls,
+                          NULL, 0, 0, NULL, NULL, 0, 0);
     goto clean_cls;
   }
 
@@ -1212,18 +1244,23 @@ handle_get_tunnel (void *cls,
   struct GNUNET_CADET_Handle *h = cls;
   unsigned int ch_n;
   unsigned int c_n;
-  struct GNUNET_CADET_Hash *conns;
-  CADET_ChannelNumber *chns;
+  const struct GNUNET_CADET_ConnectionTunnelIdentifier *conns;
+  const struct GNUNET_CADET_ChannelTunnelNumber *chns;
 
   ch_n = ntohl (msg->channels);
   c_n = ntohl (msg->connections);
 
   /* Call Callback with tunnel info. */
-  conns = (struct GNUNET_CADET_Hash *) &msg[1];
-  chns = (CADET_ChannelNumber *) &conns[c_n];
-  h->info_cb.tunnel_cb (h->info_cls, &msg->destination,
-                        ch_n, c_n, chns, conns,
-                        ntohs (msg->estate), ntohs (msg->cstate));
+  conns = (const struct GNUNET_CADET_ConnectionTunnelIdentifier *) &msg[1];
+  chns = (const struct GNUNET_CADET_ChannelTunnelNumber *) &conns[c_n];
+  h->info_cb.tunnel_cb (h->info_cls,
+                        &msg->destination,
+                        ch_n,
+                        c_n,
+                        chns,
+                        conns,
+                        ntohs (msg->estate),
+                        ntohs (msg->cstate));
 }
 
 
@@ -1233,56 +1270,55 @@ handle_get_tunnel (void *cls,
  * original state.
  *
  * @param h handle to the cadet
- *
- * @return GNUNET_YES in case of sucess, GNUNET_NO otherwise (service down...)
+ * @return #GNUNET_YES in case of success, #GNUNET_NO otherwise (service down...)
  */
 static int
 do_reconnect (struct GNUNET_CADET_Handle *h)
 {
-  GNUNET_MQ_hd_fixed_size (channel_created,
-                           GNUNET_MESSAGE_TYPE_CADET_CHANNEL_CREATE,
-                           struct GNUNET_CADET_ChannelCreateMessage);
-  GNUNET_MQ_hd_fixed_size (channel_destroy,
-                           GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY,
-                           struct GNUNET_CADET_ChannelDestroyMessage);
-  GNUNET_MQ_hd_var_size (local_data,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
-                         struct GNUNET_CADET_LocalData);
-  GNUNET_MQ_hd_fixed_size (local_ack,
-                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
-                           struct GNUNET_CADET_LocalAck);
-  GNUNET_MQ_hd_var_size (get_peers,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
-                         struct GNUNET_CADET_LocalInfoPeer);
-  GNUNET_MQ_hd_var_size (get_peer,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
-                         struct GNUNET_CADET_LocalInfoPeer);
-  GNUNET_MQ_hd_var_size (get_tunnels,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
-                         struct GNUNET_CADET_LocalInfoTunnel);
-  GNUNET_MQ_hd_var_size (get_tunnel,
-                         GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
-                         struct GNUNET_CADET_LocalInfoTunnel);
+  struct GNUNET_MQ_MessageHandler handlers[] = {
+    GNUNET_MQ_hd_fixed_size (channel_created,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE,
+                             struct GNUNET_CADET_LocalChannelCreateMessage,
+                             h),
+    GNUNET_MQ_hd_fixed_size (channel_destroy,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY,
+                             struct GNUNET_CADET_LocalChannelDestroyMessage,
+                             h),
+    GNUNET_MQ_hd_var_size (local_data,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_DATA,
+                           struct GNUNET_CADET_LocalData,
+                           h),
+    GNUNET_MQ_hd_fixed_size (local_ack,
+                             GNUNET_MESSAGE_TYPE_CADET_LOCAL_ACK,
+                             struct GNUNET_CADET_LocalAck,
+                             h),
+    GNUNET_MQ_hd_var_size (get_peers,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEERS,
+                           struct GNUNET_CADET_LocalInfoPeer,
+                           h),
+    GNUNET_MQ_hd_var_size (get_peer,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_PEER,
+                           struct GNUNET_CADET_LocalInfoPeer,
+                           h),
+    GNUNET_MQ_hd_var_size (get_tunnels,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNELS,
+                           struct GNUNET_CADET_LocalInfoTunnel,
+                           h),
+    GNUNET_MQ_hd_var_size (get_tunnel,
+                           GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_TUNNEL,
+                           struct GNUNET_CADET_LocalInfoTunnel,
+                           h),
   // FIXME
 //   GNUNET_MQ_hd_fixed_Y       size (channel_destroyed,
-//                            GNUNET_MESSAGE_TYPE_CADET_CHANNEL_NACK,
+//                            GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN_NACK_DEPRECATED,
 //                            struct GNUNET_CADET_ChannelDestroyMessage);
-  struct GNUNET_MQ_MessageHandler handlers[] = {
-    make_channel_created_handler (h),
-    make_channel_destroy_handler (h),
-    make_local_data_handler (h),
-    make_local_ack_handler (h),
-    make_get_peers_handler (h),
-    make_get_peer_handler (h),
-    make_get_tunnels_handler (h),
-    make_get_tunnel_handler (h),
     GNUNET_MQ_handler_end ()
   };
 
   LOG (GNUNET_ERROR_TYPE_DEBUG, "Connecting to CADET\n");
 
   GNUNET_assert (NULL == h->mq);
-  h->mq = GNUNET_CLIENT_connecT (h->cfg,
+  h->mq = GNUNET_CLIENT_connect (h->cfg,
                                  "cadet",
                                  handlers,
                                  &handle_mq_error,
@@ -1330,7 +1366,7 @@ reconnect (struct GNUNET_CADET_Handle *h)
 
   LOG (GNUNET_ERROR_TYPE_DEBUG,
        "Requested RECONNECT, destroying all channels\n");
-  for (ch = h->channels_head; NULL != ch; ch = h->channels_head)
+  while (NULL != (ch = h->channels_head))
     destroy_channel (ch, GNUNET_YES);
   if (NULL == h->reconnect_task)
     h->reconnect_task = GNUNET_SCHEDULER_add_delayed (h->reconnect_time,
@@ -1343,15 +1379,17 @@ reconnect (struct GNUNET_CADET_Handle *h)
 /******************************************************************************/
 
 struct GNUNET_CADET_Handle *
-GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg, void *cls,
-                     GNUNET_CADET_ChannelEndHandler cleaner,
-                     const struct GNUNET_CADET_MessageHandler *handlers)
+GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      void *cls,
+                      GNUNET_CADET_ChannelEndHandler cleaner,
+                      const struct GNUNET_CADET_MessageHandler *handlers)
 {
   struct GNUNET_CADET_Handle *h;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GNUNET_CADET_connect()\n");
   h = GNUNET_new (struct GNUNET_CADET_Handle);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, " addr %p\n", h);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "GNUNET_CADET_connect() %p\n",
+       h);
   h->cfg = cfg;
   h->cleaner = cleaner;
   h->ports = GNUNET_CONTAINER_multihashmap_create (4, GNUNET_YES);
@@ -1364,7 +1402,7 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg, void *cls,
   }
   h->cls = cls;
   h->message_handlers = handlers;
-  h->next_chid = GNUNET_CADET_LOCAL_CHANNEL_ID_CLI;
+  h->next_ccn.channel_of_client = htonl (GNUNET_CADET_LOCAL_CHANNEL_ID_CLI);
   h->reconnect_time = GNUNET_TIME_UNIT_MILLISECONDS;
   h->reconnect_task = NULL;
 
@@ -1372,11 +1410,18 @@ GNUNET_CADET_connect (const struct GNUNET_CONFIGURATION_Handle *cfg, void *cls,
   for (h->n_handlers = 0;
        handlers && handlers[h->n_handlers].type;
        h->n_handlers++) ;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "GNUNET_CADET_connect() END\n");
   return h;
 }
 
 
+/**
+ * Disconnect from the cadet service. All channels will be destroyed. All channel
+ * disconnect callbacks will be called on any still connected peers, notifying
+ * about their disconnection. The registered inbound channel cleaner will be
+ * called should any inbound channels still exist.
+ *
+ * @param handle connection to cadet to disconnect
+ */
 void
 GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
 {
@@ -1384,21 +1429,24 @@ GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
   struct GNUNET_CADET_Channel *aux;
   struct GNUNET_CADET_TransmitHandle *th;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "CADET DISCONNECT\n");
-
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "CADET DISCONNECT\n");
   ch = handle->channels_head;
   while (NULL != ch)
   {
     aux = ch->next;
-    if (ch->chid < GNUNET_CADET_LOCAL_CHANNEL_ID_SERV)
+    if (ntohl (ch->ccn.channel_of_client) >= GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
     {
       GNUNET_break (0);
-      LOG (GNUNET_ERROR_TYPE_DEBUG, "channel %X not destroyed\n", ch->chid);
+      LOG (GNUNET_ERROR_TYPE_DEBUG,
+           "channel %X not destroyed\n",
+           ntohl (ch->ccn.channel_of_client));
     }
-    destroy_channel (ch, GNUNET_YES);
+    destroy_channel (ch,
+                     GNUNET_YES);
     ch = aux;
   }
-  while ( (th = handle->th_head) != NULL)
+  while (NULL != (th = handle->th_head))
   {
     struct GNUNET_MessageHeader *msg;
 
@@ -1409,7 +1457,7 @@ GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
     msg = (struct GNUNET_MessageHeader *) &th[1];
     switch (ntohs(msg->type))
     {
-      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_CREATE:
+      case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_OPEN:
       case GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY:
       case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_OPEN:
       case GNUNET_MESSAGE_TYPE_CADET_LOCAL_PORT_CLOSE:
@@ -1453,7 +1501,6 @@ GNUNET_CADET_disconnect (struct GNUNET_CADET_Handle *handle)
  * @param port Hash representing the port number.
  * @param new_channel Function called when an channel is received.
  * @param new_channel_cls Closure for @a new_channel.
- *
  * @return Port handle.
  */
 struct GNUNET_CADET_Port *
@@ -1521,7 +1568,6 @@ GNUNET_CADET_close_port (struct GNUNET_CADET_Port *p)
  * @param peer peer identity the channel should go to
  * @param port Port hash (port number).
  * @param options CadetOption flag field, with all desired option bits set to 1.
- *
  * @return handle to the channel
  */
 struct GNUNET_CADET_Channel *
@@ -1531,27 +1577,30 @@ GNUNET_CADET_channel_create (struct GNUNET_CADET_Handle *h,
                             const struct GNUNET_HashCode *port,
                             enum GNUNET_CADET_ChannelOption options)
 {
-  struct GNUNET_CADET_ChannelCreateMessage *msg;
+  struct GNUNET_CADET_LocalChannelCreateMessage *msg;
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_Channel *ch;
+  struct GNUNET_CADET_ClientChannelNumber ccn;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG,
-       "Creating new channel to %s:%u\n",
-       GNUNET_i2s (peer), port);
-  ch = create_channel (h, 0);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  at %p\n", ch);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "  number %X\n", ch->chid);
+  ccn.channel_of_client = htonl (0);
+  ch = create_channel (h, ccn);
   ch->ctx = channel_ctx;
   ch->peer = GNUNET_PEER_intern (peer);
 
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_CHANNEL_CREATE);
-  msg->channel_id = htonl (ch->chid);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Creating new channel to %s:%u at %p number %X\n",
+       GNUNET_i2s (peer),
+       port,
+       ch,
+       ntohl (ch->ccn.channel_of_client));
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_CREATE);
+  msg->ccn = ch->ccn;
   msg->port = *port;
   msg->peer = *peer;
   msg->opt = htonl (options);
-  ch->allow_send = GNUNET_NO;
-  GNUNET_MQ_send (h->mq, env);
-
+  GNUNET_MQ_send (h->mq,
+                  env);
   return ch;
 }
 
@@ -1560,12 +1609,13 @@ void
 GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
 {
   struct GNUNET_CADET_Handle *h;
-  struct GNUNET_CADET_ChannelDestroyMessage *msg;
+  struct GNUNET_CADET_LocalChannelDestroyMessage *msg;
   struct GNUNET_MQ_Envelope *env;
   struct GNUNET_CADET_TransmitHandle *th;
   struct GNUNET_CADET_TransmitHandle *next;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Destroying channel\n");
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Destroying channel\n");
   h = channel->cadet;
   for  (th = h->th_head; th != NULL; th = next)
   {
@@ -1582,18 +1632,24 @@ GNUNET_CADET_channel_destroy (struct GNUNET_CADET_Channel *channel)
       }
       else
       {
-        LOG (GNUNET_ERROR_TYPE_WARNING, "no meta-traffic should be queued\n");
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             "no meta-traffic should be queued\n");
       }
-      GNUNET_CONTAINER_DLL_remove (h->th_head, h->th_tail, th);
+      GNUNET_CONTAINER_DLL_remove (h->th_head,
+                                   h->th_tail,
+                                   th);
       GNUNET_CADET_notify_transmit_ready_cancel (th);
     }
   }
 
-  env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_CHANNEL_DESTROY);
-  msg->channel_id = htonl (channel->chid);
-  GNUNET_MQ_send (h->mq, env);
+  env = GNUNET_MQ_msg (msg,
+                       GNUNET_MESSAGE_TYPE_CADET_LOCAL_CHANNEL_DESTROY);
+  msg->ccn = channel->ccn;
+  GNUNET_MQ_send (h->mq,
+                  env);
 
-  destroy_channel (channel, GNUNET_YES);
+  destroy_channel (channel,
+                   GNUNET_YES);
 }
 
 
@@ -1617,7 +1673,7 @@ GNUNET_CADET_channel_get_info (struct GNUNET_CADET_Channel *channel,
   {
     case GNUNET_CADET_OPTION_NOBUFFER:
     case GNUNET_CADET_OPTION_RELIABLE:
-    case GNUNET_CADET_OPTION_OOORDER:
+    case GNUNET_CADET_OPTION_OUT_OF_ORDER:
       if (0 != (option & channel->options))
         bool_flag = GNUNET_YES;
       else
@@ -1647,18 +1703,17 @@ GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
   struct GNUNET_CADET_TransmitHandle *th;
 
   GNUNET_assert (NULL != channel);
-  GNUNET_assert (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE >= notify_size);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "CADET NOTIFY TRANSMIT READY\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    on channel %X\n", channel->chid);
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    allow_send %d\n", channel->allow_send);
-  if (channel->chid >= GNUNET_CADET_LOCAL_CHANNEL_ID_SERV)
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "    to origin\n");
-  else
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "    to destination\n");
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    payload size %u\n", notify_size);
   GNUNET_assert (NULL != notify);
-  GNUNET_assert (0 == channel->packet_size); // Only one data packet allowed
-
+  GNUNET_assert (GNUNET_CONSTANTS_MAX_CADET_MESSAGE_SIZE >= notify_size);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "CADET NOTIFY TRANSMIT READY on channel %X allow_send is %u to %s with %u bytes\n",
+       channel->ccn,
+       channel->allow_send,
+       (ntohl (channel->ccn.channel_of_client) >=
+        GNUNET_CADET_LOCAL_CHANNEL_ID_CLI)
+       ? "origin"
+       : "destination",
+       (unsigned int) notify_size);
   if (GNUNET_TIME_UNIT_FOREVER_REL.rel_value_us != maxdelay.rel_value_us)
   {
     LOG (GNUNET_ERROR_TYPE_WARNING,
@@ -1668,16 +1723,15 @@ GNUNET_CADET_notify_transmit_ready (struct GNUNET_CADET_Channel *channel,
   th = GNUNET_new (struct GNUNET_CADET_TransmitHandle);
   th->channel = channel;
   th->size = notify_size;
-  channel->packet_size = th->size;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "    total size %u\n", th->size);
   th->notify = notify;
   th->notify_cls = notify_cls;
-  if (GNUNET_YES == channel->allow_send)
-    th->request_data_task = GNUNET_SCHEDULER_add_now (&request_data, th);
+  if (0 != channel->allow_send)
+    th->request_data_task
+      = GNUNET_SCHEDULER_add_now (&request_data,
+                                  th);
   else
-    add_to_queue (channel->cadet, th);
-
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "CADET NOTIFY TRANSMIT READY END\n");
+    add_to_queue (channel->cadet,
+                  th);
   return th;
 }
 
@@ -1688,9 +1742,8 @@ GNUNET_CADET_notify_transmit_ready_cancel (struct GNUNET_CADET_TransmitHandle *t
   if (NULL != th->request_data_task)
   {
     GNUNET_SCHEDULER_cancel (th->request_data_task);
+    th->request_data_task = NULL;
   }
-  th->request_data_task = NULL;
-
   remove_from_queue (th);
   GNUNET_free (th);
 }
@@ -1946,7 +1999,7 @@ GNUNET_CADET_show_channel (struct GNUNET_CADET_Handle *h,
 
   env = GNUNET_MQ_msg (msg, GNUNET_MESSAGE_TYPE_CADET_LOCAL_INFO_CHANNEL);
   msg->peer = *initiator;
-  msg->channel_id = htonl (channel_number);
+  msg->ccn.channel_of_client = htonl (channel_number);
   GNUNET_MQ_send (h->mq, env);
 
   h->info_cb.channel_cb = callback;