(no commit message)
[oweals/gnunet.git] / src / transport / transport_api.c
index c2e2061f89139673d78325c26098ee523bd05e9a..093c00bef8731ec243e152af15052a4dcdbc4e62 100644 (file)
@@ -1,10 +1,10 @@
 /*
      This file is part of GNUnet.
-     (C) 2009 Christian Grothoff (and other contributing authors)
+     (C) 2009, 2010 Christian Grothoff (and other contributing authors)
 
      GNUnet is free software; you can redistribute it and/or modify
      it under the terms of the GNU General Public License as published
-     by the Free Software Foundation; either version 2, or (at your
+     by the Free Software Foundation; either version 3, or (at your
      option) any later version.
 
      GNUnet is distributed in the hope that it will be useful, but
  * @file transport/transport_api.c
  * @brief library to access the low-level P2P IO service
  * @author Christian Grothoff
- *
- * TODO:
- * - set_quota with low bandwidth should cause peer
- *   disconnects (currently never does that) (MINOR)
  */
 #include "platform.h"
+#include "gnunet_bandwidth_lib.h"
 #include "gnunet_client_lib.h"
+#include "gnunet_constants.h"
+#include "gnunet_container_lib.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_hello_lib.h"
 #include "gnunet_protocols.h"
  */
 #define OFFER_HELLO_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30)
 
+/**
+ * After how long do we automatically retry an unsuccessful
+ * CONNECT request?
+ */
+#define CONNECT_RETRY_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, 750)
+
 /**
  * How long should ARM wait when starting up the
  * transport service before reporting back?
  */
 #define STOP_SERVICE_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
 
+
 /**
- * Entry in linked list of all of our current neighbours.
+ * What stage are we in for transmission processing?
  */
-struct NeighbourList
-{
+enum TransmitStage
+  {
+    /**
+     * No active message.
+     */
+    TS_NEW = 0,
+
+    /**
+     * Message in local queue, not given to service.
+     */
+    TS_QUEUED = 1,
+
+    /**
+     * Message given to service, not confirmed (no SEND_OK).
+     */
+    TS_TRANSMITTED = 2,
+
+    /**
+     * One message was given to service and before it was confirmed,
+     * another one was already queued (waiting for SEND_OK to pass on
+     * to service).
+     */
+    TS_TRANSMITTED_QUEUED = 3
+  };
 
-  /**
-   * This is a linked list.
-   */
-  struct NeighbourList *next;
+
+/**
+ * Handle for a transmission-ready request.
+ */
+struct GNUNET_TRANSPORT_TransmitHandle
+{
 
   /**
-   * Active transmit handle, can be NULL.  Used to move
-   * from ready to wait list on disconnect and to block
-   * two transmissions to the same peer from being scheduled
-   * at the same time.
+   * Neighbour for this handle, NULL for control-traffic.
    */
-  struct GNUNET_TRANSPORT_TransmitHandle *transmit_handle;
-
+  struct NeighbourList *neighbour;
 
   /**
-   * Identity of this neighbour.
+   * Function to call when notify_size bytes are available
+   * for transmission.
    */
-  struct GNUNET_PeerIdentity id;
+  GNUNET_CONNECTION_TransmitReadyNotify notify;
 
   /**
-   * At what time did we reset last_sent last?
+   * Closure for notify.
    */
-  struct GNUNET_TIME_Absolute last_quota_update;
+  void *notify_cls;
 
   /**
-   * How many bytes have we sent since the "last_quota_update"
-   * timestamp?
+   * transmit_ready task Id.  The task is used to introduce the
+   * artificial delay that may be required to maintain the bandwidth
+   * limits.  Later, this will be the ID of the "transmit_timeout"
+   * task which is used to signal a timeout if the transmission could
+   * not be done in a timely fashion.
    */
-  uint64_t last_sent;
+  GNUNET_SCHEDULER_TaskIdentifier notify_delay_task;
 
   /**
-   * Global quota for outbound traffic to the neighbour in bytes/ms.
+   * Timeout for this request.
    */
-  uint32_t quota_out;
+  struct GNUNET_TIME_Absolute timeout;
 
   /**
-   * Set to GNUNET_YES if we are currently allowed to
-   * transmit a message to the transport service for this
-   * peer, GNUNET_NO otherwise.
+   * How many bytes is our notify callback waiting for?
    */
-  int transmit_ok;
+  size_t notify_size;
 
   /**
-   * Set to GNUNET_YES if we have received an ACK for the
-   * given peer.  Peers that receive our HELLO always respond
-   * with an ACK to let us know that we are successfully
-   * communicating.  Note that a PING can not be used for this
-   * since PINGs are only send if a HELLO address requires
-   * confirmation (and also, PINGs are not passed to the
-   * transport API itself).
+   * How important is this message?
    */
-  int received_ack;
+  unsigned int priority;
 
 };
 
 
 /**
- * Linked list of requests from clients for our HELLO
- * that were deferred.
+ * Handle for a control message queue entry.
  */
-struct HelloWaitList
+struct ControlMessage
 {
 
   /**
-   * This is a linked list.
+   * This is a doubly-linked list.
    */
-  struct HelloWaitList *next;
+  struct ControlMessage *next;
 
   /**
-   * Reference back to our transport handle.
+   * This is a doubly-linked list.
    */
-  struct GNUNET_TRANSPORT_Handle *handle;
+  struct ControlMessage *prev;
 
   /**
-   * Callback to call once we got our HELLO.
+   * Overall transport handle.
    */
-  GNUNET_TRANSPORT_ReceiveCallback rec;
+  struct GNUNET_TRANSPORT_Handle *h;
 
   /**
-   * Closure for rec.
+   * Function to call when notify_size bytes are available
+   * for transmission.
    */
-  void *rec_cls;
+  GNUNET_CONNECTION_TransmitReadyNotify notify;
 
   /**
-   * When to time out (call rec with NULL).
+   * Closure for notify.
    */
-  struct GNUNET_TIME_Absolute timeout;
+  void *notify_cls;
 
   /**
-   * Timeout task (used to trigger timeout,
-   * cancel if we get the HELLO in time).
+   * transmit_ready task Id.  The task is used to introduce the
+   * artificial delay that may be required to maintain the bandwidth
+   * limits.  Later, this will be the ID of the "transmit_timeout"
+   * task which is used to signal a timeout if the transmission could
+   * not be done in a timely fashion.
    */
-  GNUNET_SCHEDULER_TaskIdentifier task;
+  GNUNET_SCHEDULER_TaskIdentifier notify_delay_task;
 
+  /**
+   * How many bytes is our notify callback waiting for?
+   */
+  size_t notify_size;
 
 };
 
 
 /**
- * Opaque handle for a transmission-ready request.
+ * Entry in linked list of all of our current neighbours.
  */
-struct GNUNET_TRANSPORT_TransmitHandle
+struct NeighbourList
 {
 
   /**
-   * We keep the transmit handles that are waiting for
-   * a transport-level connection in a doubly linked list.
+   * This is a linked list.
    */
-  struct GNUNET_TRANSPORT_TransmitHandle *next;
+  struct NeighbourList *next;
 
   /**
-   * We keep the transmit handles that are waiting for
-   * a transport-level connection in a doubly linked list.
+   * Overall transport handle.
    */
-  struct GNUNET_TRANSPORT_TransmitHandle *prev;
+  struct GNUNET_TRANSPORT_Handle *h;
 
   /**
-   * Handle of the main transport data structure.
+   * Active transmit handle; available if 'transmit_forbidden'
+   * is GNUNET_NO.
    */
-  struct GNUNET_TRANSPORT_Handle *handle;
+  struct GNUNET_TRANSPORT_TransmitHandle transmit_handle;
 
   /**
-   * Neighbour for this handle, can be NULL if the service
-   * is not yet connected to the target.
+   * Identity of this neighbour.
    */
-  struct NeighbourList *neighbour;
+  struct GNUNET_PeerIdentity id;
 
   /**
-   * Which peer is this transmission going to be for?  All
-   * zeros if it is control-traffic to the service.
+   * Outbound bandwidh tracker.
    */
-  struct GNUNET_PeerIdentity target;
+  struct GNUNET_BANDWIDTH_Tracker out_tracker;
 
   /**
-   * Function to call when notify_size bytes are available
-   * for transmission.
+   * Set to GNUNET_NO if we are currently allowed to accept a
+   * message to the transport service for this peer, GNUNET_YES
+   * if we have one and are waiting for transmission, GNUNET_SYSERR
+   * if we are waiting for confirmation AND have already accepted
+   * yet another message.
    */
-  GNUNET_NETWORK_TransmitReadyNotify notify;
+  enum TransmitStage transmit_stage;
 
   /**
-   * Closure for notify.
+   * Have we received a notification that this peer is connected
+   * to us right now?
    */
-  void *notify_cls;
+  int is_connected;
+
+};
+
+
+/**
+ * Linked list of requests from clients for our HELLO that were
+ * deferred.
+ */
+struct HelloWaitList
+{
 
   /**
-   * transmit_ready task Id.  The task is used to introduce the
-   * artificial delay that may be required to maintain the bandwidth
-   * limits.  Later, this will be the ID of the "transmit_timeout"
-   * task which is used to signal a timeout if the transmission could
-   * not be done in a timely fashion.
+   * This is a linked list.
    */
-  GNUNET_SCHEDULER_TaskIdentifier notify_delay_task;
+  struct HelloWaitList *next;
 
   /**
-   * Timeout for this request.
+   * Reference back to our transport handle.
    */
-  struct GNUNET_TIME_Absolute timeout;
+  struct GNUNET_TRANSPORT_Handle *handle;
 
   /**
-   * How many bytes is our notify callback waiting for?
+   * Callback to call once we got our HELLO.
    */
-  size_t notify_size;
+  GNUNET_TRANSPORT_HelloUpdateCallback rec;
+
+  /**
+   * Closure for rec.
+   */
+  void *rec_cls;
 
 };
 
@@ -255,40 +296,30 @@ struct GNUNET_TRANSPORT_Handle
   GNUNET_TRANSPORT_NotifyDisconnect nd_cb;
 
   /**
-   * The current HELLO message for this peer.  Updated
-   * whenever transports change their addresses.
+   * Head of DLL of control messages.
    */
-  struct GNUNET_HELLO_Message *my_hello;
+  struct ControlMessage *control_head;
 
   /**
-   * My client connection to the transport service.
+   * Tail of DLL of control messages.
    */
-  struct GNUNET_CLIENT_Connection *client;
+  struct ControlMessage *control_tail;
 
   /**
-   * Handle to our registration with the client for notification.
+   * The current HELLO message for this peer.  Updated
+   * whenever transports change their addresses.
    */
-  struct GNUNET_NETWORK_TransmitHandle *network_handle;
+  struct GNUNET_HELLO_Message *my_hello;
 
   /**
-   * Linked list of transmit handles that are waiting for the
-   * transport to connect to the respective peer.  When we
-   * receive notification that the transport connected to a
-   * peer, we go over this list and check if someone has already
-   * requested a transmission to the new peer; if so, we trigger
-   * the next step.
+   * My client connection to the transport service.
    */
-  struct GNUNET_TRANSPORT_TransmitHandle *connect_wait_head;
+  struct GNUNET_CLIENT_Connection *client;
 
   /**
-   * Linked list of transmit handles that are waiting for the
-   * transport to be ready for transmission to the respective
-   * peer.  When we
-   * receive notification that the transport disconnected from
-   * a peer, we go over this list and move the entry back to
-   * the connect_wait list.
+   * Handle to our registration with the client for notification.
    */
-  struct GNUNET_TRANSPORT_TransmitHandle *connect_ready_head;
+  struct GNUNET_CLIENT_TransmitHandle *network_handle;
 
   /**
    * Linked list of pending requests for our HELLO.
@@ -303,7 +334,7 @@ struct GNUNET_TRANSPORT_Handle
   /**
    * My configuration.
    */
-  struct GNUNET_CONFIGURATION_Handle *cfg;
+  const struct GNUNET_CONFIGURATION_Handle *cfg;
 
   /**
    * Linked list of the current neighbours of this peer.
@@ -311,27 +342,50 @@ struct GNUNET_TRANSPORT_Handle
   struct NeighbourList *neighbours;
 
   /**
-   * ID of the task trying to reconnect to the
-   * service.
+   * Peer identity as assumed by this process, or all zeros.
+   */
+  struct GNUNET_PeerIdentity self;
+
+  /**
+   * ID of the task trying to reconnect to the service.
    */
   GNUNET_SCHEDULER_TaskIdentifier reconnect_task;
 
+  /**
+   * ID of the task trying to trigger transmission for a peer
+   * while maintaining bandwidth quotas.
+   */
+  GNUNET_SCHEDULER_TaskIdentifier quota_task;
+
   /**
    * Delay until we try to reconnect.
    */
   struct GNUNET_TIME_Relative reconnect_delay;
 
   /**
-   * Do we currently have a transmission pending?
-   * (schedule transmission was called but has not
-   * yet succeeded)?
+   * Set once we are in the process of disconnecting from the
+   * service.
    */
-  int transmission_scheduled;
+  int in_disconnect;
+
+  /**
+   * Should we check that 'self' matches what the service thinks?
+   * (if GNUNET_NO, then 'self' is all zeros!).
+   */
+  int check_self;
 };
 
 
+// FIXME: replace with hash map!
+/**
+ * Get the neighbour list entry for the given peer
+ *
+ * @param h our context
+ * @param peer peer to look up
+ * @return NULL if no such peer entry exists
+ */
 static struct NeighbourList *
-find_neighbour (struct GNUNET_TRANSPORT_Handle *h,
+neighbour_find (struct GNUNET_TRANSPORT_Handle *h,
                 const struct GNUNET_PeerIdentity *peer)
 {
   struct NeighbourList *pos;
@@ -345,83 +399,252 @@ find_neighbour (struct GNUNET_TRANSPORT_Handle *h,
 
 
 /**
- * Schedule the task to send one message from the
- * connect_ready list to the service.
+ * Schedule the task to send one message, either from the control
+ * list or the peer message queues  to the service.
  */
 static void schedule_transmission (struct GNUNET_TRANSPORT_Handle *h);
 
 
 /**
- * Transmit message to client...
+ * Function called by the scheduler when the timeout for bandwidth
+ * availablility for the target neighbour is reached.
+ *
+ * @param cls the 'struct GNUNET_TRANSPORT_Handle*'
+ * @param tc scheduler context
+ */
+static void
+quota_transmit_ready (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_TRANSPORT_Handle *h = cls;
+
+  h->quota_task = GNUNET_SCHEDULER_NO_TASK;
+  schedule_transmission (h);
+}
+
+
+/**
+ * Figure out which transmission to a peer can be done right now.
+ * If none can, schedule a task to call 'schedule_transmission'
+ * whenever a peer transmission can be done in the future and
+ * return NULL.  Otherwise return the next transmission to be
+ * performed.
+ *
+ * @param h handle to transport
+ * @return NULL to wait longer before doing any peer transmissions
+ */
+static struct GNUNET_TRANSPORT_TransmitHandle *
+schedule_peer_transmission (struct GNUNET_TRANSPORT_Handle *h)
+{
+  struct GNUNET_TRANSPORT_TransmitHandle *ret;
+  struct GNUNET_TRANSPORT_TransmitHandle *th;
+  struct NeighbourList *n;
+  struct NeighbourList *next;
+  struct GNUNET_TIME_Relative retry_time;
+  struct GNUNET_TIME_Relative duration;
+
+  if (h->quota_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (h->sched,
+                              h->quota_task);
+      h->quota_task = GNUNET_SCHEDULER_NO_TASK;
+    }
+  retry_time = GNUNET_TIME_UNIT_FOREVER_REL;
+  ret = NULL;
+  next = h->neighbours;
+  while (NULL != (n = next))
+    {
+      next = n->next;
+      if (n->transmit_stage != TS_QUEUED)
+       continue; /* not eligible */
+      if (n->is_connected != GNUNET_YES)
+        continue;
+
+      th = &n->transmit_handle;
+      GNUNET_break (n == th->neighbour);
+      /* check outgoing quota */
+      duration = GNUNET_BANDWIDTH_tracker_get_delay (&n->out_tracker,
+                                                    th->notify_size - sizeof (struct OutboundMessage));
+      if (th->timeout.value < duration.value)
+       {
+         /* signal timeout! */
+#if DEBUG_TRANSPORT
+         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                     "Would need %llu ms before bandwidth is available for delivery to `%4s', that is too long.  Signaling timeout.\n",
+                     duration.value,
+                     GNUNET_i2s (&n->id));
+#endif
+         if (th->notify_delay_task != GNUNET_SCHEDULER_NO_TASK)
+           {
+             GNUNET_SCHEDULER_cancel (h->sched, th->notify_delay_task);
+             th->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+           }   
+         n->transmit_stage = TS_NEW;
+         if (NULL != th->notify)
+           GNUNET_assert (0 == th->notify (th->notify_cls, 0, NULL));
+         continue;
+       }
+      if (duration.value > 0)
+       {
+#if DEBUG_TRANSPORT
+         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                     "Need more bandwidth (%u b/s allowed, %u b needed), delaying delivery to `%4s' by %llu ms\n",
+                     (unsigned int) n->out_tracker.available_bytes_per_s__,
+                     (unsigned int) th->notify_size - sizeof (struct OutboundMessage),
+                     GNUNET_i2s (&n->id),
+                     duration.value);
+#endif
+         retry_time = GNUNET_TIME_relative_min (retry_time,
+                                                duration);
+         continue;
+       }
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Have %u bytes of bandwidth available for transmission to `%4s' right now\n",
+                 th->notify_size - sizeof (struct OutboundMessage),
+                 GNUNET_i2s (&n->id));
+#endif 
+
+      if ( (ret == NULL) ||
+          (ret->priority < th->priority) )
+       ret = th;
+    }
+  if (ret == NULL)
+    h->quota_task = GNUNET_SCHEDULER_add_delayed (h->sched,
+                                                 retry_time,
+                                                 &quota_transmit_ready,
+                                                 h);
+  return ret;
+}
+
+
+/**
+ * Transmit message(s) to service.
+ *
+ * @param cls handle to transport
+ * @param size number of bytes available in buf
+ * @param buf where to copy the message
+ * @return number of bytes copied to buf
  */
 static size_t
 transport_notify_ready (void *cls, size_t size, void *buf)
 {
   struct GNUNET_TRANSPORT_Handle *h = cls;
+  struct ControlMessage *cm;
   struct GNUNET_TRANSPORT_TransmitHandle *th;
   struct NeighbourList *n;
+  struct OutboundMessage obm;
   size_t ret;
+  size_t mret;
+  size_t nret;
   char *cbuf;
 
   h->network_handle = NULL;
-  h->transmission_scheduled = GNUNET_NO;
   if (buf == NULL)
     {
-#if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                 "Could not transmit to transport service, cancelling pending requests\n");
-#endif
-      th = h->connect_ready_head;
-      if (th->next != NULL)
-        th->next->prev = NULL;
-      h->connect_ready_head = th->next;
-      if (NULL != (n = th->neighbour))
-        {
-          GNUNET_assert (n->transmit_handle == th);
-          n->transmit_handle = NULL;
-        }
-      GNUNET_assert (0 == th->notify (th->notify_cls, 0, NULL));
-      GNUNET_free (th);
+      schedule_transmission (h);
       return 0;
-    } 
+    }
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Ready to transmit %u bytes to transport service\n", size);
 #endif
   cbuf = buf;
   ret = 0;
-  h->network_handle = NULL;
-  h->transmission_scheduled = GNUNET_NO;
-  do
+  while ( (NULL != (cm = h->control_head)) &&
+         (cm->notify_size <= size) )
     {
-      th = h->connect_ready_head;
-      if (th->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-       {
-         /* remove existing time out task (only applies if
-            this is not the first iteration of the loop) */
-         GNUNET_SCHEDULER_cancel (h->sched,
-                                  th->notify_delay_task);
-         th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-       }
-      GNUNET_assert (th->notify_size <= size);
-      if (th->next != NULL)
-        th->next->prev = NULL;
-      h->connect_ready_head = th->next;
-      if (NULL != (n = th->neighbour))
+      if (cm->notify_delay_task != GNUNET_SCHEDULER_NO_TASK)
         {
-          GNUNET_assert (n->transmit_handle == th);
-          n->transmit_handle = NULL;
+          GNUNET_SCHEDULER_cancel (h->sched, cm->notify_delay_task);
+          cm->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
         }
-      ret += th->notify (th->notify_cls, size, &cbuf[ret]);
-      GNUNET_free (th);
-      if (n != NULL)
-        n->last_sent += ret;
-      size -= ret;
+      GNUNET_CONTAINER_DLL_remove (h->control_head,
+                                  h->control_tail,
+                                  cm);
+      nret = cm->notify (cm->notify_cls, size, &cbuf[ret]);
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Added %u bytes of control message at %u\n",
+                 nret,
+                 ret);
+#endif
+      GNUNET_free (cm);
+      ret += nret;
+      size -= nret;
     }
-  while ((h->connect_ready_head != NULL) &&
-         (h->connect_ready_head->notify_size <= size));
-  if (h->connect_ready_head != NULL)
-    schedule_transmission (h);
+  while ( (NULL != (th = schedule_peer_transmission (h))) &&
+         (th->notify_size <= size) )
+    {
+      if (th->notify_delay_task != GNUNET_SCHEDULER_NO_TASK)
+        {
+          GNUNET_SCHEDULER_cancel (h->sched, th->notify_delay_task);
+          th->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+        }
+      n = th->neighbour;
+      switch (n->transmit_stage)
+       {
+       case TS_NEW:
+         GNUNET_break (0);
+         break;
+       case TS_QUEUED:
+         n->transmit_stage = TS_TRANSMITTED;
+         break;
+       case TS_TRANSMITTED:
+         GNUNET_break (0);
+         break;
+       case TS_TRANSMITTED_QUEUED:
+         GNUNET_break (0);
+         break;
+       default:
+         GNUNET_break (0);
+       }
+      GNUNET_assert (size >= sizeof (struct OutboundMessage));
+      mret = th->notify (th->notify_cls,
+                        size - sizeof (struct OutboundMessage),
+                        &cbuf[ret + sizeof (struct OutboundMessage)]);
+      GNUNET_assert (mret <= size - sizeof (struct OutboundMessage));
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Message of %u bytes with timeout %llums constructed for `%4s'\n",
+                 (unsigned int) mret,
+                 (unsigned long long) GNUNET_TIME_absolute_get_remaining (th->timeout).value,
+                 GNUNET_i2s (&n->id));
+#endif
+      if (mret != 0)   
+       {
+         GNUNET_assert (mret + sizeof (struct OutboundMessage) < GNUNET_SERVER_MAX_MESSAGE_SIZE);
+         obm.header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_SEND);
+         obm.header.size = htons (mret + sizeof (struct OutboundMessage));
+         obm.priority = htonl (th->priority);
+         obm.timeout = GNUNET_TIME_relative_hton (GNUNET_TIME_absolute_get_remaining (th->timeout));
+         obm.peer = n->id;
+         memcpy (&cbuf[ret], &obm, sizeof (struct OutboundMessage));
+         ret += (mret + sizeof (struct OutboundMessage));
+         size -= (mret + sizeof (struct OutboundMessage));
+         GNUNET_BANDWIDTH_tracker_consume (&n->out_tracker, mret);
+       }
+      else
+       {
+         switch (n->transmit_stage)
+           {
+           case TS_NEW:
+             GNUNET_break (0);
+             break;
+           case TS_QUEUED:
+             GNUNET_break (0);
+             break;
+           case TS_TRANSMITTED:
+             n->transmit_stage = TS_NEW;
+             break;
+           case TS_TRANSMITTED_QUEUED:
+             n->transmit_stage = TS_QUEUED;
+             continue;
+           default:
+             GNUNET_break (0);
+           }
+       }
+    }
+  schedule_transmission (h);
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Transmitting %u bytes to transport service\n", ret);
@@ -431,112 +654,56 @@ transport_notify_ready (void *cls, size_t size, void *buf)
 
 
 /**
- * Schedule the task to send one message from the
- * connect_ready list to the service.
+ * Schedule the task to send one message, either from the control
+ * list or the peer message queues  to the service.
  */
 static void
 schedule_transmission (struct GNUNET_TRANSPORT_Handle *h)
 {
+  size_t size;
+  struct GNUNET_TIME_Relative timeout;
   struct GNUNET_TRANSPORT_TransmitHandle *th;
 
-  GNUNET_assert (NULL == h->network_handle);
+  if (NULL != h->network_handle)
+    return;
   if (h->client == NULL)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                 "Could not yet schedule transmission: we are not yet connected to the transport service!\n");
-      return; /* not yet connected */
-    }
-  th = h->connect_ready_head;
-  if (th == NULL)
-    return; /* no request pending */
-  if (th->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-    {
-      /* remove existing time out task, will be integrated
-        with transmit_ready notification! */
-      GNUNET_SCHEDULER_cancel (h->sched,
-                              th->notify_delay_task);
-      th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-    }
-  h->transmission_scheduled = GNUNET_YES;
-  h->network_handle = GNUNET_CLIENT_notify_transmit_ready (h->client,
-                                                           th->notify_size,
-                                                           GNUNET_TIME_absolute_get_remaining
-                                                           (th->timeout),
-                                                           &transport_notify_ready,
-                                                           h);
-  GNUNET_assert (NULL != h->network_handle);
-}
-
-
-/**
- * Insert the given transmit handle in the given sorted
- * doubly linked list based on timeout.
- *
- * @param head pointer to the head of the linked list
- * @param th element to insert into the list
- */
-static void
-insert_transmit_handle (struct GNUNET_TRANSPORT_TransmitHandle **head,
-                        struct GNUNET_TRANSPORT_TransmitHandle *th)
-{
-  struct GNUNET_TRANSPORT_TransmitHandle *pos;
-  struct GNUNET_TRANSPORT_TransmitHandle *prev;
-
-  pos = *head;
-  prev = NULL;
-  while ((pos != NULL) && (pos->timeout.value < th->timeout.value))
-    {
-      prev = pos;
-      pos = pos->next;
-    }
-  if (prev == NULL)
-    {
-      th->next = *head;
-      if (th->next != NULL)
-        th->next->prev = th;
-      *head = th;
-    }
-  else
-    {
-      th->next = pos;
-      th->prev = prev;
-      prev->next = th;
-      if (pos != NULL)
-        pos->prev = th;
-    }
-}
-
-
-/**
- * Cancel a pending notify delay task (if pending) and also remove the
- * given transmit handle from whatever list is on.
- *
- * @param th handle for the transmission request to manipulate
- */
-static void
-remove_from_any_list (struct GNUNET_TRANSPORT_TransmitHandle *th)
-{
-  struct GNUNET_TRANSPORT_Handle *h;
-
-  h = th->handle;
-  if (th->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-    {
-      GNUNET_SCHEDULER_cancel (h->sched, th->notify_delay_task);
-      th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Could not yet schedule transmission: we are not yet connected to the transport service!\n"));
+      return;                   /* not yet connected */
     }
-  if (th->prev == NULL)
+  if (NULL != h->control_head)
     {
-      if (th == h->connect_wait_head)
-        h->connect_wait_head = th->next;
-      else
-        h->connect_ready_head = th->next;
+      size = h->control_head->notify_size;
+      timeout = GNUNET_TIME_UNIT_FOREVER_REL;
     }
   else
     {
-      th->prev->next = th->next;
+      th = schedule_peer_transmission (h);
+      if (th == NULL)
+       {
+         /* no transmission ready right now */
+#if DEBUG_TRANSPORT
+         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                     "Could not yet schedule transmission: none ready\n");
+#endif
+         return;
+       }
+      size = th->notify_size;
+      timeout = GNUNET_TIME_absolute_get_remaining (th->timeout);
     }
-  if (th->next != NULL)
-    th->next->prev = th->prev;
+#if DEBUG_TRANSPORT
+    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                "Calling notify_transmit_ready\n");
+#endif
+  h->network_handle =
+    GNUNET_CLIENT_notify_transmit_ready (h->client,
+                                        size,
+                                        timeout,
+                                        GNUNET_NO,
+                                        &transport_notify_ready,
+                                        h);
+  GNUNET_assert (NULL != h->network_handle);
 }
 
 
@@ -547,30 +714,26 @@ remove_from_any_list (struct GNUNET_TRANSPORT_TransmitHandle *th)
  * TransmitHandle.
  */
 static void
-transmit_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+control_transmit_timeout (void *cls,
+                         const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct GNUNET_TRANSPORT_TransmitHandle *th = cls;
-
-  th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-  if (th->neighbour != NULL)
-    th->neighbour->transmit_handle = NULL;
-#if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_INFO, 
-             "Request for transmission to peer `%s' timed out.\n",
-             GNUNET_i2s(&th->target));
-#endif
-  remove_from_any_list (th);
-  th->notify (th->notify_cls, 0, NULL);
+  struct ControlMessage *th = cls;
+
+  th->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+  if (NULL != th->notify)
+    th->notify (th->notify_cls, 0, NULL);
+  GNUNET_CONTAINER_DLL_remove (th->h->control_head,
+                              th->h->control_tail,
+                              th);
   GNUNET_free (th);
 }
 
 
-
-
 /**
  * Queue control request for transmission to the transport
  * service.
  *
+ * @param h handle to the transport service
  * @param size number of bytes to be transmitted
  * @param at_head request must be added to the head of the queue
  *        (otherwise request will be appended)
@@ -583,70 +746,34 @@ schedule_control_transmit (struct GNUNET_TRANSPORT_Handle *h,
                            size_t size,
                            int at_head,
                            struct GNUNET_TIME_Relative timeout,
-                           GNUNET_NETWORK_TransmitReadyNotify notify,
+                           GNUNET_CONNECTION_TransmitReadyNotify notify,
                            void *notify_cls)
 {
-  struct GNUNET_TRANSPORT_TransmitHandle *th;
+  struct ControlMessage *th;
 
-  th = GNUNET_malloc (sizeof (struct GNUNET_TRANSPORT_TransmitHandle));
-  th->handle = h;
+#if DEBUG_TRANSPORT
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Control transmit of %u bytes within %llums requested\n",
+              size, (unsigned long long) timeout.value);
+#endif
+  th = GNUNET_malloc (sizeof (struct ControlMessage));
+  th->h = h;
   th->notify = notify;
   th->notify_cls = notify_cls;
-  th->timeout = GNUNET_TIME_relative_to_absolute (timeout);
   th->notify_size = size;
-  th->notify_delay_task 
+  th->notify_delay_task
     = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                   GNUNET_NO,
-                                   GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                   GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                   timeout,
-                                   &transmit_timeout, th);    
+                                    timeout, &control_transmit_timeout, th);
   if (at_head)
-    {
-      th->next = h->connect_ready_head;
-      h->connect_ready_head = th;
-      if (th->next != NULL)
-        th->next->prev = th;
-    }
+    GNUNET_CONTAINER_DLL_insert (h->control_head,
+                                h->control_tail,
+                                th);
   else
-    {
-      insert_transmit_handle (&h->connect_ready_head, th);
-    }
-  if (GNUNET_NO == h->transmission_scheduled)
-    schedule_transmission (h);
-}
-
-
-/**
- * Update the quota values for the given neighbour now.
- */
-static void
-update_quota (struct NeighbourList *n)
-{
-  struct GNUNET_TIME_Relative delta;
-  uint64_t allowed;
-  uint64_t remaining;
-
-  delta = GNUNET_TIME_absolute_get_duration (n->last_quota_update);
-  allowed = delta.value * n->quota_out;
-  if (n->last_sent < allowed)
-    {
-      remaining = allowed - n->last_sent;
-      if (n->quota_out > 0)
-        remaining /= n->quota_out;
-      else
-        remaining = 0;
-      if (remaining > MAX_BANDWIDTH_CARRY)
-        remaining = MAX_BANDWIDTH_CARRY;
-      n->last_sent = 0;
-      n->last_quota_update = GNUNET_TIME_absolute_get ();
-      n->last_quota_update.value -= remaining;
-    }
-  else
-    {
-      n->last_sent -= allowed;
-      n->last_quota_update = GNUNET_TIME_absolute_get ();
-    }
+    GNUNET_CONTAINER_DLL_insert_after (h->control_head,
+                                      h->control_tail,
+                                      h->control_tail,
+                                      th);
+  schedule_transmission (h);
 }
 
 
@@ -662,10 +789,18 @@ struct SetQuotaContext
 
   struct GNUNET_TIME_Absolute timeout;
 
-  uint32_t quota_in;
+  struct GNUNET_BANDWIDTH_Value32NBO quota_in;
 };
 
 
+/**
+ * Send SET_QUOTA message to the service.
+ *
+ * @param cls the 'struct SetQuotaContext'
+ * @param size number of bytes available in buf
+ * @param buf where to copy the message
+ * @return number of bytes copied to buf
+ */
 static size_t
 send_set_quota (void *cls, size_t size, void *buf)
 {
@@ -675,7 +810,6 @@ send_set_quota (void *cls, size_t size, void *buf)
   if (buf == NULL)
     {
       GNUNET_SCHEDULER_add_continuation (sqc->handle->sched,
-                                         GNUNET_NO,
                                          sqc->cont,
                                          sqc->cont_cls,
                                          GNUNET_SCHEDULER_REASON_TIMEOUT);
@@ -685,17 +819,17 @@ send_set_quota (void *cls, size_t size, void *buf)
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Transmitting `%s' request with respect to `%4s'.\n",
-              "SET_QUOTA", GNUNET_i2s (&sqc->target));
+              "SET_QUOTA",
+             GNUNET_i2s (&sqc->target));
 #endif
   GNUNET_assert (size >= sizeof (struct QuotaSetMessage));
   msg = buf;
   msg->header.size = htons (sizeof (struct QuotaSetMessage));
   msg->header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_SET_QUOTA);
-  msg->quota_in = htonl (sqc->quota_in);
+  msg->quota = sqc->quota_in;
   memcpy (&msg->peer, &sqc->target, sizeof (struct GNUNET_PeerIdentity));
   if (sqc->cont != NULL)
     GNUNET_SCHEDULER_add_continuation (sqc->handle->sched,
-                                       GNUNET_NO,
                                        sqc->cont,
                                        sqc->cont_cls,
                                        GNUNET_SCHEDULER_REASON_PREREQ_DONE);
@@ -710,10 +844,8 @@ send_set_quota (void *cls, size_t size, void *buf)
  *
  * @param handle connection to transport service
  * @param target who's bandwidth quota is being changed
- * @param quota_in incoming bandwidth quota in bytes per ms; 0 can
- *        be used to force all traffic to be discarded
- * @param quota_out outgoing bandwidth quota in bytes per ms; 0 can
- *        be used to force all traffic to be discarded
+ * @param quota_in incoming bandwidth quota in bytes per ms
+ * @param quota_out outgoing bandwidth quota in bytes per ms
  * @param timeout how long to wait until signaling failure if
  *        we can not communicate the quota change
  * @param cont continuation to call when done, will be called
@@ -723,21 +855,41 @@ send_set_quota (void *cls, size_t size, void *buf)
 void
 GNUNET_TRANSPORT_set_quota (struct GNUNET_TRANSPORT_Handle *handle,
                             const struct GNUNET_PeerIdentity *target,
-                            uint32_t quota_in,
-                            uint32_t quota_out,
+                            struct GNUNET_BANDWIDTH_Value32NBO quota_in,
+                            struct GNUNET_BANDWIDTH_Value32NBO quota_out,
                             struct GNUNET_TIME_Relative timeout,
                             GNUNET_SCHEDULER_Task cont, void *cont_cls)
 {
   struct NeighbourList *n;
   struct SetQuotaContext *sqc;
 
-  n = find_neighbour (handle, target);
+  n = neighbour_find (handle, target);
   if (n != NULL)
     {
-      update_quota (n);
-      if (n->quota_out < quota_out)
-        n->last_quota_update = GNUNET_TIME_absolute_get ();
-      n->quota_out = quota_out;
+#if DEBUG_TRANSPORT
+      if (ntohl (quota_out.value__) != n->out_tracker.available_bytes_per_s__)
+       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                   "Quota changed from %u to %u for peer `%s'\n",
+                   (unsigned int) n->out_tracker.available_bytes_per_s__,
+                   (unsigned int) ntohl (quota_out.value__),
+                   GNUNET_i2s (target));
+      else
+       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                   "Quota remains at %u for peer `%s'\n",
+                   (unsigned int) n->out_tracker.available_bytes_per_s__,
+                   GNUNET_i2s (target));
+#endif
+      GNUNET_BANDWIDTH_tracker_update_quota (&n->out_tracker,
+                                            quota_out);
+    }
+  else
+    {
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Quota changed to %u for peer `%s', but I have no such neighbour!\n",
+                 (unsigned int) ntohl (quota_out.value__),
+                 GNUNET_i2s (target));
+#endif
     }
   sqc = GNUNET_malloc (sizeof (struct SetQuotaContext));
   sqc->handle = handle;
@@ -752,44 +904,10 @@ GNUNET_TRANSPORT_set_quota (struct GNUNET_TRANSPORT_Handle *handle,
 }
 
 
-/**
- * A "get_hello" request has timed out.  Signal the client
- * and clean up.
- */
-static void
-hello_wait_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
-{
-  struct HelloWaitList *hwl = cls;
-  struct HelloWaitList *pos;
-  struct HelloWaitList *prev;
-
-  prev = NULL;
-  pos = hwl->handle->hwl_head;
-  while (pos != hwl)
-    {
-      GNUNET_assert (pos != NULL);
-      prev = pos;
-      pos = pos->next;
-    }
-  if (prev == NULL)
-    hwl->handle->hwl_head = hwl->next;
-  else
-    prev->next = hwl->next;
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-              _("Timeout trying to obtain `%s' from transport service.\n"),
-              "HELLO");
-  /* signal timeout */
-  if (hwl->rec != NULL)
-    hwl->rec (hwl->rec_cls, GNUNET_TIME_UNIT_ZERO, NULL, NULL);
-  GNUNET_free (hwl);
-}
-
-
 /**
  * Obtain the HELLO message for this peer.
  *
  * @param handle connection to transport service
- * @param timeout how long to wait for the HELLO
  * @param rec function to call with the HELLO, sender will be our peer
  *            identity; message and sender will be NULL on timeout
  *            (handshake with transport service pending/failed).
@@ -798,42 +916,68 @@ hello_wait_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
  */
 void
 GNUNET_TRANSPORT_get_hello (struct GNUNET_TRANSPORT_Handle *handle,
-                            struct GNUNET_TIME_Relative timeout,
-                            GNUNET_TRANSPORT_ReceiveCallback rec,
+                            GNUNET_TRANSPORT_HelloUpdateCallback rec,
                             void *rec_cls)
 {
-  struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pk;
-  struct GNUNET_PeerIdentity me;
   struct HelloWaitList *hwl;
 
+  hwl = GNUNET_malloc (sizeof (struct HelloWaitList));
+  hwl->next = handle->hwl_head;
+  handle->hwl_head = hwl;
+  hwl->handle = handle;
+  hwl->rec = rec;
+  hwl->rec_cls = rec_cls;
   if (handle->my_hello == NULL)
+    return;
+  rec (rec_cls, (const struct GNUNET_MessageHeader *) handle->my_hello);
+}
+
+
+
+/**
+ * Stop receiving updates about changes to our HELLO message.
+ *
+ * @param handle connection to transport service
+ * @param rec function previously registered to be called with the HELLOs
+ * @param rec_cls closure for rec
+ */
+void
+GNUNET_TRANSPORT_get_hello_cancel (struct GNUNET_TRANSPORT_Handle *handle,
+                                  GNUNET_TRANSPORT_HelloUpdateCallback rec,
+                                  void *rec_cls)
+{
+  struct HelloWaitList *pos;
+  struct HelloWaitList *prev;
+
+  prev = NULL;
+  pos = handle->hwl_head;
+  while (pos != NULL)
     {
-      hwl = GNUNET_malloc (sizeof (struct HelloWaitList));
-      hwl->next = handle->hwl_head;
-      handle->hwl_head = hwl;
-      hwl->handle = handle;
-      hwl->rec = rec;
-      hwl->rec_cls = rec_cls;
-      hwl->timeout = GNUNET_TIME_relative_to_absolute (timeout);
-      hwl->task = GNUNET_SCHEDULER_add_delayed (handle->sched,
-                                                GNUNET_YES,
-                                                GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                                GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                                timeout,
-                                                &hello_wait_timeout, hwl);
-      return;
+      if ( (pos->rec == rec) &&
+          (pos->rec_cls == rec_cls) )
+       break;
+      prev = pos;
+      pos = pos->next;
     }
-  GNUNET_assert (GNUNET_OK == GNUNET_HELLO_get_key (handle->my_hello, &pk));
-  GNUNET_CRYPTO_hash (&pk,
-                      sizeof (struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
-                      &me.hashPubKey);
-
-  rec (rec_cls,
-       GNUNET_TIME_UNIT_ZERO,
-       &me, (const struct GNUNET_MessageHeader *) handle->my_hello);
+  GNUNET_break (pos != NULL);
+  if (pos == NULL)
+    return;
+  if (prev == NULL)
+    handle->hwl_head = pos->next;
+  else
+    prev->next = pos->next;
+  GNUNET_free (pos);
 }
 
 
+/**
+ * Send HELLO message to the service.
+ *
+ * @param cls the HELLO message to send
+ * @param size number of bytes available in buf
+ * @param buf where to copy the message
+ * @return number of bytes copied to buf
+ */
 static size_t
 send_hello (void *cls, size_t size, void *buf)
 {
@@ -842,8 +986,8 @@ send_hello (void *cls, size_t size, void *buf)
 
   if (buf == NULL)
     {
-#if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+#if DEBUG_TRANSPORT_TIMEOUT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Timeout while trying to transmit `%s' request.\n",
                   "HELLO");
 #endif
@@ -876,18 +1020,23 @@ GNUNET_TRANSPORT_offer_hello (struct GNUNET_TRANSPORT_Handle *handle,
 {
   struct GNUNET_MessageHeader *hc;
   uint16_t size;
+  struct GNUNET_PeerIdentity peer;
 
-  if (handle->client == NULL)
-    {
-#if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                  "Not connected to transport service, dropping offered HELLO\n");
-#endif
-      return;
-    }
   GNUNET_break (ntohs (hello->type) == GNUNET_MESSAGE_TYPE_HELLO);
   size = ntohs (hello->size);
   GNUNET_break (size >= sizeof (struct GNUNET_MessageHeader));
+  if (GNUNET_OK != GNUNET_HELLO_get_id ((const struct GNUNET_HELLO_Message*) hello,
+                                       &peer))
+    {
+      GNUNET_break (0);
+      return;
+    }
+#if DEBUG_TRANSPORT
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Offering `%s' message of `%4s' to transport for validation.\n",
+             "HELLO",
+             GNUNET_i2s (&peer));
+#endif
   hc = GNUNET_malloc (size);
   memcpy (hc, hello, size);
   schedule_control_transmit (handle,
@@ -897,21 +1046,25 @@ GNUNET_TRANSPORT_offer_hello (struct GNUNET_TRANSPORT_Handle *handle,
 
 
 /**
- * Function we use for handling incoming messages.
+ * Transmit START message to service.
+ *
+ * @param cls unused
+ * @param size number of bytes available in buf
+ * @param buf where to copy the message
+ * @return number of bytes copied to buf
  */
-static void demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg);
-
-
 static size_t
 send_start (void *cls, size_t size, void *buf)
 {
-  struct GNUNET_MessageHeader *s = buf;
+  struct GNUNET_TRANSPORT_Handle *h = cls;
+  struct StartMessage s;
 
   if (buf == NULL)
     {
+      /* Can only be shutdown, just give up */
 #if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  "Timeout while trying to transmit `%s' request.\n",
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Shutdown while trying to transmit `%s' request.\n",
                   "START");
 #endif
       return 0;
@@ -920,168 +1073,150 @@ send_start (void *cls, size_t size, void *buf)
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               "Transmitting `%s' request.\n", "START");
 #endif
-  GNUNET_assert (size >= sizeof (struct GNUNET_MessageHeader));
-  s->size = htons (sizeof (struct GNUNET_MessageHeader));
-  s->type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_START);
-  return sizeof (struct GNUNET_MessageHeader);
+  GNUNET_assert (size >= sizeof (struct StartMessage));
+  s.header.size = htons (sizeof (struct StartMessage));
+  s.header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_START);
+  s.do_check = htonl (h->check_self);
+  s.self = h->self;
+  memcpy (buf, &s, sizeof (struct StartMessage));
+  return sizeof (struct StartMessage);
 }
 
 
 /**
- * We're ready to transmit the request that the transport service
- * should connect to a new peer.  In addition to sending the
- * request, schedule the next phase for the transmission processing
- * that caused the connect request in the first place.
+ * Free neighbour.
+ *
+ * @param n the entry to free
  */
-static size_t
-request_connect (void *cls, size_t size, void *buf)
+static void
+neighbour_free (struct NeighbourList *n)
 {
-  struct GNUNET_TRANSPORT_TransmitHandle *th = cls;
-  struct TryConnectMessage *tcm;
   struct GNUNET_TRANSPORT_Handle *h;
+  struct NeighbourList *prev;
+  struct NeighbourList *pos;
 
-  GNUNET_assert (th->notify_delay_task == GNUNET_SCHEDULER_NO_PREREQUISITE_TASK);
-  h = th->handle;
-  if (buf == NULL)
-    {
-#if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Failed to transmit `%s' request for `%4s' to service.\n",
-                 "TRY_CONNECT",
-                 GNUNET_i2s(&th->target));
-#endif
-      th->notify (th->notify_cls, 0, NULL);
-      GNUNET_free (th);
-      return 0;
-    }
+  h = n->h;
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Transmitting `%s' message for `%4s'.\n",
-              "TRY_CONNECT", GNUNET_i2s (&th->target));
+              "Removing neighbour `%s' from list of connected peers.\n",
+              GNUNET_i2s (&n->id));
 #endif
-  GNUNET_assert (size >= sizeof (struct TryConnectMessage));
-  tcm = buf;
-  tcm->header.size = htons (sizeof (struct TryConnectMessage));
-  tcm->header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_TRY_CONNECT);
-  tcm->reserved = htonl (0);
-  memcpy (&tcm->peer, &th->target, sizeof (struct GNUNET_PeerIdentity));
-  th->notify_delay_task
-    = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                    GNUNET_NO,
-                                    GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                    GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                    GNUNET_TIME_absolute_get_remaining
-                                    (th->timeout), &transmit_timeout, th);
-  insert_transmit_handle (&h->connect_wait_head, th);
-  return sizeof (struct TryConnectMessage);
+  GNUNET_break (n->is_connected == GNUNET_NO);
+  GNUNET_break (n->transmit_stage == TS_NEW);
+
+  prev = NULL;
+  pos = h->neighbours;
+  while (pos != n)
+    {
+      prev = pos;
+      pos = pos->next;
+    }
+  if (prev == NULL)
+    h->neighbours = n->next;
+  else
+    prev->next = n->next;
+  GNUNET_free (n);
 }
 
 
 /**
- * Schedule a request to connect to the given
- * neighbour (and if successful, add the specified
- * handle to the wait list).
+ * Mark neighbour as disconnected.
  *
- * @param th handle for a request to transmit once we
- *        have connected
+ * @param n the entry to mark as disconnected
  */
 static void
-try_connect (struct GNUNET_TRANSPORT_TransmitHandle *th)
+neighbour_disconnect (struct NeighbourList *n)
 {
-  GNUNET_assert (th->notify_delay_task == GNUNET_SCHEDULER_NO_PREREQUISITE_TASK);  
-  schedule_control_transmit (th->handle,
-                             sizeof (struct TryConnectMessage),
-                             GNUNET_NO,
-                             GNUNET_TIME_absolute_get_remaining (th->timeout),
-                             &request_connect, th);
+  struct GNUNET_TRANSPORT_Handle *h = n->h;
+#if DEBUG_TRANSPORT
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Removing neighbour `%s' from list of connected peers.\n",
+              GNUNET_i2s (&n->id));
+#endif
+  GNUNET_break (n->is_connected == GNUNET_YES);
+  n->is_connected = GNUNET_NO;
+  if (h->nc_cb != NULL)
+    h->nd_cb (h->cls, &n->id);
+  if (n->transmit_stage == TS_NEW)
+    neighbour_free (n);
 }
 
 
 /**
- * Remove neighbour from our list
+ * Function we use for handling incoming messages.
+ *
+ * @param cls closure (struct GNUNET_TRANSPORT_Handle *)
+ * @param msg message received, NULL on timeout or fatal error
  */
-static void
-remove_neighbour (struct GNUNET_TRANSPORT_Handle *h,
-                  const struct GNUNET_PeerIdentity *peer)
-{
-  struct NeighbourList *prev;
-  struct NeighbourList *pos;
-  struct GNUNET_TRANSPORT_TransmitHandle *th;
-
-  prev = NULL;
-  pos = h->neighbours;
-  while ((pos != NULL) &&
-         (0 != memcmp (peer, &pos->id, sizeof (struct GNUNET_PeerIdentity))))
-    {
-      prev = pos;
-      pos = pos->next;
-    }
-  if (pos == NULL)
-    {
-      GNUNET_break (0);
-      return;
-    }
-  if (prev == NULL)
-    h->neighbours = pos->next;
-  else
-    prev->next = pos->next;
-  if (NULL != (th = pos->transmit_handle))
-    {
-      pos->transmit_handle = NULL;
-      th->neighbour = NULL;
-      remove_from_any_list (th);
-      try_connect (th);
-    }
-  if (h->nc_cb != NULL)
-    h->nd_cb (h->cls, peer);
-  GNUNET_free (pos);
-}
+static void demultiplexer (void *cls,
+                          const struct GNUNET_MessageHeader *msg);
 
 
 /**
  * Try again to connect to transport service.
+ *
+ * @param cls the handle to the transport service
+ * @param tc scheduler context
  */
 static void
-reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+reconnect (void *cls,
+          const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct GNUNET_TRANSPORT_Handle *h = cls;
-  struct GNUNET_TRANSPORT_TransmitHandle *pos;
+  struct ControlMessage *pos;
   struct NeighbourList *n;
+  struct NeighbourList *next;
 
-  /* Forget about all neighbours that we used to be connected
-     to */
-  while (NULL != (n = h->neighbours))
-    remove_neighbour (h, &n->id);
+  h->reconnect_task = GNUNET_SCHEDULER_NO_TASK;
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
+    {
+      /* shutdown, just give up */
+      return;
+    }
+  /* Forget about all neighbours that we used to be connected to */
+  n = h->neighbours;
+  while (NULL != n)
+    {
+#if DEBUG_TRANSPORT_DISCONNECT
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Disconnecting due to reconnect being called\n");
+#endif
+      next = n->next;
+      if (n->is_connected)
+       neighbour_disconnect (n);
+      n = next;
+    }
 #if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connecting to transport service.\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+             "Connecting to transport service.\n");
 #endif
   GNUNET_assert (h->client == NULL);
-  h->reconnect_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
   h->client = GNUNET_CLIENT_connect (h->sched, "transport", h->cfg);
   GNUNET_assert (h->client != NULL);
-  /* make sure we don't send "START" twice,
-     remove existing entry from queue (if present) */
-  pos = h->connect_ready_head;
+  /* make sure we don't send "START" twice, remove existing entry from
+     queue (if present) */
+  pos = h->control_head;
   while (pos != NULL)
     {
       if (pos->notify == &send_start)
         {
-          if (pos->prev == NULL)
-            h->connect_ready_head = pos->next;
-          else
-            pos->prev->next = pos->next;
-          if (pos->next != NULL)
-            pos->next->prev = pos->prev;
-          GNUNET_assert (pos->neighbour == NULL);
+         GNUNET_CONTAINER_DLL_remove (h->control_head,
+                                      h->control_tail,
+                                      pos);
+          if (GNUNET_SCHEDULER_NO_TASK != pos->notify_delay_task)
+            {
+              GNUNET_SCHEDULER_cancel (h->sched, pos->notify_delay_task);
+              pos->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+            }
           GNUNET_free (pos);
           break;
         }
       pos = pos->next;
     }
   schedule_control_transmit (h,
-                             sizeof (struct GNUNET_MessageHeader),
+                             sizeof (struct StartMessage),
                              GNUNET_YES,
-                             GNUNET_TIME_UNIT_FOREVER_REL, &send_start, NULL);
+                             GNUNET_TIME_UNIT_FOREVER_REL, &send_start, h);
   GNUNET_CLIENT_receive (h->client,
                          &demultiplexer, h, GNUNET_TIME_UNIT_FOREVER_REL);
 }
@@ -1090,6 +1225,8 @@ reconnect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 /**
  * Function that will schedule the job that will try
  * to connect us again to the client.
+ *
+ * @param h transport service to reconnect
  */
 static void
 schedule_reconnect (struct GNUNET_TRANSPORT_Handle *h)
@@ -1100,230 +1237,114 @@ schedule_reconnect (struct GNUNET_TRANSPORT_Handle *h)
               h->reconnect_delay.value);
 #endif
   GNUNET_assert (h->client == NULL);
-  GNUNET_assert (h->reconnect_task == GNUNET_SCHEDULER_NO_PREREQUISITE_TASK);
+  GNUNET_assert (h->reconnect_task == GNUNET_SCHEDULER_NO_TASK);
   h->reconnect_task
     = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                    GNUNET_NO,
-                                    GNUNET_SCHEDULER_PRIORITY_DEFAULT,
-                                    GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
                                     h->reconnect_delay, &reconnect, h);
-  h->reconnect_delay = GNUNET_TIME_UNIT_SECONDS;
-}
-
-
-/**
- * We are connected to the respective peer, check the
- * bandwidth limits and schedule the transmission.
- */
-static void schedule_request (struct GNUNET_TRANSPORT_TransmitHandle *th);
-
-
-/**
- * Function called by the scheduler when the timeout
- * for bandwidth availablility for the target
- * neighbour is reached.
- */
-static void
-transmit_ready (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
-{
-  struct GNUNET_TRANSPORT_TransmitHandle *th = cls;
-
-  th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-  schedule_request (th);
-}
-
-
-/**
- * Remove the given transmit handle from the wait list.  Does NOT free
- * it.
- */
-static void
-remove_from_wait_list (struct GNUNET_TRANSPORT_TransmitHandle *th)
-{
-  if (th->prev == NULL)
-    th->handle->connect_wait_head = th->next;
+  if (h->reconnect_delay.value == 0)
+    {
+      h->reconnect_delay = GNUNET_TIME_UNIT_MILLISECONDS;
+    }
   else
-    th->prev->next = th->next;
-  if (th->next != NULL)
-    th->next->prev = th->prev;
+    {
+      h->reconnect_delay = GNUNET_TIME_relative_multiply (h->reconnect_delay, 2);
+      h->reconnect_delay = GNUNET_TIME_relative_min (GNUNET_TIME_UNIT_SECONDS,
+                                                    h->reconnect_delay);
+    }
 }
 
 
 /**
- * We are connected to the respective peer, check the
- * bandwidth limits and schedule the transmission.
+ * Send request connect message to the service.
+ *
+ * @param cls the TransportRequestConnectMessage
+ * @param size number of bytes available in buf
+ * @param buf where to copy the message
+ * @return number of bytes copied to buf
  */
-static void
-schedule_request (struct GNUNET_TRANSPORT_TransmitHandle *th)
+static size_t
+send_transport_request_connect (void *cls, size_t size, void *buf)
 {
-  struct GNUNET_TRANSPORT_Handle *h;
-  struct GNUNET_TIME_Relative duration;
-  struct NeighbourList *n;
-  uint64_t available;
+  struct TransportRequestConnectMessage *trcm = cls;
 
-  h = th->handle;
-  n = th->neighbour;
-  if (th->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-    {
-      GNUNET_SCHEDULER_cancel (h->sched, th->notify_delay_task);
-      th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-    }
-  /* check outgoing quota */
-  duration = GNUNET_TIME_absolute_get_duration (n->last_quota_update);
-  if (duration.value > MIN_QUOTA_REFRESH_TIME)
-    {
-      update_quota (n);
-      duration = GNUNET_TIME_absolute_get_duration (n->last_quota_update);
-    }
-  available = duration.value * n->quota_out;
-  if (available < n->last_sent + th->notify_size)
+  if (buf == NULL)
     {
-      /* calculate how much bandwidth we'd still need to
-         accumulate and based on that how long we'll have
-         to wait... */
-      available = n->last_sent + th->notify_size - available;
-      duration = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                                available / n->quota_out);
-      if (th->timeout.value <
-          GNUNET_TIME_relative_to_absolute (duration).value)
-        {
-          /* signal timeout! */
-#if DEBUG_TRANSPORT
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Would need %llu ms before bandwidth is available for delivery to `%4s', that is too long.  Signaling timeout.\n",
-                      duration.value,
-                     GNUNET_i2s(&th->target));
-#endif
-          remove_from_wait_list (th);
-          th->notify (th->notify_cls, 0, NULL);
-          GNUNET_free (th);
-          return;
-        }
 #if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Need more bandwidth, delaying delivery to `%4s' by %llu ms\n",
-                 GNUNET_i2s(&th->target),
-                  duration.value);
+                  "Buffer null for %s\n",
+                  "REQUEST_CONNECT");
 #endif
-      th->notify_delay_task
-        = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                        GNUNET_NO,
-                                        GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                        GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                        duration, &transmit_ready, th);
-      return;
+      GNUNET_free (trcm);
+      return 0;
     }
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Bandwidth available for transmission to `%4s'\n",
-              GNUNET_i2s (&n->id));
-#endif
-  if (GNUNET_NO == n->transmit_ok)
-    {
-      /* we may be ready, but transport service is not;
-         wait for SendOkMessage or timeout */
-#if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Need to wait for transport service `%s' message\n",
-                  "SEND_OK");
-#endif
-      th->notify_delay_task
-        = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                        GNUNET_NO,
-                                        GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                        GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                        GNUNET_TIME_absolute_get_remaining
-                                        (th->timeout), &transmit_timeout, th);
-      return;
-    }
-  n->transmit_ok = GNUNET_NO;
-  remove_from_wait_list (th);
-#if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Moving message for `%4s' to ready list\n",
-             GNUNET_i2s(&n->id));
+              "Transmitting `%s' request for `%4s'.\n",
+              "REQUEST_CONNECT",
+              GNUNET_i2s (&trcm->peer));
 #endif
-  insert_transmit_handle (&h->connect_ready_head, th);
-  if (GNUNET_NO == h->transmission_scheduled)
-    schedule_transmission (h);
+  GNUNET_assert (size >= sizeof (struct TransportRequestConnectMessage));
+  memcpy(buf, trcm, sizeof(struct TransportRequestConnectMessage));
+  return sizeof(struct TransportRequestConnectMessage);
 }
 
+/**
+ * Create and send a request connect message to
+ * the transport service for a particular peer.
+ *
+ * @param h handle to the transport service
+ * @param n the neighbor to send the request connect message about
+ *
+ */
+static void send_request_connect_message(struct GNUNET_TRANSPORT_Handle *h, struct NeighbourList *n)
+{
+  struct TransportRequestConnectMessage *trcm;
+
+  trcm = GNUNET_malloc(sizeof(struct TransportRequestConnectMessage));
+  trcm->header.type = htons(GNUNET_MESSAGE_TYPE_TRANSPORT_REQUEST_CONNECT);
+  trcm->header.size = htons(sizeof(struct TransportRequestConnectMessage));
+  memcpy(&trcm->peer, &n->id, sizeof(struct GNUNET_PeerIdentity));
+  schedule_control_transmit (h,
+                             sizeof (struct TransportRequestConnectMessage),
+                             GNUNET_NO,
+                             GNUNET_TIME_UNIT_FOREVER_REL, &send_transport_request_connect, trcm);
+}
 
 /**
  * Add neighbour to our list
+ *
+ * @return NULL if this API is currently disconnecting from the service
  */
-static void
-add_neighbour (struct GNUNET_TRANSPORT_Handle *h,
-               uint32_t quota_out,
-               struct GNUNET_TIME_Relative latency,
+static struct NeighbourList *
+neighbour_add (struct GNUNET_TRANSPORT_Handle *h,
                const struct GNUNET_PeerIdentity *pid)
 {
   struct NeighbourList *n;
-  struct GNUNET_TRANSPORT_TransmitHandle *prev;
-  struct GNUNET_TRANSPORT_TransmitHandle *pos;
-  struct GNUNET_TRANSPORT_TransmitHandle *next;
 
+  if (GNUNET_YES == h->in_disconnect)
+    return NULL;
   /* check for duplicates */
-  if (NULL != find_neighbour (h, pid))
+  if (NULL != (n = neighbour_find (h, pid)))
     {
       GNUNET_break (0);
-      return;
+      return n;
     }
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Creating entry for new neighbour `%4s'.\n", GNUNET_i2s (pid));
+              "Creating entry for neighbour `%4s'.\n",
+             GNUNET_i2s (pid));
 #endif
   n = GNUNET_malloc (sizeof (struct NeighbourList));
   n->id = *pid;
-  n->last_quota_update = GNUNET_TIME_absolute_get ();
-  n->quota_out = quota_out;
+  GNUNET_BANDWIDTH_tracker_init (&n->out_tracker,
+                                GNUNET_CONSTANTS_DEFAULT_BW_IN_OUT,
+                                MAX_BANDWIDTH_CARRY_S);
   n->next = h->neighbours;
-  n->transmit_ok = GNUNET_YES;
+  n->h = h;
   h->neighbours = n;
-  if (h->nc_cb != NULL)
-    h->nc_cb (h->cls, &n->id, latency);
-  prev = NULL;
-  pos = h->connect_wait_head;
-  while (pos != NULL)
-    {
-      next = pos->next;
-      if (0 == memcmp (pid,
-                       &pos->target, sizeof (struct GNUNET_PeerIdentity)))
-        {
-          pos->neighbour = n;
-          GNUNET_assert (NULL == n->transmit_handle);
-          n->transmit_handle = pos;
-          if (prev == NULL)
-            h->connect_wait_head = next;
-          else
-            prev->next = next;
-          if (GNUNET_YES == n->received_ack)
-            {
-#if DEBUG_TRANSPORT
-             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                         "Found pending request for `%4s' will trigger it now.\n",
-                         GNUNET_i2s (&pos->target));
-#endif
-             if (pos->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-               {
-                 GNUNET_SCHEDULER_cancel (h->sched, pos->notify_delay_task);
-                 pos->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-               }
-              schedule_request (pos);
-            }
-          else
-            {
-#if DEBUG_TRANSPORT
-             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                         "Found pending request for `%4s' but still need `%s' before proceeding.\n",
-                         GNUNET_i2s (&pos->target),
-                         "ACK");
-#endif
-            }
-          break;
-        }
-      prev = pos;
-      pos = next;
-    }
+
+
+  return n;
 }
 
 
@@ -1331,17 +1352,19 @@ add_neighbour (struct GNUNET_TRANSPORT_Handle *h,
  * Connect to the transport service.  Note that the connection may
  * complete (or fail) asynchronously.
  *
-
  * @param sched scheduler to use
  * @param cfg configuration to use
+ * @param self our own identity (API should check that it matches
+ *             the identity found by transport), or NULL (no check)
  * @param cls closure for the callbacks
  * @param rec receive function to call
  * @param nc function to call on connect events
- * @param dc function to call on disconnect events
+ * @param nd function to call on disconnect events
  */
 struct GNUNET_TRANSPORT_Handle *
 GNUNET_TRANSPORT_connect (struct GNUNET_SCHEDULER_Handle *sched,
-                          struct GNUNET_CONFIGURATION_Handle *cfg,
+                          const struct GNUNET_CONFIGURATION_Handle *cfg,
+                         const struct GNUNET_PeerIdentity *self,
                           void *cls,
                           GNUNET_TRANSPORT_ReceiveCallback rec,
                           GNUNET_TRANSPORT_NotifyConnect nc,
@@ -1349,11 +1372,12 @@ GNUNET_TRANSPORT_connect (struct GNUNET_SCHEDULER_Handle *sched,
 {
   struct GNUNET_TRANSPORT_Handle *ret;
 
-  GNUNET_ARM_start_service ("peerinfo",
-                            cfg, sched, START_SERVICE_TIMEOUT, NULL, NULL);
-  GNUNET_ARM_start_service ("transport",
-                            cfg, sched, START_SERVICE_TIMEOUT, NULL, NULL);
   ret = GNUNET_malloc (sizeof (struct GNUNET_TRANSPORT_Handle));
+  if (self != NULL)
+    {
+      ret->self = *self;
+      ret->check_self = GNUNET_YES;
+    }
   ret->sched = sched;
   ret->cfg = cfg;
   ret->cls = cls;
@@ -1366,23 +1390,6 @@ GNUNET_TRANSPORT_connect (struct GNUNET_SCHEDULER_Handle *sched,
 }
 
 
-/**
- * These stop activities must be run in a fresh
- * scheduler that is NOT in shutdown mode.
- */
-static void
-stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
-{
-  struct GNUNET_TRANSPORT_Handle *handle = cls;
-  GNUNET_ARM_stop_service ("transport",
-                           handle->cfg,
-                           tc->sched, STOP_SERVICE_TIMEOUT, NULL, NULL);
-  GNUNET_ARM_stop_service ("peerinfo",
-                           handle->cfg,
-                           tc->sched, STOP_SERVICE_TIMEOUT, NULL, NULL);
-}
-
-
 /**
  * Disconnect from the transport service.
  */
@@ -1393,51 +1400,87 @@ GNUNET_TRANSPORT_disconnect (struct GNUNET_TRANSPORT_Handle *handle)
   struct NeighbourList *n;
   struct HelloWaitList *hwl;
   struct GNUNET_CLIENT_Connection *client;
+  struct ControlMessage *cm;
 
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Transport disconnect called!\n");
 #endif
-  while (NULL != (th = handle->connect_ready_head))
-    {
-      handle->connect_ready_head = th->next;
-      th->notify (th->notify_cls, 0, NULL);
-      GNUNET_free (th);
-    }
-  while (NULL != (th = handle->connect_wait_head))
-    {
-      handle->connect_wait_head = th->next;
-      if (th->notify_delay_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
-        {
-          GNUNET_SCHEDULER_cancel (handle->sched, th->notify_delay_task);
-          th->notify_delay_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-        }
-      th->notify (th->notify_cls, 0, NULL);
-      GNUNET_free (th);
-    }
+  handle->in_disconnect = GNUNET_YES;
   while (NULL != (n = handle->neighbours))
     {
       handle->neighbours = n->next;
+      switch (n->transmit_stage)
+       {
+       case TS_NEW:
+       case TS_TRANSMITTED:
+         /* nothing to do */
+         break;
+       case TS_QUEUED:
+       case TS_TRANSMITTED_QUEUED:
+         th = &n->transmit_handle;
+         if (th->notify_delay_task != GNUNET_SCHEDULER_NO_TASK)
+           {
+             GNUNET_SCHEDULER_cancel (handle->sched,
+                                      th->notify_delay_task);
+             th->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+           }
+         GNUNET_assert (0 == th->notify (th->notify_cls, 0, NULL));
+         break;
+       default:
+         GNUNET_break (0);
+       }
       GNUNET_free (n);
     }
   while (NULL != (hwl = handle->hwl_head))
     {
       handle->hwl_head = hwl->next;
-      GNUNET_SCHEDULER_cancel (handle->sched, hwl->task);
       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                  _("Disconnect while trying to obtain `%s' from transport service.\n"),
-                 "HELLO");
+                  _
+                  ("Disconnect while notification for `%s' still registered.\n"),
+                  "HELLO");
       if (hwl->rec != NULL)
-        hwl->rec (hwl->rec_cls, GNUNET_TIME_UNIT_ZERO, NULL, NULL);
+        hwl->rec (hwl->rec_cls, NULL);
       GNUNET_free (hwl);
     }
-  if (handle->reconnect_task != GNUNET_SCHEDULER_NO_PREREQUISITE_TASK)
+
+  /* Check for still scheduled control messages, cancel delay tasks if so */
+  /* Added because somehow a notify_delay_task is remaining scheduled and is ever so annoying */
+  while ( (NULL != (cm = handle->control_head)))
+    {
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Disconnect before control message sent!\n");
+#endif
+      if (cm->notify_delay_task != GNUNET_SCHEDULER_NO_TASK)
+        {
+          GNUNET_SCHEDULER_cancel (handle->sched, cm->notify_delay_task);
+          cm->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+        }
+      GNUNET_CONTAINER_DLL_remove (handle->control_head,
+                                   handle->control_tail,
+                                   cm);
+      GNUNET_free (cm);
+    }
+  /* end check */
+
+  if (handle->reconnect_task != GNUNET_SCHEDULER_NO_TASK)
     {
       GNUNET_SCHEDULER_cancel (handle->sched, handle->reconnect_task);
-      handle->reconnect_task = GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
+      handle->reconnect_task = GNUNET_SCHEDULER_NO_TASK;
+    }
+  if (handle->quota_task != GNUNET_SCHEDULER_NO_TASK)
+    {
+      GNUNET_SCHEDULER_cancel (handle->sched, handle->quota_task);
+      handle->quota_task = GNUNET_SCHEDULER_NO_TASK;
     }
   GNUNET_free_non_null (handle->my_hello);
   handle->my_hello = NULL;
-  GNUNET_SCHEDULER_run (&stop_task, handle);
+
+  if (NULL != handle->network_handle)
+    {
+      GNUNET_CLIENT_notify_transmit_ready_cancel (handle->network_handle);
+      handle->network_handle = NULL;
+    }
   if (NULL != (client = handle->client))
     {
 #if DEBUG_TRANSPORT
@@ -1445,18 +1488,16 @@ GNUNET_TRANSPORT_disconnect (struct GNUNET_TRANSPORT_Handle *handle)
                   "Disconnecting from transport service for good.\n");
 #endif
       handle->client = NULL;
-      GNUNET_CLIENT_disconnect (client);
+      GNUNET_CLIENT_disconnect (client, GNUNET_NO);
     }
-  if (client == NULL)
-    GNUNET_free (handle);
+  GNUNET_free (handle);
 }
 
 
 /**
- * Type of a function to call when we receive a message
- * from the service.
+ * Function we use for handling incoming messages.
  *
- * @param cls closure
+ * @param cls closure (struct GNUNET_TRANSPORT_Handle *)
  * @param msg message received, NULL on timeout or fatal error
  */
 static void
@@ -1469,47 +1510,32 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
   const struct GNUNET_MessageHeader *imm;
   const struct SendOkMessage *okm;
   struct HelloWaitList *hwl;
+  struct HelloWaitList *next_hwl;
   struct NeighbourList *n;
-  struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pkey;
   struct GNUNET_PeerIdentity me;
-  struct GNUNET_TRANSPORT_TransmitHandle *th;
   uint16_t size;
 
-  if ((msg == NULL) || (h->client == NULL))
+  if (h->client == NULL)
+    {
+      /* shutdown initiated from 'GNUNET_TRANSPORT_disconnect',
+        finish clean up work! */
+      GNUNET_free (h);
+      return;
+    }
+  if (msg == NULL)
     {
-      if (h->client != NULL)
-        {
 #if DEBUG_TRANSPORT
-          GNUNET_log (GNUNET_ERROR_TYPE_INFO,
-                      "Error receiving from transport service, disconnecting temporarily.\n");
+      GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+                 "Error receiving from transport service, disconnecting temporarily.\n");
 #endif
-          if (h->network_handle != NULL)
-            {
-              GNUNET_NETWORK_notify_transmit_ready_cancel (h->network_handle);
-              h->network_handle = NULL;
-              h->transmission_scheduled = GNUNET_NO;
-             th = h->connect_ready_head;
-             /* add timeout again, we cancelled the transmit_ready task! */
-             GNUNET_assert (th->notify_delay_task == GNUNET_SCHEDULER_NO_PREREQUISITE_TASK);
-             th->notify_delay_task 
-               = GNUNET_SCHEDULER_add_delayed (h->sched,
-                                               GNUNET_NO,
-                                               GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                               GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                               GNUNET_TIME_absolute_get_remaining(th->timeout),
-                                               &transmit_timeout, 
-                                               th);    
-            }
-          GNUNET_CLIENT_disconnect (h->client);
-          h->client = NULL;
-          schedule_reconnect (h);
-        }
-      else
-        {
-          /* shutdown initiated from 'GNUNET_TRANSPORT_disconnect',
-             finish clean up work! */
-          GNUNET_free (h);
-        }
+      if (h->network_handle != NULL)
+       {
+         GNUNET_CLIENT_notify_transmit_ready_cancel (h->network_handle);
+         h->network_handle = NULL;
+       }
+      GNUNET_CLIENT_disconnect (h->client, GNUNET_NO);
+      h->client = NULL;
+      schedule_reconnect (h);
       return;
     }
   GNUNET_CLIENT_receive (h->client,
@@ -1519,16 +1545,12 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
     {
     case GNUNET_MESSAGE_TYPE_HELLO:
       if (GNUNET_OK !=
-          GNUNET_HELLO_get_key ((const struct GNUNET_HELLO_Message *) msg,
-                                &pkey))
+          GNUNET_HELLO_get_id ((const struct GNUNET_HELLO_Message *) msg,
+                               &me))
         {
           GNUNET_break (0);
           break;
         }
-      GNUNET_CRYPTO_hash (&pkey,
-                          sizeof (struct
-                                  GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded),
-                          &me.hashPubKey);
 #if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Receiving (my own) `%s' message, I am `%4s'.\n",
@@ -1543,14 +1565,13 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
         }
       h->my_hello = GNUNET_malloc (size);
       memcpy (h->my_hello, msg, size);
-      while (NULL != (hwl = h->hwl_head))
+      hwl = h->hwl_head;
+      while (NULL != hwl)
         {
-          h->hwl_head = hwl->next;
-          GNUNET_SCHEDULER_cancel (h->sched, hwl->task);
-          GNUNET_TRANSPORT_get_hello (h,
-                                      GNUNET_TIME_UNIT_ZERO,
-                                      hwl->rec, hwl->rec_cls);
-          GNUNET_free (hwl);
+         next_hwl = hwl->next;
+          hwl->rec (hwl->rec_cls,
+                   (const struct GNUNET_MessageHeader *) h->my_hello);
+         hwl = next_hwl;
         }
       break;
     case GNUNET_MESSAGE_TYPE_TRANSPORT_CONNECT:
@@ -1565,9 +1586,18 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
                   "Receiving `%s' message for `%4s'.\n",
                   "CONNECT", GNUNET_i2s (&cim->id));
 #endif
-      add_neighbour (h,
-                     ntohl (cim->quota_out),
-                     GNUNET_TIME_relative_ntoh (cim->latency), &cim->id);
+      n = neighbour_find (h, &cim->id);
+      if (n == NULL)
+       n = neighbour_add (h,
+                          &cim->id);
+      if (n == NULL)
+       return;
+      GNUNET_break (n->is_connected == GNUNET_NO);
+      n->is_connected = GNUNET_YES;
+      if (h->nc_cb != NULL)
+       h->nc_cb (h->cls, &n->id,
+                 GNUNET_TIME_relative_ntoh (cim->latency),
+                 ntohl (cim->distance));
       break;
     case GNUNET_MESSAGE_TYPE_TRANSPORT_DISCONNECT:
       if (size != sizeof (struct DisconnectInfoMessage))
@@ -1576,12 +1606,16 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
           break;
         }
       dim = (const struct DisconnectInfoMessage *) msg;
-#if DEBUG_TRANSPORT
+#if DEBUG_TRANSPORT_DISCONNECT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Receiving `%s' message for `%4s'.\n",
-                  "DISCONNECT", GNUNET_i2s (&dim->peer));
+                  "DISCONNECT",
+                 GNUNET_i2s (&dim->peer));
 #endif
-      remove_neighbour (h, &dim->peer);
+      n = neighbour_find (h, &dim->peer);
+      GNUNET_break (n != NULL);
+      if (n != NULL)
+       neighbour_disconnect (n);       
       break;
     case GNUNET_MESSAGE_TYPE_TRANSPORT_SEND_OK:
       if (size != sizeof (struct SendOkMessage))
@@ -1593,24 +1627,28 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
 #if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Receiving `%s' message, transmission %s.\n", "SEND_OK",
-                 ntohl(okm->success) == GNUNET_OK ? "succeeded" : "failed");
+                  ntohl (okm->success) == GNUNET_OK ? "succeeded" : "failed");
 #endif
-      n = find_neighbour (h, &okm->peer);
+      n = neighbour_find (h, &okm->peer);
       GNUNET_assert (n != NULL);
-      n->transmit_ok = GNUNET_YES;
-      if (n->transmit_handle != NULL)
-        {
-#if DEBUG_TRANSPORT
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Processing pending message\n");
-#endif
-          GNUNET_SCHEDULER_cancel (h->sched,
-                                   n->transmit_handle->notify_delay_task);
-          n->transmit_handle->notify_delay_task =
-            GNUNET_SCHEDULER_NO_PREREQUISITE_TASK;
-          GNUNET_assert (GNUNET_YES == n->received_ack);
-          schedule_request (n->transmit_handle);
-        }
+      switch (n->transmit_stage)
+       {
+       case TS_NEW:
+         GNUNET_break (0);
+         break;
+       case TS_QUEUED:
+         GNUNET_break (0);
+         break;
+       case TS_TRANSMITTED:
+         n->transmit_stage = TS_NEW;
+         break;
+       case TS_TRANSMITTED_QUEUED:
+         n->transmit_stage = TS_QUEUED;
+         schedule_transmission (h);
+         break;
+       default:
+         GNUNET_break (0);
+       }
       break;
     case GNUNET_MESSAGE_TYPE_TRANSPORT_RECV:
 #if DEBUG_TRANSPORT
@@ -1631,44 +1669,25 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
           GNUNET_break (0);
           break;
         }
-      switch (ntohs (imm->type))
-        {
-        case GNUNET_MESSAGE_TYPE_TRANSPORT_ACK:
 #if DEBUG_TRANSPORT
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Receiving `%s' message from `%4s'.\n",
-                      "ACK", GNUNET_i2s (&im->peer));
-#endif
-          n = find_neighbour (h, &im->peer);
-          if (n == NULL)
-            {
-              GNUNET_break (0);
-              break;
-            }
-          if (n->received_ack == GNUNET_NO)
-            {
-              n->received_ack = GNUNET_YES;
-              if (NULL != n->transmit_handle)
-                {
-#if DEBUG_TRANSPORT
-                  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                              "Peer connected, scheduling delayed message for deliverery now.\n");
-#endif
-                  schedule_request (n->transmit_handle);
-                }
-            }
-          break;
-        default:
-#if DEBUG_TRANSPORT
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Received message of type %u from `%4s'.\n",
-                      ntohs (imm->type), GNUNET_i2s (&im->peer));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                 "Received message of type %u from `%4s'.\n",
+                 ntohs (imm->type), GNUNET_i2s (&im->peer));
 #endif
-          if (h->rec != NULL)
-            h->rec (h->cls,
-                    GNUNET_TIME_relative_ntoh (im->latency), &im->peer, imm);
-          break;
-        }
+      n = neighbour_find (h, &im->peer);
+      if (n == NULL)
+       {
+         GNUNET_break (0);
+         break;
+       }
+      if (n->is_connected != GNUNET_YES)
+       {
+         GNUNET_break (0);
+         break;
+       }
+      if (h->rec != NULL)
+       h->rec (h->cls, &im->peer, imm,
+               GNUNET_TIME_relative_ntoh (im->latency), ntohl(im->distance));
       break;
     default:
       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
@@ -1681,70 +1700,58 @@ demultiplexer (void *cls, const struct GNUNET_MessageHeader *msg)
 }
 
 
-struct ClientTransmitWrapper
-{
-  GNUNET_NETWORK_TransmitReadyNotify notify;
-  void *notify_cls;
-  struct GNUNET_TRANSPORT_TransmitHandle *th;
-};
-
-
 /**
- * Transmit message of a client destined for another
- * peer to the service.
+ * Called when our transmit request timed out before any transport
+ * reported success connecting to the desired peer or before the
+ * transport was ready to receive.  Signal error and free
+ * TransmitHandle.
+ *
+ * @param cls the 'struct GNUNET_TRANSPORT_TransmitHandle*' that is timing out
+ * @param tc scheduler context
  */
-static size_t
-client_notify_wrapper (void *cls, size_t size, void *buf)
+static void
+peer_transmit_timeout (void *cls,
+                      const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct ClientTransmitWrapper *ctw = cls;
-  struct OutboundMessage *obm;
-  struct GNUNET_MessageHeader *hdr;
-  size_t ret;
+  struct GNUNET_TRANSPORT_TransmitHandle *th = cls;
+  struct NeighbourList *n;
+  GNUNET_CONNECTION_TransmitReadyNotify notify;
+  void *notify_cls;
 
-  if (size == 0)
-    {
+  th->notify_delay_task = GNUNET_SCHEDULER_NO_TASK;
+  n = th->neighbour;
 #if DEBUG_TRANSPORT
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Transmission request could not be satisfied.\n");
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+             "Triggering timeout for request to transmit to `%4s' (%d)\n",
+             GNUNET_i2s (&n->id),
+             n->transmit_stage);
 #endif
-      ret = ctw->notify (ctw->notify_cls, 0, NULL);
-      GNUNET_assert (ret == 0);
-      GNUNET_free (ctw);
-      return 0;
-    }
-  GNUNET_assert (size >= sizeof (struct OutboundMessage));
-  obm = buf;
-  ret = ctw->notify (ctw->notify_cls,
-                     size - sizeof (struct OutboundMessage),
-                     (void *) &obm[1]);
-  if (ret == 0)
+  notify = th->notify;
+  notify_cls = th->notify_cls;
+  switch (n->transmit_stage)
     {
-      /* Need to reset flag, no SEND means no SEND_OK! */
-      ctw->th->neighbour->transmit_ok = GNUNET_YES;
-      GNUNET_free (ctw);
-      return 0;
+    case TS_NEW:
+      GNUNET_break (0);
+      break;
+    case TS_QUEUED:
+      n->transmit_stage = TS_NEW;
+      if (n->is_connected == GNUNET_NO)
+       neighbour_free (n);
+      break;
+    case TS_TRANSMITTED:
+      GNUNET_break (0);
+      break;
+    case TS_TRANSMITTED_QUEUED:
+      n->transmit_stage = TS_TRANSMITTED;
+      break;
+    default:
+      GNUNET_break (0);
     }
-  GNUNET_assert (ret >= sizeof (struct GNUNET_MessageHeader));
-  hdr = (struct GNUNET_MessageHeader *) &obm[1];
-  GNUNET_assert (ntohs (hdr->size) == ret);
-  GNUNET_assert (ret + sizeof (struct OutboundMessage) <
-                 GNUNET_SERVER_MAX_MESSAGE_SIZE);
-#if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Transmitting `%s' message with data for `%4s'\n",
-              "SEND", GNUNET_i2s (&ctw->th->target));
-#endif
-  ret += sizeof (struct OutboundMessage);
-  obm->header.type = htons (GNUNET_MESSAGE_TYPE_TRANSPORT_SEND);
-  obm->header.size = htons (ret);
-  obm->reserved = htonl (0);
-  obm->peer = ctw->th->target;
-  GNUNET_free (ctw);
-  return ret;
+  if (NULL != notify)
+    notify (notify_cls, 0, NULL);
 }
 
 
-
 /**
  * Check if we could queue a message of the given size for
  * transmission.  The transport service will take both its
@@ -1754,6 +1761,7 @@ client_notify_wrapper (void *cls, size_t size, void *buf)
  * @param handle connection to transport service
  * @param target who should receive the message
  * @param size how big is the message we want to transmit?
+ * @param priority how important is the message?
  * @param timeout after how long should we give up (and call
  *        notify with buf NULL and size 0)?
  * @param notify function to call when we are ready to
@@ -1768,117 +1776,127 @@ GNUNET_TRANSPORT_notify_transmit_ready (struct GNUNET_TRANSPORT_Handle
                                         *handle,
                                         const struct GNUNET_PeerIdentity
                                         *target, size_t size,
+                                        unsigned int priority,
                                         struct GNUNET_TIME_Relative timeout,
-                                        GNUNET_NETWORK_TransmitReadyNotify
+                                        GNUNET_CONNECTION_TransmitReadyNotify
                                         notify, void *notify_cls)
 {
-  struct GNUNET_TRANSPORT_TransmitHandle *pos;
   struct GNUNET_TRANSPORT_TransmitHandle *th;
   struct NeighbourList *n;
-  struct ClientTransmitWrapper *ctw;
 
   if (size + sizeof (struct OutboundMessage) >=
       GNUNET_SERVER_MAX_MESSAGE_SIZE)
     {
+#if DEBUG_TRANSPORT
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Message size is %d, max allowed is %d.\n",
+                  size + sizeof (struct OutboundMessage), GNUNET_SERVER_MAX_MESSAGE_SIZE - 1);
+#endif
       GNUNET_break (0);
       return NULL;
     }
-#if DEBUG_TRANSPORT
+
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-              "Asking transport service for transmission of %u bytes to peer `%4s'.\n",
-              size, GNUNET_i2s (target));
-#endif
-  n = find_neighbour (handle, target);
-  ctw = GNUNET_malloc (sizeof (struct ClientTransmitWrapper));
-  th = GNUNET_malloc (sizeof (struct GNUNET_TRANSPORT_TransmitHandle));
-  ctw->notify = notify;
-  ctw->notify_cls = notify_cls;
-  ctw->th = th;
-  th->handle = handle;
-  th->target = *target;
-  th->notify = &client_notify_wrapper;
-  th->notify_cls = ctw;
-  th->notify_size = size + sizeof (struct OutboundMessage);
-  th->timeout = GNUNET_TIME_relative_to_absolute (timeout);
-  th->neighbour = n;
-  if (NULL == n)
+              "Asking transport service for transmission of %u bytes to peer `%4s' within %llu ms.\n",
+              size, GNUNET_i2s (target),
+             (unsigned long long) timeout.value);
+
+  n = neighbour_find (handle, target);
+  if (n == NULL)
     {
-      pos = handle->connect_wait_head;
-      while (pos != NULL)
-        {
-          GNUNET_assert (0 != memcmp (target,
-                                      &pos->target,
-                                      sizeof (struct GNUNET_PeerIdentity)));
-          pos = pos->next;
-        }
-#if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  "Will now try to connect to `%4s'.\n", GNUNET_i2s (target));
-#endif
-      try_connect (th);
-      return th;
+                  "Created neighbour entry for peer `%s'\n",
+                  GNUNET_i2s (target));
+      n = neighbour_add (handle, target);
+
     }
 
-#if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Transmission request queued for transmission to transport service.\n");
-#endif
-  GNUNET_assert (NULL == n->transmit_handle);
-  n->transmit_handle = th;
-  if (GNUNET_YES != n->received_ack)
+  /**
+   *  Send a request connect message if not connected,
+   *  otherwise we will never send anything to
+   *  transport service
+   */
+  if (n->is_connected == GNUNET_NO)
+    {
+      send_request_connect_message(handle, n);
+    }
+
+  if (n == NULL)
     {
-#if DEBUG_TRANSPORT
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                 "Connection to `%4s' is not yet confirmed connected, scheduling timeout (%llu ms) only.\n",
-                 GNUNET_i2s (target), timeout.value);
-#endif
-      th->notify_delay_task
-       = GNUNET_SCHEDULER_add_delayed (handle->sched,
-                                       GNUNET_NO,
-                                       GNUNET_SCHEDULER_PRIORITY_KEEP,
-                                       GNUNET_SCHEDULER_NO_PREREQUISITE_TASK,
-                                       timeout, &transmit_timeout, th);
-      return th;
+                 "Could not create neighbour entry for peer `%s'\n",
+                 GNUNET_i2s (target));
+      return NULL;
     }
-  
-#if DEBUG_TRANSPORT
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Peer `%4s' is ready to receive, scheduling message for delivery now.\n",
-             GNUNET_i2s (target));
-#endif
-  schedule_request (th);
+  switch (n->transmit_stage)
+    {
+    case TS_NEW:
+      n->transmit_stage = TS_QUEUED;
+      break;
+    case TS_QUEUED:
+      GNUNET_break (0);
+      return NULL;
+    case TS_TRANSMITTED:
+      n->transmit_stage = TS_TRANSMITTED_QUEUED;
+      break;
+    case TS_TRANSMITTED_QUEUED:
+      GNUNET_break (0);
+      return NULL;
+    default:
+      GNUNET_break (0);
+      return NULL;
+    }
+  th = &n->transmit_handle;
+  th->neighbour = n;
+  th->notify = notify;
+  th->notify_cls = notify_cls;
+  th->timeout = GNUNET_TIME_relative_to_absolute (timeout);
+  th->notify_size = size + sizeof (struct OutboundMessage);
+  th->priority = priority;
+  th->notify_delay_task
+    = GNUNET_SCHEDULER_add_delayed (handle->sched, timeout,
+                                    &peer_transmit_timeout, th);
+  schedule_transmission (handle);
   return th;
 }
 
 
 /**
- * Cancel the specified transmission-ready
- * notification.
+ * Cancel the specified transmission-ready notification.
  */
 void
 GNUNET_TRANSPORT_notify_transmit_ready_cancel (struct
                                                GNUNET_TRANSPORT_TransmitHandle
                                                *th)
 {
-  struct GNUNET_TRANSPORT_Handle *h;
+  struct NeighbourList *n;
 
+  n = th->neighbour;
 #if DEBUG_TRANSPORT
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-             "Transmission request of %u bytes to `%4s' was cancelled.\n",
-             th->notify_size - sizeof(struct OutboundMessage),
-             GNUNET_i2s (&th->target));
+              "Transmission request of %u bytes to `%4s' was cancelled.\n",
+              th->notify_size - sizeof (struct OutboundMessage),
+              GNUNET_i2s (&n->id));
 #endif
-  GNUNET_assert (th->notify == &client_notify_wrapper);
-  remove_from_any_list (th);
-  h = th->handle;
-  if ((h->connect_ready_head == NULL) && (h->network_handle != NULL))
+  switch (n->transmit_stage)
     {
-      GNUNET_NETWORK_notify_transmit_ready_cancel (h->network_handle);
-      h->network_handle = NULL;
-      h->transmission_scheduled = GNUNET_NO;
+    case TS_NEW:
+      GNUNET_break (0);
+      break;
+    case TS_QUEUED:
+      n->transmit_stage = TS_NEW;
+      if (n->is_connected == GNUNET_NO)
+       neighbour_free (n);
+      break;
+    case TS_TRANSMITTED:
+      GNUNET_break (0);
+      break;
+    case TS_TRANSMITTED_QUEUED:
+      n->transmit_stage = TS_TRANSMITTED;
+      break;
+    default:
+      GNUNET_break (0);
     }
-  GNUNET_free (th->notify_cls);
-  GNUNET_free (th);
 }