clique topology
[oweals/gnunet.git] / src / testbed / gnunet-service-testbed.c
index b26bd9d97fdcb0821f87f0b0b81096ea294513c3..c76f75ca8df262c1087ed9af05673897e0d758e5 100644 (file)
@@ -159,6 +159,115 @@ struct Route
 struct LinkControllersContext;
 
 
+/**
+ * A DLL of host registrations to be made
+ */
+struct HostRegistration
+{
+  /**
+   * next registration in the DLL
+   */
+  struct HostRegistration *next;
+
+  /**
+   * previous registration in the DLL
+   */
+  struct HostRegistration *prev;
+
+  /**
+   * The callback to call after this registration's status is available
+   */
+  GNUNET_TESTBED_HostRegistrationCompletion cb;
+
+  /**
+   * The closure for the above callback
+   */
+  void *cb_cls;
+
+  /**
+   * The host that has to be registered
+   */
+  struct GNUNET_TESTBED_Host *host;
+};
+
+
+/**
+ * This context information will be created for each host that is registered at
+ * slave controllers during overlay connects.
+ */
+struct RegisteredHostContext
+{
+  /**
+   * The host which is being registered
+   */
+  struct GNUNET_TESTBED_Host *reg_host;
+
+  /**
+   * The host of the controller which has to connect to the above rhost
+   */
+  struct GNUNET_TESTBED_Host *host;
+
+  /**
+   * The gateway to which this operation is forwarded to
+   */
+  struct Slave *gateway;
+
+  /**
+   * The gateway through which peer2's controller can be reached
+   */
+  struct Slave *gateway2;
+
+  /**
+   * Handle for sub-operations
+   */
+  struct GNUNET_TESTBED_Operation *sub_op;
+
+  /**
+   * The client which initiated the link controller operation
+   */
+  struct GNUNET_SERVER_Client *client;
+
+  /**
+   * Head of the ForwardedOverlayConnectContext DLL
+   */
+  struct ForwardedOverlayConnectContext *focc_dll_head;
+
+  /**
+   * Tail of the ForwardedOverlayConnectContext DLL
+   */
+  struct ForwardedOverlayConnectContext *focc_dll_tail;
+  
+  /**
+   * Enumeration of states for this context
+   */
+  enum RHCState {
+
+    /**
+     * The initial state
+     */
+    RHC_INIT = 0,
+
+    /**
+     * State where we attempt to get peer2's controller configuration
+     */
+    RHC_GET_CFG,
+
+    /**
+     * State where we attempt to link the controller of peer 1 to the controller
+     * of peer2
+     */
+    RHC_LINK,
+
+    /**
+     * State where we attempt to do the overlay connection again
+     */
+    RHC_OL_CONNECT
+    
+  } state;
+
+};
+
+
 /**
  * Structure representing a connected(directly-linked) controller
  */
@@ -185,10 +294,31 @@ struct Slave
    */
   struct LinkControllersContext *lcc;
 
+  /**
+   * Head of the host registration DLL
+   */
+  struct HostRegistration *hr_dll_head;
+
+  /**
+   * Tail of the host registration DLL
+   */
+  struct HostRegistration *hr_dll_tail;
+
+  /**
+   * The current host registration handle
+   */
+  struct GNUNET_TESTBED_HostRegistrationHandle *rhandle;
+
+  /**
+   * Hashmap to hold Registered host contexts
+   */
+  struct GNUNET_CONTAINER_MultiHashMap *reghost_map;
+
   /**
    * The id of the host this controller is running on
    */
   uint32_t host_id;
+
 };
 
 
@@ -240,9 +370,9 @@ struct LCFContext
   struct GNUNET_SERVER_Client *client;
 
   /**
-   * The host registration handle while registered hosts in this context
+   * Handle for operations which are forwarded while linking controllers
    */
-  struct GNUNET_TESTBED_HostRegistrationHandle *rhandle;
+  struct ForwardedOperationContext *fopc;
 
   /**
    * The id of the operation which created this context
@@ -294,6 +424,7 @@ struct LCFContextQueue
  */
 struct Peer
 {
+  
   union
   {
     struct
@@ -319,9 +450,9 @@ struct Peer
     struct
     {
       /**
-       * The controller this peer is started through
+       * The slave this peer is started through
        */
-      struct GNUNET_TESTBED_Controller *controller;
+      struct Slave *slave;
 
       /**
        * The id of the remote host this peer is running on
@@ -342,6 +473,24 @@ struct Peer
    */
   uint32_t id;
 
+  /**
+   * References to peers are using in forwarded overlay contexts and remote
+   * overlay connect contexts. A peer can only be destroyed after all such
+   * contexts are destroyed. For this, we maintain a reference counter. When we
+   * use a peer in any such context, we increment this counter. We decrement it
+   * when we are destroying these contexts
+   */
+  uint32_t reference_cnt;
+
+  /**
+   * While destroying a peer, due to the fact that there could be references to
+   * this peer, we delay the peer destroy to a further time. We do this by using
+   * this flag to destroy the peer while destroying a context in which this peer
+   * has been used. When the flag is set to 1 and reference_cnt = 0 we destroy
+   * the peer
+   */
+  uint32_t destroy_flag;
+
 };
 
 
@@ -350,6 +499,16 @@ struct Peer
  */
 struct OverlayConnectContext
 {
+  /**
+   * The next pointer for maintaining a DLL
+   */
+  struct OverlayConnectContext *next;
+
+  /**
+   * The prev pointer for maintaining a DLL
+   */
+  struct OverlayConnectContext *prev;
+  
   /**
    * The client which has requested for overlay connection
    */
@@ -426,6 +585,11 @@ struct OverlayConnectContext
    */
   GNUNET_SCHEDULER_TaskIdentifier timeout_task;
 
+  /**
+   * The id of the cleanup task
+   */
+  GNUNET_SCHEDULER_TaskIdentifier cleanup_task;
+
   /**
    * The id of peer A
    */
@@ -452,6 +616,16 @@ struct OverlayConnectContext
  */
 struct RequestOverlayConnectContext
 {
+  /**
+   * the next pointer for DLL
+   */
+  struct RequestOverlayConnectContext *next;
+
+  /**
+   * the prev pointer for DLL
+   */
+  struct RequestOverlayConnectContext *prev;
+
   /**
    * The transport handle of peer B
    */
@@ -491,6 +665,16 @@ struct RequestOverlayConnectContext
  */
 struct ForwardedOperationContext
 {
+  /**
+   * The next pointer for DLL
+   */
+  struct ForwardedOperationContext *next;
+
+  /**
+   * The prev pointer for DLL
+   */
+  struct ForwardedOperationContext *prev;
+  
   /**
    * The generated operation context
    */
@@ -516,6 +700,11 @@ struct ForwardedOperationContext
    */
   uint64_t operation_id;
 
+  /**
+   * The type of the operation which is forwarded
+   */
+  enum OperationType type;
+
 };
 
 
@@ -544,24 +733,14 @@ struct LinkControllersContext
 struct ForwardedOverlayConnectContext
 {
   /**
-   * The gateway controller to which this operation is forwarded to
+   * next ForwardedOverlayConnectContext in the DLL
    */
-  struct GNUNET_TESTBED_Controller *gateway;
+  struct ForwardedOverlayConnectContext *next;
 
   /**
-   * The gateway controller through which peer2's controller can be reached
+   * previous ForwardedOverlayConnectContext in the DLL
    */
-  struct GNUNET_TESTBED_Controller *gateway2;
-
-  /**
-   * Handle for sub-operations
-   */
-  struct GNUNET_TESTBED_Operation *sub_op;
-
-  /**
-   * The client which initiated the link controller operation
-   */
-  struct GNUNET_SERVER_Client *client;
+  struct ForwardedOverlayConnectContext *prev;
 
   /**
    * A copy of the original overlay connect message
@@ -573,34 +752,6 @@ struct ForwardedOverlayConnectContext
    */
   uint64_t operation_id;
 
-  /**
-   * Enumeration of states for this context
-   */
-  enum FOCCState {
-
-    /**
-     * The initial state
-     */
-    FOCC_INIT = 0,
-
-    /**
-     * State where we attempt to get peer2's controller configuration
-     */
-    FOCC_GET_CFG,
-
-    /**
-     * State where we attempt to link the controller of peer 1 to the controller
-     * of peer2
-     */
-    FOCC_LINK,
-
-    /**
-     * State where we attempt to do the overlay connection again
-     */
-    FOCC_OL_CONNECT
-    
-  } state;
-
   /**
    * the id of peer 1
    */
@@ -629,10 +780,16 @@ static struct Context *master_context;
  */
 static char *hostname;
 
+
 /***********/
 /* Handles */
 /***********/
 
+/**
+ * Our configuration
+ */
+static struct GNUNET_CONFIGURATION_Handle *our_config;
+
 /**
  * Current Transmit Handle; NULL if no notify transmit exists currently
  */
@@ -663,7 +820,38 @@ static struct MessageQueue *mq_head;
 static struct MessageQueue *mq_tail;
 
 /**
- * Array of host list
+ * DLL head for OverlayConnectContext DLL - to be used to clean up during shutdown
+ */
+static struct OverlayConnectContext *occq_head;
+
+/**
+ * DLL tail for OverlayConnectContext DLL
+ */
+static struct OverlayConnectContext *occq_tail;
+
+/**
+ * DLL head for RequectOverlayConnectContext DLL - to be used to clean up during
+ * shutdown
+ */
+static struct RequestOverlayConnectContext *roccq_head;
+
+/**
+ * DLL tail for RequectOverlayConnectContext DLL
+ */
+static struct RequestOverlayConnectContext *roccq_tail;
+
+/**
+ * DLL head for forwarded operation contexts
+ */
+static struct ForwardedOperationContext *fopcq_head;
+
+/**
+ * DLL tail for forwarded operation contexts
+ */
+static struct ForwardedOperationContext *fopcq_tail;
+
+/**
+ * Array of hosts
  */
 static struct GNUNET_TESTBED_Host **host_list;
 
@@ -751,6 +939,7 @@ transmit_ready_notify (void *cls, size_t size, void *buf)
   size = ntohs (mq_entry->msg->size);
   memcpy (buf, mq_entry->msg, size);
   GNUNET_free (mq_entry->msg);
+  GNUNET_SERVER_client_drop (mq_entry->client);
   GNUNET_CONTAINER_DLL_remove (mq_head, mq_tail, mq_entry);
   GNUNET_free (mq_entry);
   mq_entry = mq_head;
@@ -785,6 +974,7 @@ queue_message (struct GNUNET_SERVER_Client *client,
   mq_entry = GNUNET_malloc (sizeof (struct MessageQueue));
   mq_entry->msg = msg;
   mq_entry->client = client;
+  GNUNET_SERVER_client_keep (client);
   LOG_DEBUG ("Queueing message of type %u, size %u for sending\n", type,
              ntohs (msg->size));
   GNUNET_CONTAINER_DLL_insert_tail (mq_head, mq_tail, mq_entry);
@@ -882,13 +1072,16 @@ route_list_add (struct Route *route)
 static void
 slave_list_add (struct Slave *slave)
 {
+  uint32_t orig_size;
+
+  orig_size = slave_list_size;  
   if (slave->host_id >= slave_list_size)
   {
+    while (slave->host_id >= slave_list_size)
+      slave_list_size += LIST_GROW_STEP;
     slave_list =
-        TESTBED_realloc (slave_list, sizeof (struct Slave *) * slave_list_size,
-                         sizeof (struct Slave *) * (slave_list_size +
-                                                    LIST_GROW_STEP));
-    slave_list_size += LIST_GROW_STEP;
+        TESTBED_realloc (slave_list, sizeof (struct Slave *) * orig_size,
+                         sizeof (struct Slave *) * slave_list_size);
   }
   GNUNET_assert (NULL == slave_list[slave->host_id]);
   slave_list[slave->host_id] = slave;
@@ -1039,6 +1232,106 @@ send_operation_success_msg (struct GNUNET_SERVER_Client *client,
 }
 
 
+/**
+ * Callback which will be called to after a host registration succeeded or failed
+ *
+ * @param cls the handle to the slave at which the registration is completed
+ * @param emsg the error message; NULL if host registration is successful
+ */
+static void 
+hr_completion (void *cls, const char *emsg);
+
+
+/**
+ * Attempts to register the next host in the host registration queue
+ *
+ * @param slave the slave controller whose host registration queue is checked
+ *          for host registrations
+ */
+static void
+register_next_host (struct Slave *slave)
+{
+  struct HostRegistration *hr;
+  
+  hr = slave->hr_dll_head;
+  GNUNET_assert (NULL != hr);
+  GNUNET_assert (NULL == slave->rhandle);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Registering host %u at %u\n",
+       GNUNET_TESTBED_host_get_id_ (hr->host),
+       GNUNET_TESTBED_host_get_id_ (host_list[slave->host_id]));
+  slave->rhandle = GNUNET_TESTBED_register_host (slave->controller,
+                                                 hr->host,
+                                                 hr_completion,
+                                                 slave);
+}
+
+
+/**
+ * Callback which will be called to after a host registration succeeded or failed
+ *
+ * @param cls the handle to the slave at which the registration is completed
+ * @param emsg the error message; NULL if host registration is successful
+ */
+static void 
+hr_completion (void *cls, const char *emsg)
+{
+  struct Slave *slave = cls;
+  struct HostRegistration *hr;
+
+  slave->rhandle = NULL;
+  hr = slave->hr_dll_head;
+  GNUNET_assert (NULL != hr);
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Registering host %u at %u successful\n",
+       GNUNET_TESTBED_host_get_id_ (hr->host),
+       GNUNET_TESTBED_host_get_id_ (host_list[slave->host_id]));
+  GNUNET_CONTAINER_DLL_remove (slave->hr_dll_head,
+                               slave->hr_dll_tail,
+                               hr);
+  if (NULL != hr->cb)
+    hr->cb (hr->cb_cls, emsg);
+  GNUNET_free (hr);
+  if (NULL != slave->hr_dll_head)
+    register_next_host (slave);
+}
+
+
+/**
+ * Adds a host registration's request to a slave's registration queue
+ *
+ * @param slave the slave controller at which the given host has to be
+ *          registered 
+ * @param cb the host registration completion callback
+ * @param cb_cls the closure for the host registration completion callback
+ * @param host the host which has to be registered
+ */
+static void
+queue_host_registration (struct Slave *slave,
+                         GNUNET_TESTBED_HostRegistrationCompletion cb,
+                         void *cb_cls,
+                         struct GNUNET_TESTBED_Host *host)
+{
+  struct HostRegistration *hr;
+  int call_register;
+
+  LOG (GNUNET_ERROR_TYPE_DEBUG,
+       "Queueing host registration for host %u at %u\n",
+       GNUNET_TESTBED_host_get_id_ (host),
+       GNUNET_TESTBED_host_get_id_ (host_list[slave->host_id]));
+  hr = GNUNET_malloc (sizeof (struct HostRegistration));
+  hr->cb = cb;
+  hr->cb_cls = cb_cls;
+  hr->host = host;
+  call_register = (NULL == slave->hr_dll_head) ? GNUNET_YES : GNUNET_NO;
+  GNUNET_CONTAINER_DLL_insert_tail (slave->hr_dll_head,
+                                    slave->hr_dll_tail,
+                                    hr);
+  if (GNUNET_YES == call_register)
+    register_next_host (slave);
+}
+
+
 /**
  * The  Link Controller forwarding task
  *
@@ -1060,7 +1353,6 @@ lcf_proc_cc (void *cls, const char *emsg)
 {
   struct LCFContext *lcf = cls;
 
-  lcf->rhandle = NULL;
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
   switch (lcf->state)
   {
@@ -1090,11 +1382,10 @@ lcf_proc_cc (void *cls, const char *emsg)
 
 
 /**
- * Callback to be called when forwarded link controllers operation is
- * successfull. We have to relay the reply msg back to the client
+ * Callback to relay the reply msg of a forwarded operation back to the client
  *
  * @param cls ForwardedOperationContext
- * @param msg the peer create success message
+ * @param msg the message to relay
  */
 static void
 forwarded_operation_reply_relay (void *cls,
@@ -1107,17 +1398,17 @@ forwarded_operation_reply_relay (void *cls,
   msize = ntohs (msg->size);
   LOG_DEBUG ("Relaying message with type: %u, size: %u\n", ntohs (msg->type),
              msize);
-  dup_msg = GNUNET_malloc (msize);
-  (void) memcpy (dup_msg, msg, msize);
+  dup_msg = GNUNET_copy_message (msg);
   queue_message (fopc->client, dup_msg);
   GNUNET_SERVER_client_drop (fopc->client);
   GNUNET_SCHEDULER_cancel (fopc->timeout_task);
+  GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
   GNUNET_free (fopc);
 }
 
 
 /**
- * Task to free resources when forwarded link controllers has been timedout
+ * Task to free resources when forwarded operation has been timedout
  *
  * @param cls the ForwardedOperationContext
  * @param tc the task context from scheduler
@@ -1129,12 +1420,66 @@ forwarded_operation_timeout (void *cls,
   struct ForwardedOperationContext *fopc = cls;
 
   GNUNET_TESTBED_forward_operation_msg_cancel_ (fopc->opc);
+  LOG (GNUNET_ERROR_TYPE_WARNING, "A forwarded operation has timed out\n");
   send_operation_fail_msg (fopc->client, fopc->operation_id, "Timeout");
   GNUNET_SERVER_client_drop (fopc->client);
+  GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
   GNUNET_free (fopc);
 }
 
 
+/**
+ * The  Link Controller forwarding task
+ *
+ * @param cls the LCFContext
+ * @param tc the Task context from scheduler
+ */
+static void
+lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+/**
+ * Callback to be called when forwarded link controllers operation is
+ * successfull. We have to relay the reply msg back to the client
+ *
+ * @param cls the LCFContext
+ * @param msg the message to relay
+ */
+static void
+lcf_forwarded_operation_reply_relay (void *cls,
+                                     const struct GNUNET_MessageHeader *msg)
+{
+  struct LCFContext *lcf = cls;
+
+  GNUNET_assert (NULL != lcf->fopc);
+  forwarded_operation_reply_relay (lcf->fopc, msg);
+  lcf->fopc = NULL;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
+  lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
+}
+
+
+/**
+ * Task to free resources when forwarded link controllers has been timedout
+ *
+ * @param cls the LCFContext
+ * @param tc the task context from scheduler
+ */
+static void
+lcf_forwarded_operation_timeout (void *cls,
+                                 const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct LCFContext *lcf = cls;
+
+  GNUNET_assert (NULL != lcf->fopc);
+  lcf->fopc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  forwarded_operation_timeout (lcf->fopc, tc);
+  lcf->fopc = NULL;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
+  lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
+}
+
+
 /**
  * The  Link Controller forwarding task
  *
@@ -1146,7 +1491,6 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct LCFContext *lcf = cls;
   struct LCFContextQueue *lcfq;
-  struct ForwardedOperationContext *fopc;
 
   lcf_proc_task_id = GNUNET_SCHEDULER_NO_TASK;
   switch (lcf->state)
@@ -1156,10 +1500,9 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
         GNUNET_TESTBED_is_host_registered_ (host_list[lcf->delegated_host_id],
                                             lcf->gateway->controller))
     {
-      lcf->rhandle =
-          GNUNET_TESTBED_register_host (lcf->gateway->controller,
-                                        host_list[lcf->delegated_host_id],
-                                        lcf_proc_cc, lcf);
+      queue_host_registration (lcf->gateway,
+                               lcf_proc_cc, lcf,
+                               host_list[lcf->delegated_host_id]);
     }
     else
     {
@@ -1172,10 +1515,9 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
         GNUNET_TESTBED_is_host_registered_ (host_list[lcf->slave_host_id],
                                             lcf->gateway->controller))
     {
-      lcf->rhandle =
-          GNUNET_TESTBED_register_host (lcf->gateway->controller,
-                                        host_list[lcf->slave_host_id],
-                                        lcf_proc_cc, lcf);
+      queue_host_registration (lcf->gateway,
+                               lcf_proc_cc, lcf,
+                               host_list[lcf->slave_host_id]);
     }
     else
     {
@@ -1184,20 +1526,21 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     }
     break;
   case SLAVE_HOST_REGISTERED:
-    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-    fopc->client = lcf->client;
-    fopc->operation_id = lcf->operation_id;
-    fopc->opc =
+    lcf->fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
+    lcf->fopc->client = lcf->client;
+    lcf->fopc->operation_id = lcf->operation_id;
+    lcf->fopc->type = OP_LINK_CONTROLLERS;
+    lcf->fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (lcf->gateway->controller,
                                                lcf->operation_id,
-                                               &lcf->msg->header,
-                                               &forwarded_operation_reply_relay,
-                                               fopc);
-    fopc->timeout_task =
-        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
-                                      fopc);
+                                               &lcf->msg->header,
+                                               &lcf_forwarded_operation_reply_relay,
+                                               lcf);
+    lcf->fopc->timeout_task =
+        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &lcf_forwarded_operation_timeout,
+                                      lcf);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, lcf->fopc);
     lcf->state = FINISHED;
-    lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
     break;
   case FINISHED:
     lcfq = lcfq_head;
@@ -1213,6 +1556,55 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 }
 
 
+/**
+ * Cleans up ForwardedOverlayConnectContext
+ *
+ * @param focc the ForwardedOverlayConnectContext to cleanup
+ */
+static void
+cleanup_focc (struct ForwardedOverlayConnectContext *focc)
+{
+  GNUNET_free_non_null (focc->orig_msg);
+  GNUNET_free (focc);
+}
+
+
+/**
+ * Processes a forwarded overlay connect context in the queue of the given RegisteredHostContext
+ *
+ * @param rhc the RegisteredHostContext
+ */
+static void
+process_next_focc (struct RegisteredHostContext *rhc);
+
+
+/**
+ * Timeout task for cancelling a forwarded overlay connect connect
+ *
+ * @param cls the ForwardedOverlayConnectContext
+ * @param tc the task context from the scheduler
+ */
+static void
+forwarded_overlay_connect_timeout (void *cls,
+                                   const struct GNUNET_SCHEDULER_TaskContext
+                                   *tc)
+{
+  struct ForwardedOperationContext *fopc = cls;
+  struct RegisteredHostContext *rhc;
+  struct ForwardedOverlayConnectContext *focc;
+  
+  rhc = fopc->cls;
+  focc = rhc->focc_dll_head;
+  GNUNET_CONTAINER_DLL_remove (rhc->focc_dll_head, rhc->focc_dll_tail, focc);
+  cleanup_focc (focc);
+  LOG_DEBUG ("Overlay linking between peers %u and %u failed\n",
+            focc->peer1, focc->peer2);
+  forwarded_operation_timeout (cls, tc);
+  if (NULL != rhc->focc_dll_head)
+    process_next_focc (rhc);
+}
+
+
 /**
  * Callback to be called when forwarded overlay connection operation has a reply
  * from the sub-controller successfull. We have to relay the reply msg back to
@@ -1223,24 +1615,56 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
  */
 static void
 forwarded_overlay_connect_listener (void *cls,
-                                    const struct GNUNET_MessageHeader *msg);
+                                    const struct GNUNET_MessageHeader *msg)
+{
+  struct ForwardedOperationContext *fopc = cls;
+  struct RegisteredHostContext *rhc;
+  struct ForwardedOverlayConnectContext *focc;
+  
+  rhc = fopc->cls;
+  forwarded_operation_reply_relay (cls, msg);
+  focc = rhc->focc_dll_head;
+  GNUNET_CONTAINER_DLL_remove (rhc->focc_dll_head, rhc->focc_dll_tail, focc);
+  cleanup_focc (focc);
+  if (NULL != rhc->focc_dll_head)
+    process_next_focc (rhc);
+}
 
 
 /**
- * Cleans up ForwardedOverlayConnectContext
+ * Processes a forwarded overlay connect context in the queue of the given RegisteredHostContext
  *
- * @param focc the ForwardedOverlayConnectContext to cleanup
+ * @param rhc the RegisteredHostContext
  */
 static void
-cleanup_focc (struct ForwardedOverlayConnectContext *focc)
+process_next_focc (struct RegisteredHostContext *rhc)
 {
-  if (NULL != focc->sub_op)
-    GNUNET_TESTBED_operation_done (focc->sub_op);
-  GNUNET_SERVER_client_drop (focc->client);
-  GNUNET_free_non_null (focc->orig_msg);
-  GNUNET_free (focc);
+  struct ForwardedOperationContext *fopc;
+  struct ForwardedOverlayConnectContext *focc;
+
+  focc = rhc->focc_dll_head;
+  GNUNET_assert (NULL != focc);
+  GNUNET_assert (RHC_OL_CONNECT == rhc->state);
+  fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
+  GNUNET_SERVER_client_keep (rhc->client);
+  fopc->client = rhc->client;  
+  fopc->operation_id = focc->operation_id;
+  fopc->cls = rhc;
+  fopc->type = OP_OVERLAY_CONNECT;
+  fopc->opc =
+        GNUNET_TESTBED_forward_operation_msg_ (rhc->gateway->controller,
+                                               focc->operation_id, focc->orig_msg,
+                                               &forwarded_overlay_connect_listener,
+                                               fopc);
+  GNUNET_free (focc->orig_msg);
+  focc->orig_msg = NULL;
+  fopc->timeout_task =
+      GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_overlay_connect_timeout,
+                                    fopc);
+  GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
 }
 
+
 /**
  * Callback for event from slave controllers
  *
@@ -1251,65 +1675,39 @@ static void
 slave_event_callback (void *cls,
                       const struct GNUNET_TESTBED_EventInformation *event)
 {
-  struct ForwardedOverlayConnectContext *focc;
-  struct ForwardedOperationContext *fopc;
-  struct GNUNET_CONFIGURATION_Handle *slave_cfg;
+  struct RegisteredHostContext *rhc;
+  struct GNUNET_CONFIGURATION_Handle *cfg;
   struct GNUNET_TESTBED_Operation *old_op;
-  char *emsg;
-
-  /* We currently only get here when doing overlay connect operations and that
-     too while trying out sub operations */
-  GNUNET_assert (GNUNET_TESTBED_ET_OPERATION_FINISHED != event->type);
-  focc = event->details.operation_finished.op_cls;
-  if (NULL != event->details.operation_finished.emsg)
-  {
-    GNUNET_asprintf (&emsg, "Failure executing suboperation: %s",
-                     event->details.operation_finished.emsg);
-    send_operation_fail_msg (focc->client, focc->operation_id,
-                             emsg);
-    GNUNET_free (emsg);
-    cleanup_focc (focc);
-    return;
-  }
-  switch (focc->state)
-  {
-  case FOCC_GET_CFG:
-    slave_cfg = event->details.operation_finished.generic;
-    old_op = focc->sub_op;
-    focc->state = FOCC_LINK;
-    focc->sub_op = GNUNET_TESTBED_controller_link_ (focc,
-                                                    focc->gateway,
-                                                    focc->peer2_host_id,
-                                                    peer_list[focc->peer1]->details.remote.remote_host_id,
-                                                    slave_cfg,
-                                                    GNUNET_NO);
+  
+  /* We currently only get here when working on RegisteredHostContexts */
+  GNUNET_assert (GNUNET_TESTBED_ET_OPERATION_FINISHED == event->type);
+  rhc = event->details.operation_finished.op_cls;
+  GNUNET_assert (rhc->sub_op == event->details.operation_finished.operation);
+  switch (rhc->state)
+  {
+  case RHC_GET_CFG:
+    cfg = event->details.operation_finished.generic;
+    old_op = rhc->sub_op;
+    rhc->state = RHC_LINK;
+    rhc->sub_op =
+        GNUNET_TESTBED_controller_link (rhc,
+                                        rhc->gateway->controller,
+                                        rhc->reg_host,
+                                        rhc->host,
+                                        cfg,
+                                        GNUNET_NO);
     GNUNET_TESTBED_operation_done (old_op);
     break;
-  case FOCC_LINK:
-    GNUNET_TESTBED_operation_done (focc->sub_op);
-    focc->sub_op = NULL;
-    focc->state = FOCC_OL_CONNECT;
-    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-    fopc->client = focc->client;
-    focc->client = NULL;
-    fopc->operation_id = focc->operation_id;
-    fopc->cls = NULL;
-    fopc->opc =
-        GNUNET_TESTBED_forward_operation_msg_ (focc->gateway,
-                                               focc->operation_id, focc->orig_msg,
-                                               &forwarded_operation_reply_relay,
-                                               fopc);
-    GNUNET_free (focc->orig_msg);
-    focc->orig_msg = NULL;
-    fopc->timeout_task =
-       GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
-                                     fopc);
-    cleanup_focc (focc);
+  case RHC_LINK:
+    LOG_DEBUG ("OL: Linking controllers successfull\n");
+    GNUNET_TESTBED_operation_done (rhc->sub_op);
+    rhc->sub_op = NULL;
+    rhc->state = RHC_OL_CONNECT;
+    process_next_focc (rhc);
     break;
   default:
     GNUNET_assert (0);
   }
-  return;
 }
 
 
@@ -1423,7 +1821,9 @@ handle_init (void *cls, struct GNUNET_SERVER_Client *client,
   master_context->system =
       GNUNET_TESTING_system_create ("testbed", master_context->master_ip, hostname);
   host =
-      GNUNET_TESTBED_host_create_with_id (master_context->host_id, NULL, NULL,
+      GNUNET_TESTBED_host_create_with_id (master_context->host_id,
+                                          master_context->master_ip,
+                                          NULL,
                                           0);
   host_list_add (host);
   GNUNET_SERVER_client_keep (client);
@@ -1458,19 +1858,22 @@ handle_add_host (void *cls, struct GNUNET_SERVER_Client *client,
 
   msg = (const struct GNUNET_TESTBED_AddHostMessage *) message;
   msize = ntohs (msg->header.size);
-  username = (char *) &(msg[1]);
+  username = (char *) &msg[1];
   username_length = ntohs (msg->user_name_length);
-  GNUNET_assert (msize > (sizeof (struct GNUNET_TESTBED_AddHostMessage) + username_length + 1));        /* msg must contain hostname */
   if (0 != username_length)
-    GNUNET_assert ('\0' == username[username_length]);
-  username_length = (0 == username_length) ? 0 : username_length + 1;
+    username_length++;
+  /* msg must contain hostname */
+  GNUNET_assert (msize > (sizeof (struct GNUNET_TESTBED_AddHostMessage) +
+                         username_length + 1));
+  if (0 != username_length)
+    GNUNET_assert ('\0' == username[username_length - 1]);
   hostname = username + username_length;
   hostname_length =
       msize - (sizeof (struct GNUNET_TESTBED_AddHostMessage) + username_length);
   GNUNET_assert ('\0' == hostname[hostname_length - 1]);
   GNUNET_assert (strlen (hostname) == hostname_length - 1);
   host_id = ntohl (msg->host_id);
-  LOG_DEBUG ("Received ADDHOST message\n");
+  LOG_DEBUG ("Received ADDHOST %u message\n", host_id);
   LOG_DEBUG ("-------host id: %u\n", host_id);
   LOG_DEBUG ("-------hostname: %s\n", hostname);
   if (0 != username_length)
@@ -1485,7 +1888,6 @@ handle_add_host (void *cls, struct GNUNET_SERVER_Client *client,
       GNUNET_TESTBED_host_create_with_id (host_id, hostname, username,
                                           ntohs (msg->ssh_port));
   GNUNET_assert (NULL != host);
-  GNUNET_SERVER_receive_done (client, GNUNET_OK);
   reply_size = sizeof (struct GNUNET_TESTBED_HostConfirmedMessage);
   if (GNUNET_OK != host_list_add (host))
   {
@@ -1498,11 +1900,16 @@ handle_add_host (void *cls, struct GNUNET_SERVER_Client *client,
     memcpy (&reply[1], emsg, strlen (emsg) + 1);
   }
   else
+  {
+    LOG_DEBUG ("Added host %u at %u\n",
+              host_id, master_context->host_id);
     reply = GNUNET_malloc (reply_size);
+  }
   reply->header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_ADDHOSTCONFIRM);
   reply->header.size = htons (reply_size);
   reply->host_id = htonl (host_id);
   queue_message (client, &reply->header);
+  GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
 
 
@@ -1649,7 +2056,8 @@ handle_link_controllers (void *cls, struct GNUNET_SERVER_Client *client,
   slave_host_id = ntohl (msg->slave_host_id);
   if ((slave_host_id >= host_list_size) || (NULL == host_list[slave_host_id]))
   {
-    LOG (GNUNET_ERROR_TYPE_WARNING, "Slave host not registered with us\n");
+    LOG (GNUNET_ERROR_TYPE_WARNING, "Slave host %u not registered with us\n",
+        slave_host_id);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
@@ -1671,7 +2079,7 @@ handle_link_controllers (void *cls, struct GNUNET_SERVER_Client *client,
     {
       LOG (GNUNET_ERROR_TYPE_WARNING, "Host %u already connected\n",
            delegated_host_id);
-      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+      GNUNET_SERVER_receive_done (client, GNUNET_OK);
       return;
     }
     config = GNUNET_malloc (config_size);
@@ -1711,6 +2119,7 @@ handle_link_controllers (void *cls, struct GNUNET_SERVER_Client *client,
     }
     slave = GNUNET_malloc (sizeof (struct Slave));
     slave->host_id = delegated_host_id;
+    slave->reghost_map = GNUNET_CONTAINER_multihashmap_create (100, GNUNET_NO);
     slave_list_add (slave);
     if (1 != msg->is_subordinate)
     {
@@ -1808,20 +2217,16 @@ static void
 peer_create_forward_timeout (void *cls,
                              const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct ForwardedOperationContext *fo_ctxt = cls;
+  struct ForwardedOperationContext *fopc = cls;
 
-  /* send error msg to client */
-  GNUNET_free (fo_ctxt->cls);
-  send_operation_fail_msg (fo_ctxt->client, fo_ctxt->operation_id, "Timedout");
-  GNUNET_SERVER_client_drop (fo_ctxt->client);
-  GNUNET_TESTBED_forward_operation_msg_cancel_ (fo_ctxt->opc);
-  GNUNET_free (fo_ctxt);
+  GNUNET_free (fopc->cls);
+  forwarded_operation_timeout (fopc, tc);
 }
 
 
 /**
- * Callback to be called when forwarded peer create operation is
- * successfull. We have to relay the reply msg back to the client
+ * Callback to be called when forwarded peer create operation is successfull. We
+ * have to relay the reply msg back to the client
  *
  * @param cls ForwardedOperationContext
  * @param msg the peer create success message
@@ -1829,29 +2234,68 @@ peer_create_forward_timeout (void *cls,
 static void
 peer_create_success_cb (void *cls, const struct GNUNET_MessageHeader *msg)
 {
-  struct ForwardedOperationContext *fo_ctxt = cls;
-  const struct GNUNET_TESTBED_PeerCreateSuccessEventMessage *success_msg;
-  struct GNUNET_MessageHeader *dup_msg;
+  struct ForwardedOperationContext *fopc = cls;
   struct Peer *remote_peer;
-  uint16_t msize;
 
-  GNUNET_SCHEDULER_cancel (fo_ctxt->timeout_task);
   if (ntohs (msg->type) == GNUNET_MESSAGE_TYPE_TESTBED_PEERCREATESUCCESS)
   {
-    success_msg =
-        (const struct GNUNET_TESTBED_PeerCreateSuccessEventMessage *) msg;
-    GNUNET_assert (NULL != fo_ctxt->cls);
-    remote_peer = fo_ctxt->cls;
-    GNUNET_assert (remote_peer->details.remote.remote_host_id
-                   == ntohl (success_msg->peer_id));
+    GNUNET_assert (NULL != fopc->cls);
+    remote_peer = fopc->cls;
     peer_list_add (remote_peer);
   }
-  msize = ntohs (msg->size);
-  dup_msg = GNUNET_malloc (msize);
-  (void) memcpy (dup_msg, msg, msize);
-  queue_message (fo_ctxt->client, dup_msg);
-  GNUNET_SERVER_client_drop (fo_ctxt->client);
-  GNUNET_free (fo_ctxt);
+  forwarded_operation_reply_relay (fopc, msg);
+}
+
+
+/**
+ * Function to destroy a peer
+ *
+ * @param the peer structure to destroy
+ */
+static void
+destroy_peer (struct Peer *peer)
+{
+  GNUNET_break (0 == peer->reference_cnt);
+  if (GNUNET_YES == peer->is_remote)
+  {
+    peer_list_remove (peer);
+    GNUNET_free (peer);
+    return;
+  }
+  if (GNUNET_YES == peer->details.local.is_running)
+  {
+    GNUNET_TESTING_peer_stop (peer->details.local.peer);
+    peer->details.local.is_running = GNUNET_NO;
+  }
+  GNUNET_TESTING_peer_destroy (peer->details.local.peer);
+  GNUNET_CONFIGURATION_destroy (peer->details.local.cfg);
+  peer_list_remove (peer);
+  GNUNET_free (peer);
+}
+
+
+/**
+ * Callback to be called when forwarded peer destroy operation is successfull. We
+ * have to relay the reply msg back to the client
+ *
+ * @param cls ForwardedOperationContext
+ * @param msg the peer create success message
+ */
+static void
+peer_destroy_success_cb (void *cls, const struct GNUNET_MessageHeader *msg)
+{
+  struct ForwardedOperationContext *fopc = cls;
+  struct Peer *remote_peer;
+
+  if (GNUNET_MESSAGE_TYPE_TESTBED_GENERICOPSUCCESS == ntohs (msg->type))
+  {
+    remote_peer = fopc->cls;
+    GNUNET_assert (NULL != remote_peer);
+    remote_peer->destroy_flag = GNUNET_YES;
+    if (0 == remote_peer->reference_cnt)
+      destroy_peer (remote_peer);
+  }
+  forwarded_operation_reply_relay (fopc, msg);
 }
 
 
@@ -1980,13 +2424,14 @@ handle_peer_create (void *cls, struct GNUNET_SERVER_Client *client,
   peer = GNUNET_malloc (sizeof (struct Peer));
   peer->is_remote = GNUNET_YES;
   peer->id = peer_id;
-  peer->details.remote.controller = slave_list[route->dest]->controller;
+  peer->details.remote.slave = slave_list[route->dest];
   peer->details.remote.remote_host_id = host_id;
   fo_ctxt = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
   GNUNET_SERVER_client_keep (client);
   fo_ctxt->client = client;
   fo_ctxt->operation_id = GNUNET_ntohll (msg->operation_id);
   fo_ctxt->cls = peer; //slave_list[route->dest]->controller;
+  fo_ctxt->type = OP_PEER_CREATE;
   fo_ctxt->opc =
       GNUNET_TESTBED_forward_operation_msg_ (slave_list [route->dest]->controller,
                                              fo_ctxt->operation_id,
@@ -1995,6 +2440,7 @@ handle_peer_create (void *cls, struct GNUNET_SERVER_Client *client,
   fo_ctxt->timeout_task =
       GNUNET_SCHEDULER_add_delayed (TIMEOUT, &peer_create_forward_timeout,
                                     fo_ctxt);
+  GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fo_ctxt);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
 
@@ -2035,22 +2481,27 @@ handle_peer_destroy (void *cls, struct GNUNET_SERVER_Client *client,
     fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
-    fopc->operation_id = GNUNET_ntohll (msg->operation_id);
-    fopc->opc =
-        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.controller,
+    fopc->cls = peer;
+    fopc->type = OP_PEER_DESTROY;
+    fopc->operation_id = GNUNET_ntohll (msg->operation_id);    
+    fopc->opc = 
+        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
-                                               &forwarded_operation_reply_relay,
+                                               &peer_destroy_success_cb,
                                                fopc);
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-  GNUNET_TESTING_peer_destroy (peer->details.local.peer);
-  GNUNET_CONFIGURATION_destroy (peer->details.local.cfg);
-  peer_list_remove (peer);
-  GNUNET_free (peer);
+  peer->destroy_flag = GNUNET_YES;
+  if (0 == peer->reference_cnt)
+    destroy_peer (peer);
+  else
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Delaying peer destroy as peer is currently in use\n");
   send_operation_success_msg (client, GNUNET_ntohll (msg->operation_id));
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
@@ -2090,14 +2541,16 @@ handle_peer_start (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_START;
     fopc->opc =
-        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.controller,
+        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
                                                &forwarded_operation_reply_relay,
                                                fopc);
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2154,14 +2607,16 @@ handle_peer_stop (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_STOP;
     fopc->opc =
-        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.controller,
+        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
                                                &forwarded_operation_reply_relay,
                                                fopc);
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2224,14 +2679,16 @@ handle_peer_get_config (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_INFO;
     fopc->opc =
-        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.controller,
+        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
                                                &forwarded_operation_reply_relay,
                                                fopc);
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
-                                      fopc);    
+                                      fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc); 
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2258,16 +2715,13 @@ handle_peer_get_config (void *cls, struct GNUNET_SERVER_Client *client,
 
 
 /**
- * Task for cleaing up overlay connect context structure
+ * Cleanup overlay connect context structure
  *
- * @param cls the overlay connect context
- * @param tc the task context
+ * @param occ the overlay connect context
  */
 static void
-occ_cleanup (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+cleanup_occ (struct OverlayConnectContext *occ)
 {
-  struct OverlayConnectContext *occ = cls;
-
   LOG_DEBUG ("Cleaning up occ\n");
   GNUNET_free_non_null (occ->emsg);
   GNUNET_free_non_null (occ->hello);
@@ -2276,18 +2730,55 @@ occ_cleanup (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_TESTBED_forward_operation_msg_cancel_ (occ->opc);
   if (GNUNET_SCHEDULER_NO_TASK != occ->send_hello_task)
     GNUNET_SCHEDULER_cancel (occ->send_hello_task);
+  if (GNUNET_SCHEDULER_NO_TASK != occ->cleanup_task)
+    GNUNET_SCHEDULER_cancel (occ->cleanup_task);
+  if (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task)
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
   if (NULL != occ->ch)
+  {
     GNUNET_CORE_disconnect (occ->ch);
+    occ->peer->reference_cnt--;
+  }
   if (NULL != occ->ghh)
     GNUNET_TRANSPORT_get_hello_cancel (occ->ghh);
   if (NULL != occ->p1th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p1th);
+    occ->peer->reference_cnt--;
+  }
   if (NULL != occ->p2th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p2th);
+    peer_list[occ->other_peer_id]->reference_cnt--;
+  }
+  if ((GNUNET_YES == occ->peer->destroy_flag)
+      && (0 == occ->peer->reference_cnt))
+    destroy_peer (occ->peer);
+  if ((NULL == occ->peer2_controller)
+      && (GNUNET_YES == peer_list[occ->other_peer_id]->destroy_flag)
+        && (0 == peer_list[occ->other_peer_id]->reference_cnt))
+      destroy_peer (peer_list[occ->other_peer_id]);  
+  GNUNET_CONTAINER_DLL_remove (occq_head, occq_tail, occ);
   GNUNET_free (occ);
 }
 
 
+/**
+ * Task for cleaing up overlay connect context structure
+ *
+ * @param cls the overlay connect context
+ * @param tc the task context
+ */
+static void
+do_cleanup_occ (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct OverlayConnectContext *occ = cls;
+  
+  occ->cleanup_task = GNUNET_SCHEDULER_NO_TASK;
+  cleanup_occ (occ);
+}
+
+
 /**
  * Task which will be run when overlay connect request has been timed out
  *
@@ -2301,8 +2792,11 @@ timeout_overlay_connect (void *cls,
   struct OverlayConnectContext *occ = cls;
 
   occ->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  LOG (GNUNET_ERROR_TYPE_WARNING,
+       "Timeout while connecting peers %u and %u\n",
+       occ->peer_id, occ->other_peer_id);
   send_operation_fail_msg (occ->client, occ->op_id, occ->emsg);
-  occ_cleanup (occ, tc);
+  cleanup_occ (occ);
 }
 
 
@@ -2358,7 +2852,10 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = NULL;
   if (NULL != occ->p2th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p2th);
+    peer_list[occ->other_peer_id]->reference_cnt--;
+  }
   occ->p2th = NULL;
   LOG_DEBUG ("Peers connected - Sending overlay connect success\n");
   msg = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
@@ -2370,7 +2867,8 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
   msg->peer2 = htonl (occ->other_peer_id);
   msg->operation_id = GNUNET_htonll (occ->op_id);
   queue_message (occ->client, &msg->header);
-  GNUNET_SCHEDULER_add_now (&occ_cleanup, occ);
+  occ->cleanup_task = GNUNET_SCHEDULER_add_now (&do_cleanup_occ, occ);
+  //cleanup_occ (occ);
 }
 
 
@@ -2478,9 +2976,11 @@ hello_update_cb (void *cls, const struct GNUNET_MessageHeader *hello)
   occ->ghh = NULL;
   GNUNET_TRANSPORT_disconnect (occ->p1th);
   occ->p1th = NULL;
+  occ->peer->reference_cnt--;
   GNUNET_free_non_null (occ->emsg);
   if (NULL == occ->peer2_controller)
-  {   
+  {
+    peer_list[occ->other_peer_id]->reference_cnt++;
     occ->p2th =
        GNUNET_TRANSPORT_connect (peer_list[occ->other_peer_id]->details.local.cfg,
                                  &occ->other_peer_identity, NULL, NULL, NULL,
@@ -2525,6 +3025,7 @@ core_startup_cb (void *cls, struct GNUNET_CORE_Handle *server,
   occ->emsg = NULL;
   memcpy (&occ->peer_identity, my_identity,
           sizeof (struct GNUNET_PeerIdentity));
+  occ->peer->reference_cnt++;
   occ->p1th =
       GNUNET_TRANSPORT_connect (occ->peer->details.local.cfg,
                                 &occ->peer_identity, NULL, NULL, NULL, NULL);
@@ -2572,6 +3073,7 @@ overlay_connect_get_config (void *cls, const struct GNUNET_MessageHeader *msg)
          sizeof (struct GNUNET_PeerIdentity));
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = GNUNET_strdup ("Timeout while connecting to CORE");
+  occ->peer->reference_cnt++;
   occ->ch =
       GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
                            &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
@@ -2588,49 +3090,91 @@ overlay_connect_get_config (void *cls, const struct GNUNET_MessageHeader *msg)
 
 
 /**
- * Callback to be called when forwarded overlay connection operation has a reply
- * from the sub-controller successfull. We have to relay the reply msg back to
- * the client
+ * Callback which will be called to after a host registration succeeded or failed
  *
- * @param cls ForwardedOperationContext
- * @param msg the peer create success message
+ * @param cls the RegisteredHostContext
+ * @param emsg the error message; NULL if host registration is successful
  */
-static void
-forwarded_overlay_connect_listener (void *cls,
-                                    const struct GNUNET_MessageHeader *msg)
+static void 
+registeredhost_registration_completion (void *cls, const char *emsg)
 {
-  struct ForwardedOperationContext *fopc = cls;
-  struct ForwardedOverlayConnectContext *focc;
+  struct RegisteredHostContext *rhc = cls;
+  struct GNUNET_CONFIGURATION_Handle *cfg;
+  uint32_t peer2_host_id;
 
-  focc = fopc->cls;
-  if (NULL == focc)
-  {
-    forwarded_operation_reply_relay (cls, msg);
+  /* if (NULL != rhc->focc_dll_head) */
+  /*   process_next_focc (rhc); */
+  peer2_host_id = GNUNET_TESTBED_host_get_id_ (rhc->reg_host);
+  GNUNET_assert (RHC_INIT == rhc->state);
+  GNUNET_assert (NULL == rhc->sub_op);
+  if ((NULL == rhc->gateway2)
+      || ((peer2_host_id < slave_list_size) /* Check if we have the needed config */
+          && (NULL != slave_list[peer2_host_id])))
+  {
+    rhc->state = RHC_LINK;
+    cfg = (NULL == rhc->gateway2) ? our_config : slave_list[peer2_host_id]->cfg;
+    rhc->sub_op =
+        GNUNET_TESTBED_controller_link (rhc,
+                                        rhc->gateway->controller,
+                                        rhc->reg_host,
+                                        rhc->host,
+                                        cfg,
+                                        GNUNET_NO);
     return;
   }
-  switch (focc->state)
+  rhc->state = RHC_GET_CFG;
+  rhc->sub_op =  GNUNET_TESTBED_get_slave_config (rhc,
+                                                  rhc->gateway2->controller,
+                                                  rhc->reg_host);
+}
+
+
+/**
+ * Iterator to match a registered host context
+ *
+ * @param cls pointer 2 pointer of RegisteredHostContext
+ * @param key current key code
+ * @param value value in the hash map
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int 
+reghost_match_iterator (void *cls,
+                        const struct GNUNET_HashCode * key,
+                        void *value)
+{
+  struct RegisteredHostContext **rh = cls;
+  struct RegisteredHostContext *rh_val = value;
+
+  if ((rh_val->host == (*rh)->host) && (rh_val->reg_host == (*rh)->reg_host))
   {
-  case FOCC_INIT:
-    if (GNUNET_MESSAGE_TYPE_TESTBED_NEEDCONTROLLERCONFIG != ntohs (msg->type))
-    {
-      GNUNET_break (0);  /* Something failed; you may check output of
-                            sub-controllers */
-      cleanup_focc (focc);
-      forwarded_operation_reply_relay (cls, msg);
-      return;
-    }
-    GNUNET_assert (NULL == focc->sub_op);
-    focc->state = FOCC_GET_CFG;
-    focc->sub_op = GNUNET_TESTBED_get_slave_config_ (focc, focc->gateway2,
-                                                     focc->peer2_host_id);
-    break;
-  default:
-    GNUNET_assert (0);
+    GNUNET_free (*rh);
+    *rh = rh_val;
+    return GNUNET_NO;
   }
-  GNUNET_SERVER_client_drop (fopc->client);
-  GNUNET_SCHEDULER_cancel (fopc->timeout_task);
-  fopc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
-  GNUNET_free (fopc);
+  return GNUNET_YES;
+}
+
+
+/**
+ * Function to generate the hashcode corresponding to a RegisteredHostContext
+ *
+ * @param reg_host the host which is being registered in RegisteredHostContext
+ * @param host the host of the controller which has to connect to the above rhost
+ * @return the hashcode
+ */
+static struct GNUNET_HashCode
+hash_hosts (struct GNUNET_TESTBED_Host *reg_host,
+            struct GNUNET_TESTBED_Host *host)
+{
+  struct GNUNET_HashCode hash;
+  uint32_t host_ids[2];
+
+  host_ids[0] = GNUNET_TESTBED_host_get_id_ (reg_host);
+  host_ids[1] = GNUNET_TESTBED_host_get_id_ (host);
+  GNUNET_CRYPTO_hash (host_ids, sizeof (host_ids), &hash);
+  return hash;
 }
 
 
@@ -2646,17 +3190,17 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
                         const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TESTBED_OverlayConnectMessage *msg;
-  struct OverlayConnectContext *occ;
   const struct GNUNET_CORE_MessageHandler no_handlers[] = {
     {NULL, 0, 0}
   };
   struct Peer *peer;
+  struct OverlayConnectContext *occ;
+  struct GNUNET_TESTBED_Controller *peer2_controller;
   uint64_t operation_id;
   uint32_t p1;
   uint32_t p2; 
   uint32_t peer2_host_id;
 
-  
   msg = (const struct GNUNET_TESTBED_OverlayConnectMessage *) message;
   p1 = ntohl (msg->peer1);
   p2 = ntohl (msg->peer2);
@@ -2665,6 +3209,8 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
   GNUNET_assert (NULL != peer_list[p1]);
   peer = peer_list[p1];
   operation_id = GNUNET_ntohll (msg->operation_id);  
+  LOG_DEBUG ("Received overlay connect for peers %u and %u with op id: 0x%lx\n",
+            p1, p2, operation_id);
   if (GNUNET_YES == peer->is_remote)
   {
     struct ForwardedOperationContext *fopc;
@@ -2672,57 +3218,104 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     struct Route *route_to_peer1_host;
 
     LOG_DEBUG ("Forwarding overlay connect\n");
-    GNUNET_SERVER_client_keep (client);
-    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-    fopc->client = client;
-    fopc->operation_id = operation_id;
     route_to_peer2_host = NULL;
     route_to_peer1_host = NULL;
     route_to_peer2_host = find_dest_route (peer2_host_id);
-    if (NULL != route_to_peer2_host)
+    if ((NULL != route_to_peer2_host) 
+        || (peer2_host_id == master_context->host_id))
     {
+      /* Peer 2 either below us OR with us */
       route_to_peer1_host = 
           find_dest_route (peer_list[p1]->details.remote.remote_host_id);
+      /* Because we get this message only if we know where peer 1 is */
       GNUNET_assert (NULL != route_to_peer1_host);
-      if (route_to_peer2_host->dest != route_to_peer1_host->dest)
+      if ((peer2_host_id == master_context->host_id) 
+          || (route_to_peer2_host->dest != route_to_peer1_host->dest))
       {
-        struct ForwardedOverlayConnectContext *focc;
-        uint16_t msize;
-
-        msize = sizeof (struct GNUNET_TESTBED_OverlayConnectMessage);
-        focc = GNUNET_malloc (sizeof (struct ForwardedOverlayConnectContext));
-        focc->gateway = peer->details.remote.controller;
-        focc->gateway2 = slave_list[route_to_peer2_host->dest]->controller;
-        focc->peer1 = p1;
-        focc->peer2 = p2;
-        focc->peer2_host_id = peer2_host_id;
-        focc->state = FOCC_INIT;
-        focc->orig_msg = GNUNET_malloc (msize);
-        (void) memcpy (focc->orig_msg, message, msize);
+        /* Peer2 is either with us OR peer1 and peer2 can be reached through
+           different gateways */
+        struct GNUNET_HashCode hash;
+        struct RegisteredHostContext *rhc;
+        int skip_focc;
+
+        rhc = GNUNET_malloc (sizeof (struct RegisteredHostContext));
+        if (NULL != route_to_peer2_host)
+          rhc->reg_host = host_list[route_to_peer2_host->dest];
+        else
+          rhc->reg_host = host_list[master_context->host_id];
+        rhc->host = host_list[route_to_peer1_host->dest];
+        GNUNET_assert (NULL != rhc->reg_host);
+        GNUNET_assert (NULL != rhc->host);
+        rhc->gateway = peer->details.remote.slave;
+        rhc->gateway2 = (NULL == route_to_peer2_host) ? NULL :
+            slave_list[route_to_peer2_host->dest];
+        rhc->state = RHC_INIT;
         GNUNET_SERVER_client_keep (client);
-        focc->client = client;
-        focc->operation_id = operation_id;
-        fopc->cls = focc;
+        rhc->client = client;
+        hash = hash_hosts (rhc->reg_host, rhc->host);
+        skip_focc = GNUNET_NO;
+        if ((GNUNET_NO == 
+             GNUNET_CONTAINER_multihashmap_contains
+             (peer->details.remote.slave->reghost_map, &hash))
+            || (GNUNET_SYSERR != GNUNET_CONTAINER_multihashmap_get_multiple
+                (peer->details.remote.slave->reghost_map, &hash,
+                 reghost_match_iterator, &rhc)))
+        {
+          /* create and add a new registerd host context */
+          /* add the focc to its queue */
+          GNUNET_CONTAINER_multihashmap_put
+              (peer->details.remote.slave->reghost_map,
+               &hash, rhc, GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+          GNUNET_assert (NULL != host_list[peer2_host_id]);
+          queue_host_registration (peer->details.remote.slave,
+                                   registeredhost_registration_completion, rhc,
+                                   host_list[peer2_host_id]);
+        }
+        else {
+          /* rhc is now set to the existing one from the hash map by
+             reghost_match_iterator() */
+          /* if queue is empty then ignore creating focc and proceed with
+             normal forwarding */
+          if (NULL == rhc->focc_dll_head)
+            skip_focc = GNUNET_YES;
+        }
+        if (GNUNET_NO == skip_focc)
+        {
+          struct ForwardedOverlayConnectContext *focc;
+
+          focc = GNUNET_malloc (sizeof (struct ForwardedOverlayConnectContext));
+          focc->peer1 = p1;
+          focc->peer2 = p2;
+          focc->peer2_host_id = peer2_host_id;
+          focc->orig_msg = GNUNET_copy_message (message);
+          focc->operation_id = operation_id;
+          GNUNET_CONTAINER_DLL_insert_tail (rhc->focc_dll_head,
+                                            rhc->focc_dll_tail,
+                                            focc);
+          GNUNET_SERVER_receive_done (client, GNUNET_OK);
+          return;
+        }
       }
     }
+    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
+    GNUNET_SERVER_client_keep (client);
+    fopc->client = client;
+    fopc->operation_id = operation_id;
+    fopc->type = OP_OVERLAY_CONNECT;
     fopc->opc = 
-       GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.controller,
+       GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                               operation_id, message,
-                                              &forwarded_overlay_connect_listener,
+                                              &forwarded_operation_reply_relay,
                                               fopc);
     fopc->timeout_task =
        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                      fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));
-  GNUNET_SERVER_client_keep (client);
-  occ->client = client;
-  occ->peer_id = p1;
-  occ->other_peer_id = p2;
-  occ->peer = peer_list[p1];
-  occ->op_id = GNUNET_ntohll (msg->operation_id);  
+
+  peer2_controller = NULL;
   if ((p2 >= peer_list_size) || (NULL == peer_list[p2]))
   {   
     if ((peer2_host_id >= slave_list_size)
@@ -2730,7 +3323,8 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     {
       struct GNUNET_TESTBED_NeedControllerConfig *reply;
 
-      GNUNET_free (occ);
+      LOG_DEBUG ("Need controller configuration for connecting peers %u and %u\n",
+                p1, p2);
       reply = GNUNET_malloc (sizeof (struct
                                      GNUNET_TESTBED_NeedControllerConfig)); 
       reply->header.size = htons (sizeof (struct
@@ -2744,12 +3338,11 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     }
     else
     {
-      occ->peer2_controller = slave_list[peer2_host_id]->controller;
-      if (NULL == occ->peer2_controller)
+      //occ->peer2_controller = slave_list[peer2_host_id]->controller;
+      peer2_controller = slave_list[peer2_host_id]->controller;
+      if (NULL == peer2_controller)
       {
         GNUNET_break (0);       /* What's going on? */
-        GNUNET_SERVER_client_drop (client);
-        GNUNET_free (occ);
         GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
         return;
       }
@@ -2757,9 +3350,18 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
   }
   else
   {
-    if (GNUNET_YES == peer_list[occ->other_peer_id]->is_remote)
-      occ->peer2_controller = peer_list[occ->other_peer_id]->details.remote.controller;
+    if (GNUNET_YES == peer_list[p2]->is_remote)
+      peer2_controller = peer_list[p2]->details.remote.slave->controller;
   }
+  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));
+  GNUNET_CONTAINER_DLL_insert_tail (occq_head, occq_tail, occ);
+  GNUNET_SERVER_client_keep (client);
+  occ->client = client;
+  occ->peer_id = p1;
+  occ->other_peer_id = p2;
+  occ->peer = peer_list[p1];
+  occ->op_id = GNUNET_ntohll (msg->operation_id);
+  occ->peer2_controller = peer2_controller;
   /* Get the identity of the second peer */
   if (NULL != occ->peer2_controller)
   {
@@ -2788,6 +3390,7 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
                                    &occ->other_peer_identity);
   /* Connect to the core of 1st peer and wait for the 2nd peer to connect */
   occ->emsg = GNUNET_strdup ("Timeout while connecting to CORE");
+  occ->peer->reference_cnt++;
   occ->ch =
       GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
                            &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
@@ -2819,6 +3422,7 @@ cleanup_rocc (struct RequestOverlayConnectContext *rocc)
     GNUNET_SCHEDULER_cancel (rocc->timeout_rocc_task_id);
   GNUNET_TRANSPORT_disconnect (rocc->th);
   GNUNET_free_non_null (rocc->hello);
+  GNUNET_CONTAINER_DLL_remove (roccq_head, roccq_tail, rocc);
   GNUNET_free (rocc);
 }
 
@@ -2937,13 +3541,13 @@ handle_overlay_request_connect (void *cls, struct GNUNET_SERVER_Client *client,
   {
     struct GNUNET_MessageHeader *msg2;
     
-    msg2 = GNUNET_malloc (msize);
-    (void) memcpy (msg2, message, msize);
-    GNUNET_TESTBED_queue_message_ (peer->details.remote.controller, msg2);
+    msg2 = GNUNET_copy_message (message);
+    GNUNET_TESTBED_queue_message_ (peer->details.remote.slave->controller, msg2);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
   rocc = GNUNET_malloc (sizeof (struct RequestOverlayConnectContext));
+  GNUNET_CONTAINER_DLL_insert_tail (roccq_head, roccq_tail, rocc);
   rocc->th = GNUNET_TRANSPORT_connect (peer->details.local.cfg, NULL, rocc, 
                                        NULL, &transport_connect_notify, NULL);
   if (NULL == rocc->th)
@@ -2992,6 +3596,7 @@ handle_slave_get_config (void *cls, struct GNUNET_SERVER_Client *client,
   op_id = GNUNET_ntohll (msg->operation_id);
   if ((slave_list_size <= slave_id) || (NULL == slave_list[slave_id]))
   {
+    /* FIXME: Add forwardings for this type of message here.. */
     send_operation_fail_msg (client, op_id, "Slave not found");
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
@@ -3046,6 +3651,44 @@ ss_map_free_iterator (void *cls, const struct GNUNET_HashCode *key, void *value)
 }
 
 
+/**
+ * Iterator for freeing hash map entries in a slave's reghost_map
+ *
+ * @param cls handle to the slave
+ * @param key current key code
+ * @param value value in the hash map
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int 
+reghost_free_iterator (void *cls,
+                       const struct GNUNET_HashCode * key,
+                       void *value)
+{
+  struct Slave *slave = cls;
+  struct RegisteredHostContext *rhc = value;
+  struct ForwardedOverlayConnectContext *focc;
+
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (slave->reghost_map,
+                                                       key, value));
+  while (NULL != (focc = rhc->focc_dll_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (rhc->focc_dll_head,
+                                 rhc->focc_dll_tail,
+                                 focc);
+    cleanup_focc (focc);
+  }
+  if (NULL != rhc->sub_op)
+    GNUNET_TESTBED_operation_cancel (rhc->sub_op);
+  if (NULL != rhc->client)
+  GNUNET_SERVER_client_drop (rhc->client);
+  GNUNET_free (value);
+  return GNUNET_YES;
+}
+
+
 /**
  * Task to clean up and shutdown nicely
  *
@@ -3056,6 +3699,9 @@ static void
 shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct LCFContextQueue *lcfq;
+  struct OverlayConnectContext *occ;
+  struct RequestOverlayConnectContext *rocc;
+  struct ForwardedOperationContext *fopc;
   uint32_t id;
 
   shutdown_task_id = GNUNET_SCHEDULER_NO_TASK;
@@ -3063,6 +3709,32 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   (void) GNUNET_CONTAINER_multihashmap_iterate (ss_map, &ss_map_free_iterator,
                                                 NULL);
   GNUNET_CONTAINER_multihashmap_destroy (ss_map);
+  /* cleanup any remaining forwarded operations */
+  while (NULL != (fopc = fopcq_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
+    GNUNET_TESTBED_forward_operation_msg_cancel_ (fopc->opc);
+    if (GNUNET_SCHEDULER_NO_TASK != fopc->timeout_task)
+      GNUNET_SCHEDULER_cancel (fopc->timeout_task);
+    GNUNET_SERVER_client_drop (fopc->client);
+    switch (fopc->type)
+    {
+    case OP_PEER_CREATE:
+      GNUNET_free (fopc->cls);
+      break;
+    case OP_PEER_START:
+    case OP_PEER_STOP:
+    case OP_PEER_DESTROY:
+    case OP_PEER_INFO:
+    case OP_OVERLAY_CONNECT:
+    case OP_LINK_CONTROLLERS:
+    case OP_GET_SLAVE_CONFIG:
+      break;
+    case OP_FORWARDED:
+      GNUNET_assert (0);
+    };
+    GNUNET_free (fopc);
+  }
   if (NULL != lcfq_head)
   {
     if (GNUNET_SCHEDULER_NO_TASK != lcf_proc_task_id)
@@ -3070,8 +3742,6 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       GNUNET_SCHEDULER_cancel (lcf_proc_task_id);
       lcf_proc_task_id = GNUNET_SCHEDULER_NO_TASK;
     }
-    if (NULL != lcfq_head->lcf->rhandle)
-      GNUNET_TESTBED_cancel_registration (lcfq_head->lcf->rhandle);
   }
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
   for (lcfq = lcfq_head; NULL != lcfq; lcfq = lcfq_head)
@@ -3081,10 +3751,19 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     GNUNET_CONTAINER_DLL_remove (lcfq_head, lcfq_tail, lcfq);
     GNUNET_free (lcfq);
   }
+  while (NULL != (occ = occq_head))
+    cleanup_occ (occ);
+  while (NULL != (rocc = roccq_head))
+    cleanup_rocc (rocc);
   /* Clear peer list */
   for (id = 0; id < peer_list_size; id++)
     if (NULL != peer_list[id])
     {
+      /* If destroy flag is set it means that this peer should have been
+         destroyed by a context which we destroy before */
+      GNUNET_break (GNUNET_NO == peer_list[id]->destroy_flag);
+      /* counter should be zero as we free all contexts before */
+      GNUNET_break (0 == peer_list[id]->reference_cnt);
       if (GNUNET_NO == peer_list[id]->is_remote)
       {
        if (GNUNET_YES == peer_list[id]->details.local.is_running)
@@ -3109,6 +3788,21 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   for (id = 0; id < slave_list_size; id++)
     if (NULL != slave_list[id])
     {
+      struct HostRegistration *hr_entry;
+      
+      while (NULL != (hr_entry = slave_list[id]->hr_dll_head))
+      {
+        GNUNET_CONTAINER_DLL_remove (slave_list[id]->hr_dll_head,
+                                     slave_list[id]->hr_dll_tail,
+                                     hr_entry);
+        GNUNET_free (hr_entry);
+      }
+      if (NULL != slave_list[id]->rhandle)
+        GNUNET_TESTBED_cancel_registration (slave_list[id]->rhandle);
+      (void) GNUNET_CONTAINER_multihashmap_iterate (slave_list[id]->reghost_map,
+                                                    reghost_free_iterator,
+                                                    slave_list[id]);
+      GNUNET_CONTAINER_multihashmap_destroy (slave_list[id]->reghost_map);
       if (NULL != slave_list[id]->cfg)
        GNUNET_CONFIGURATION_destroy (slave_list[id]->cfg);
       if (NULL != slave_list[id]->controller)
@@ -3127,6 +3821,7 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     master_context = NULL;
   }
   GNUNET_free_non_null (hostname);
+  GNUNET_CONFIGURATION_destroy (our_config);
 }
 
 
@@ -3193,14 +3888,16 @@ testbed_run (void *cls, struct GNUNET_SERVER_Handle *server,
 
   GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string 
                 (cfg, "testbed", "HOSTNAME", &hostname));
+  our_config = GNUNET_CONFIGURATION_dup (cfg);
   GNUNET_SERVER_add_handlers (server, message_handlers);
   GNUNET_SERVER_disconnect_notify (server, &client_disconnect_cb, NULL);
   ss_map = GNUNET_CONTAINER_multihashmap_create (5, GNUNET_NO);
   shutdown_task_id =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
-                                    &shutdown_task, NULL);
+      GNUNET_SCHEDULER_add_delayed_with_priority (GNUNET_TIME_UNIT_FOREVER_REL,
+                                                  GNUNET_SCHEDULER_PRIORITY_IDLE,
+                                                  &shutdown_task, NULL);
   LOG_DEBUG ("Testbed startup complete\n");
-  event_mask = GNUNET_TESTBED_ET_OPERATION_FINISHED;
+  event_mask = 1LL << GNUNET_TESTBED_ET_OPERATION_FINISHED;
 }