topology option support for GNUNET_TESTBED_run() via configuration
[oweals/gnunet.git] / src / testbed / gnunet-service-testbed.c
index 90ff872351c10fe357f47138fc00fb92683230b7..ec3e000ee8fdd36cc0cf851a5e53c67fb59d5f93 100644 (file)
@@ -369,6 +369,11 @@ struct LCFContext
    */
   struct GNUNET_SERVER_Client *client;
 
+  /**
+   * Handle for operations which are forwarded while linking controllers
+   */
+  struct ForwardedOperationContext *fopc;
+
   /**
    * The id of the operation which created this context
    */
@@ -468,6 +473,24 @@ struct Peer
    */
   uint32_t id;
 
+  /**
+   * References to peers are using in forwarded overlay contexts and remote
+   * overlay connect contexts. A peer can only be destroyed after all such
+   * contexts are destroyed. For this, we maintain a reference counter. When we
+   * use a peer in any such context, we increment this counter. We decrement it
+   * when we are destroying these contexts
+   */
+  uint32_t reference_cnt;
+
+  /**
+   * While destroying a peer, due to the fact that there could be references to
+   * this peer, we delay the peer destroy to a further time. We do this by using
+   * this flag to destroy the peer while destroying a context in which this peer
+   * has been used. When the flag is set to 1 and reference_cnt = 0 we destroy
+   * the peer
+   */
+  uint32_t destroy_flag;
+
 };
 
 
@@ -577,11 +600,6 @@ struct OverlayConnectContext
    */
   uint32_t other_peer_id;
 
-  /**
-   * Number of times we tried to send hello; used to increase delay in offering
-   * hellos
-   */
-  uint16_t retries;
 };
 
 
@@ -593,6 +611,16 @@ struct OverlayConnectContext
  */
 struct RequestOverlayConnectContext
 {
+  /**
+   * the next pointer for DLL
+   */
+  struct RequestOverlayConnectContext *next;
+
+  /**
+   * the prev pointer for DLL
+   */
+  struct RequestOverlayConnectContext *prev;
+
   /**
    * The transport handle of peer B
    */
@@ -618,12 +646,6 @@ struct RequestOverlayConnectContext
    */
   GNUNET_SCHEDULER_TaskIdentifier timeout_rocc_task_id;
   
-  /**
-   * Number of times we tried to send hello; used to increase delay in offering
-   * hellos
-   */
-  uint16_t retries;
-  
 };
 
 
@@ -632,6 +654,16 @@ struct RequestOverlayConnectContext
  */
 struct ForwardedOperationContext
 {
+  /**
+   * The next pointer for DLL
+   */
+  struct ForwardedOperationContext *next;
+
+  /**
+   * The prev pointer for DLL
+   */
+  struct ForwardedOperationContext *prev;
+  
   /**
    * The generated operation context
    */
@@ -657,6 +689,11 @@ struct ForwardedOperationContext
    */
   uint64_t operation_id;
 
+  /**
+   * The type of the operation which is forwarded
+   */
+  enum OperationType type;
+
 };
 
 
@@ -781,6 +818,27 @@ static struct OverlayConnectContext *occq_head;
  */
 static struct OverlayConnectContext *occq_tail;
 
+/**
+ * DLL head for RequectOverlayConnectContext DLL - to be used to clean up during
+ * shutdown
+ */
+static struct RequestOverlayConnectContext *roccq_head;
+
+/**
+ * DLL tail for RequectOverlayConnectContext DLL
+ */
+static struct RequestOverlayConnectContext *roccq_tail;
+
+/**
+ * DLL head for forwarded operation contexts
+ */
+static struct ForwardedOperationContext *fopcq_head;
+
+/**
+ * DLL tail for forwarded operation contexts
+ */
+static struct ForwardedOperationContext *fopcq_tail;
+
 /**
  * Array of hosts
  */
@@ -1313,11 +1371,10 @@ lcf_proc_cc (void *cls, const char *emsg)
 
 
 /**
- * Callback to be called when forwarded link controllers operation is
- * successfull. We have to relay the reply msg back to the client
+ * Callback to relay the reply msg of a forwarded operation back to the client
  *
  * @param cls ForwardedOperationContext
- * @param msg the peer create success message
+ * @param msg the message to relay
  */
 static void
 forwarded_operation_reply_relay (void *cls,
@@ -1330,17 +1387,17 @@ forwarded_operation_reply_relay (void *cls,
   msize = ntohs (msg->size);
   LOG_DEBUG ("Relaying message with type: %u, size: %u\n", ntohs (msg->type),
              msize);
-  dup_msg = GNUNET_malloc (msize);
-  (void) memcpy (dup_msg, msg, msize);
+  dup_msg = GNUNET_copy_message (msg);
   queue_message (fopc->client, dup_msg);
   GNUNET_SERVER_client_drop (fopc->client);
   GNUNET_SCHEDULER_cancel (fopc->timeout_task);
+  GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
   GNUNET_free (fopc);
 }
 
 
 /**
- * Task to free resources when forwarded link controllers has been timedout
+ * Task to free resources when forwarded operation has been timedout
  *
  * @param cls the ForwardedOperationContext
  * @param tc the task context from scheduler
@@ -1355,10 +1412,63 @@ forwarded_operation_timeout (void *cls,
   LOG (GNUNET_ERROR_TYPE_WARNING, "A forwarded operation has timed out\n");
   send_operation_fail_msg (fopc->client, fopc->operation_id, "Timeout");
   GNUNET_SERVER_client_drop (fopc->client);
+  GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
   GNUNET_free (fopc);
 }
 
 
+/**
+ * The  Link Controller forwarding task
+ *
+ * @param cls the LCFContext
+ * @param tc the Task context from scheduler
+ */
+static void
+lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+
+/**
+ * Callback to be called when forwarded link controllers operation is
+ * successfull. We have to relay the reply msg back to the client
+ *
+ * @param cls the LCFContext
+ * @param msg the message to relay
+ */
+static void
+lcf_forwarded_operation_reply_relay (void *cls,
+                                     const struct GNUNET_MessageHeader *msg)
+{
+  struct LCFContext *lcf = cls;
+
+  GNUNET_assert (NULL != lcf->fopc);
+  forwarded_operation_reply_relay (lcf->fopc, msg);
+  lcf->fopc = NULL;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
+  lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
+}
+
+
+/**
+ * Task to free resources when forwarded link controllers has been timedout
+ *
+ * @param cls the LCFContext
+ * @param tc the task context from scheduler
+ */
+static void
+lcf_forwarded_operation_timeout (void *cls,
+                                 const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct LCFContext *lcf = cls;
+
+  GNUNET_assert (NULL != lcf->fopc);
+  lcf->fopc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  forwarded_operation_timeout (lcf->fopc, tc);
+  lcf->fopc = NULL;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == lcf_proc_task_id);
+  lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
+}
+
+
 /**
  * The  Link Controller forwarding task
  *
@@ -1370,7 +1480,6 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct LCFContext *lcf = cls;
   struct LCFContextQueue *lcfq;
-  struct ForwardedOperationContext *fopc;
 
   lcf_proc_task_id = GNUNET_SCHEDULER_NO_TASK;
   switch (lcf->state)
@@ -1406,20 +1515,21 @@ lcf_proc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     }
     break;
   case SLAVE_HOST_REGISTERED:
-    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-    fopc->client = lcf->client;
-    fopc->operation_id = lcf->operation_id;
-    fopc->opc =
+    lcf->fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
+    lcf->fopc->client = lcf->client;
+    lcf->fopc->operation_id = lcf->operation_id;
+    lcf->fopc->type = OP_LINK_CONTROLLERS;
+    lcf->fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (lcf->gateway->controller,
                                                lcf->operation_id,
                                                &lcf->msg->header,
-                                               &forwarded_operation_reply_relay,
-                                               fopc);
-    fopc->timeout_task =
-        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
-                                      fopc);
+                                               &lcf_forwarded_operation_reply_relay,
+                                               lcf);
+    lcf->fopc->timeout_task =
+        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &lcf_forwarded_operation_timeout,
+                                      lcf);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, lcf->fopc);
     lcf->state = FINISHED;
-    lcf_proc_task_id = GNUNET_SCHEDULER_add_now (&lcf_proc_task, lcf);
     break;
   case FINISHED:
     lcfq = lcfq_head;
@@ -1529,6 +1639,7 @@ process_next_focc (struct RegisteredHostContext *rhc)
   fopc->client = rhc->client;  
   fopc->operation_id = focc->operation_id;
   fopc->cls = rhc;
+  fopc->type = OP_OVERLAY_CONNECT;
   fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (rhc->gateway->controller,
                                                focc->operation_id, focc->orig_msg,
@@ -1539,6 +1650,7 @@ process_next_focc (struct RegisteredHostContext *rhc)
   fopc->timeout_task =
       GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_overlay_connect_timeout,
                                     fopc);
+  GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
 }
 
 
@@ -1694,7 +1806,7 @@ handle_init (void *cls, struct GNUNET_SERVER_Client *client,
   master_context->client = client;
   master_context->host_id = ntohl (msg->host_id);
   master_context->master_ip = GNUNET_strdup (controller_hostname);
-  LOG_DEBUG ("Master Controller IP: %s\n", master_context->master_ip);
+  LOG_DEBUG ("Our IP: %s\n", master_context->master_ip);
   master_context->system =
       GNUNET_TESTING_system_create ("testbed", master_context->master_ip, hostname);
   host =
@@ -2094,14 +2206,10 @@ static void
 peer_create_forward_timeout (void *cls,
                              const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct ForwardedOperationContext *fo_ctxt = cls;
+  struct ForwardedOperationContext *fopc = cls;
 
-  /* send error msg to client */
-  GNUNET_free (fo_ctxt->cls);
-  send_operation_fail_msg (fo_ctxt->client, fo_ctxt->operation_id, "Timedout");
-  GNUNET_SERVER_client_drop (fo_ctxt->client);
-  GNUNET_TESTBED_forward_operation_msg_cancel_ (fo_ctxt->opc);
-  GNUNET_free (fo_ctxt);
+  GNUNET_free (fopc->cls);
+  forwarded_operation_timeout (fopc, tc);
 }
 
 
@@ -2115,21 +2223,43 @@ peer_create_forward_timeout (void *cls,
 static void
 peer_create_success_cb (void *cls, const struct GNUNET_MessageHeader *msg)
 {
-  struct ForwardedOperationContext *fo_ctxt = cls;
-  struct GNUNET_MessageHeader *dup_msg;
+  struct ForwardedOperationContext *fopc = cls;
   struct Peer *remote_peer;
 
-  GNUNET_SCHEDULER_cancel (fo_ctxt->timeout_task);
   if (ntohs (msg->type) == GNUNET_MESSAGE_TYPE_TESTBED_PEERCREATESUCCESS)
   {
-    GNUNET_assert (NULL != fo_ctxt->cls);
-    remote_peer = fo_ctxt->cls;
+    GNUNET_assert (NULL != fopc->cls);
+    remote_peer = fopc->cls;
     peer_list_add (remote_peer);
   }
-  dup_msg = GNUNET_copy_message (msg);
-  queue_message (fo_ctxt->client, dup_msg);
-  GNUNET_SERVER_client_drop (fo_ctxt->client);
-  GNUNET_free (fo_ctxt);
+  forwarded_operation_reply_relay (fopc, msg);
+}
+
+
+/**
+ * Function to destroy a peer
+ *
+ * @param peer the peer structure to destroy
+ */
+static void
+destroy_peer (struct Peer *peer)
+{
+  GNUNET_break (0 == peer->reference_cnt);
+  if (GNUNET_YES == peer->is_remote)
+  {
+    peer_list_remove (peer);
+    GNUNET_free (peer);
+    return;
+  }
+  if (GNUNET_YES == peer->details.local.is_running)
+  {
+    GNUNET_TESTING_peer_stop (peer->details.local.peer);
+    peer->details.local.is_running = GNUNET_NO;
+  }
+  GNUNET_TESTING_peer_destroy (peer->details.local.peer);
+  GNUNET_CONFIGURATION_destroy (peer->details.local.cfg);
+  peer_list_remove (peer);
+  GNUNET_free (peer);
 }
 
 
@@ -2143,21 +2273,18 @@ peer_create_success_cb (void *cls, const struct GNUNET_MessageHeader *msg)
 static void
 peer_destroy_success_cb (void *cls, const struct GNUNET_MessageHeader *msg)
 {
-  struct ForwardedOperationContext *fo_ctxt = cls;
-  struct GNUNET_MessageHeader *dup_msg;
+  struct ForwardedOperationContext *fopc = cls;
   struct Peer *remote_peer;
 
-  GNUNET_SCHEDULER_cancel (fo_ctxt->timeout_task);  
   if (GNUNET_MESSAGE_TYPE_TESTBED_GENERICOPSUCCESS == ntohs (msg->type))
   {
-    remote_peer = fo_ctxt->cls;
+    remote_peer = fopc->cls;
     GNUNET_assert (NULL != remote_peer);
-    peer_list_remove (remote_peer);
+    remote_peer->destroy_flag = GNUNET_YES;
+    if (0 == remote_peer->reference_cnt)
+      destroy_peer (remote_peer);
   }
-  dup_msg = GNUNET_copy_message (msg);
-  queue_message (fo_ctxt->client, dup_msg);
-  GNUNET_SERVER_client_drop (fo_ctxt->client);
-  GNUNET_free (fo_ctxt);
+  forwarded_operation_reply_relay (fopc, msg);
 }
 
 
@@ -2293,6 +2420,7 @@ handle_peer_create (void *cls, struct GNUNET_SERVER_Client *client,
   fo_ctxt->client = client;
   fo_ctxt->operation_id = GNUNET_ntohll (msg->operation_id);
   fo_ctxt->cls = peer; //slave_list[route->dest]->controller;
+  fo_ctxt->type = OP_PEER_CREATE;
   fo_ctxt->opc =
       GNUNET_TESTBED_forward_operation_msg_ (slave_list [route->dest]->controller,
                                              fo_ctxt->operation_id,
@@ -2301,6 +2429,7 @@ handle_peer_create (void *cls, struct GNUNET_SERVER_Client *client,
   fo_ctxt->timeout_task =
       GNUNET_SCHEDULER_add_delayed (TIMEOUT, &peer_create_forward_timeout,
                                     fo_ctxt);
+  GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fo_ctxt);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
 
@@ -2342,6 +2471,7 @@ handle_peer_destroy (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->cls = peer;
+    fopc->type = OP_PEER_DESTROY;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);    
     fopc->opc = 
         GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
@@ -2351,18 +2481,16 @@ handle_peer_destroy (void *cls, struct GNUNET_SERVER_Client *client,
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-  if (GNUNET_YES == peer->details.local.is_running)
-  {
-    GNUNET_TESTING_peer_stop (peer->details.local.peer);
-    peer->details.local.is_running = GNUNET_NO;
-  }
-  GNUNET_TESTING_peer_destroy (peer->details.local.peer);
-  GNUNET_CONFIGURATION_destroy (peer->details.local.cfg);
-  peer_list_remove (peer);
-  GNUNET_free (peer);
+  peer->destroy_flag = GNUNET_YES;
+  if (0 == peer->reference_cnt)
+    destroy_peer (peer);
+  else
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+         "Delaying peer destroy as peer is currently in use\n");
   send_operation_success_msg (client, GNUNET_ntohll (msg->operation_id));
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
@@ -2402,6 +2530,7 @@ handle_peer_start (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_START;
     fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
@@ -2410,6 +2539,7 @@ handle_peer_start (void *cls, struct GNUNET_SERVER_Client *client,
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2466,6 +2596,7 @@ handle_peer_stop (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_STOP;
     fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
@@ -2474,6 +2605,7 @@ handle_peer_stop (void *cls, struct GNUNET_SERVER_Client *client,
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                       fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2536,6 +2668,7 @@ handle_peer_get_config (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = GNUNET_ntohll (msg->operation_id);
+    fopc->type = OP_PEER_INFO;
     fopc->opc =
         GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                                fopc->operation_id, &msg->header,
@@ -2543,7 +2676,8 @@ handle_peer_get_config (void *cls, struct GNUNET_SERVER_Client *client,
                                                fopc);
     fopc->timeout_task =
         GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
-                                      fopc);    
+                                      fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc); 
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -2590,13 +2724,29 @@ cleanup_occ (struct OverlayConnectContext *occ)
   if (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task)
     GNUNET_SCHEDULER_cancel (occ->timeout_task);
   if (NULL != occ->ch)
+  {
     GNUNET_CORE_disconnect (occ->ch);
+    occ->peer->reference_cnt--;
+  }
   if (NULL != occ->ghh)
     GNUNET_TRANSPORT_get_hello_cancel (occ->ghh);
   if (NULL != occ->p1th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p1th);
+    occ->peer->reference_cnt--;
+  }
   if (NULL != occ->p2th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p2th);
+    peer_list[occ->other_peer_id]->reference_cnt--;
+  }
+  if ((GNUNET_YES == occ->peer->destroy_flag)
+      && (0 == occ->peer->reference_cnt))
+    destroy_peer (occ->peer);
+  if ((NULL == occ->peer2_controller)
+      && (GNUNET_YES == peer_list[occ->other_peer_id]->destroy_flag)
+        && (0 == peer_list[occ->other_peer_id]->reference_cnt))
+      destroy_peer (peer_list[occ->other_peer_id]);  
   GNUNET_CONTAINER_DLL_remove (occq_head, occq_tail, occ);
   GNUNET_free (occ);
 }
@@ -2691,7 +2841,10 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = NULL;
   if (NULL != occ->p2th)
+  {
     GNUNET_TRANSPORT_disconnect (occ->p2th);
+    peer_list[occ->other_peer_id]->reference_cnt--;
+  }
   occ->p2th = NULL;
   LOG_DEBUG ("Peers connected - Sending overlay connect success\n");
   msg = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
@@ -2755,7 +2908,8 @@ send_hello (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     occ->send_hello_task =
         GNUNET_SCHEDULER_add_delayed
         (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                        100 * (pow (2, occ->retries++))),
+                                        100 + GNUNET_CRYPTO_random_u32
+                                        (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
          &send_hello, occ);
   }
   GNUNET_free (other_peer_str);  
@@ -2812,9 +2966,11 @@ hello_update_cb (void *cls, const struct GNUNET_MessageHeader *hello)
   occ->ghh = NULL;
   GNUNET_TRANSPORT_disconnect (occ->p1th);
   occ->p1th = NULL;
+  occ->peer->reference_cnt--;
   GNUNET_free_non_null (occ->emsg);
   if (NULL == occ->peer2_controller)
-  {   
+  {
+    peer_list[occ->other_peer_id]->reference_cnt++;
     occ->p2th =
        GNUNET_TRANSPORT_connect (peer_list[occ->other_peer_id]->details.local.cfg,
                                  &occ->other_peer_identity, NULL, NULL, NULL,
@@ -2859,6 +3015,7 @@ core_startup_cb (void *cls, struct GNUNET_CORE_Handle *server,
   occ->emsg = NULL;
   memcpy (&occ->peer_identity, my_identity,
           sizeof (struct GNUNET_PeerIdentity));
+  occ->peer->reference_cnt++;
   occ->p1th =
       GNUNET_TRANSPORT_connect (occ->peer->details.local.cfg,
                                 &occ->peer_identity, NULL, NULL, NULL, NULL);
@@ -2906,6 +3063,7 @@ overlay_connect_get_config (void *cls, const struct GNUNET_MessageHeader *msg)
          sizeof (struct GNUNET_PeerIdentity));
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = GNUNET_strdup ("Timeout while connecting to CORE");
+  occ->peer->reference_cnt++;
   occ->ch =
       GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
                            &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
@@ -3056,12 +3214,16 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     if ((NULL != route_to_peer2_host) 
         || (peer2_host_id == master_context->host_id))
     {
+      /* Peer 2 either below us OR with us */
       route_to_peer1_host = 
           find_dest_route (peer_list[p1]->details.remote.remote_host_id);
+      /* Because we get this message only if we know where peer 1 is */
       GNUNET_assert (NULL != route_to_peer1_host);
       if ((peer2_host_id == master_context->host_id) 
           || (route_to_peer2_host->dest != route_to_peer1_host->dest))
       {
+        /* Peer2 is either with us OR peer1 and peer2 can be reached through
+           different gateways */
         struct GNUNET_HashCode hash;
         struct RegisteredHostContext *rhc;
         int skip_focc;
@@ -3101,24 +3263,21 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
         }
         else {
           /* rhc is now set to the existing one from the hash map by
-             reghost_match_iterator */
+             reghost_match_iterator() */
           /* if queue is empty then ignore creating focc and proceed with
-             normal forwarding */
-          if (NULL == rhc->focc_dll_head)
+             normal forwarding */                
+          if (RHC_OL_CONNECT == rhc->state)
             skip_focc = GNUNET_YES;
         }
         if (GNUNET_NO == skip_focc)
         {
           struct ForwardedOverlayConnectContext *focc;
-          uint16_t msize;
 
-          msize = sizeof (struct GNUNET_TESTBED_OverlayConnectMessage);
           focc = GNUNET_malloc (sizeof (struct ForwardedOverlayConnectContext));
           focc->peer1 = p1;
           focc->peer2 = p2;
           focc->peer2_host_id = peer2_host_id;
-          focc->orig_msg = GNUNET_malloc (msize);
-          (void) memcpy (focc->orig_msg, message, msize);
+          focc->orig_msg = GNUNET_copy_message (message);
           focc->operation_id = operation_id;
           GNUNET_CONTAINER_DLL_insert_tail (rhc->focc_dll_head,
                                             rhc->focc_dll_tail,
@@ -3132,6 +3291,7 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     GNUNET_SERVER_client_keep (client);
     fopc->client = client;
     fopc->operation_id = operation_id;
+    fopc->type = OP_OVERLAY_CONNECT;
     fopc->opc = 
        GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
                                               operation_id, message,
@@ -3140,6 +3300,7 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     fopc->timeout_task =
        GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_operation_timeout,
                                      fopc);
+    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
@@ -3150,31 +3311,18 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
     if ((peer2_host_id >= slave_list_size)
        || (NULL ==slave_list[peer2_host_id]))
     {
-      struct GNUNET_TESTBED_NeedControllerConfig *reply;
-
-      LOG_DEBUG ("Need controller configuration for connecting peers %u and %u\n",
-                p1, p2);
-      reply = GNUNET_malloc (sizeof (struct
-                                     GNUNET_TESTBED_NeedControllerConfig)); 
-      reply->header.size = htons (sizeof (struct
-                                          GNUNET_TESTBED_NeedControllerConfig));
-      reply->header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_NEEDCONTROLLERCONFIG);
-      reply->controller_host_id = msg->peer2_host_id;
-      reply->operation_id = msg->operation_id;
-      queue_message (client, &reply->header);      
-      GNUNET_SERVER_receive_done (client, GNUNET_OK);
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           "Configuration of peer2's controller missing for connecting peers"
+           "%u and %u\n", p1, p2);      
+      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
       return;
     }
-    else
+    peer2_controller = slave_list[peer2_host_id]->controller;
+    if (NULL == peer2_controller)
     {
-      //occ->peer2_controller = slave_list[peer2_host_id]->controller;
-      peer2_controller = slave_list[peer2_host_id]->controller;
-      if (NULL == peer2_controller)
-      {
-        GNUNET_break (0);       /* What's going on? */
-        GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-        return;
-      }
+      GNUNET_break (0);       /* What's going on? */
+      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+      return;
     }
   }
   else
@@ -3219,6 +3367,7 @@ handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
                                    &occ->other_peer_identity);
   /* Connect to the core of 1st peer and wait for the 2nd peer to connect */
   occ->emsg = GNUNET_strdup ("Timeout while connecting to CORE");
+  occ->peer->reference_cnt++;
   occ->ch =
       GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
                            &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
@@ -3250,6 +3399,7 @@ cleanup_rocc (struct RequestOverlayConnectContext *rocc)
     GNUNET_SCHEDULER_cancel (rocc->timeout_rocc_task_id);
   GNUNET_TRANSPORT_disconnect (rocc->th);
   GNUNET_free_non_null (rocc->hello);
+  GNUNET_CONTAINER_DLL_remove (roccq_head, roccq_tail, rocc);
   GNUNET_free (rocc);
 }
 
@@ -3312,7 +3462,8 @@ attempt_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   rocc->attempt_connect_task_id = 
       GNUNET_SCHEDULER_add_delayed 
       (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                      100 * (pow (2, rocc->retries++))),
+                                      100 + GNUNET_CRYPTO_random_u32
+                                      (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
        &attempt_connect_task, rocc);
 }
 
@@ -3368,13 +3519,13 @@ handle_overlay_request_connect (void *cls, struct GNUNET_SERVER_Client *client,
   {
     struct GNUNET_MessageHeader *msg2;
     
-    msg2 = GNUNET_malloc (msize);
-    (void) memcpy (msg2, message, msize);
+    msg2 = GNUNET_copy_message (message);
     GNUNET_TESTBED_queue_message_ (peer->details.remote.slave->controller, msg2);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
   rocc = GNUNET_malloc (sizeof (struct RequestOverlayConnectContext));
+  GNUNET_CONTAINER_DLL_insert_tail (roccq_head, roccq_tail, rocc);
   rocc->th = GNUNET_TRANSPORT_connect (peer->details.local.cfg, NULL, rocc, 
                                        NULL, &transport_connect_notify, NULL);
   if (NULL == rocc->th)
@@ -3423,6 +3574,7 @@ handle_slave_get_config (void *cls, struct GNUNET_SERVER_Client *client,
   op_id = GNUNET_ntohll (msg->operation_id);
   if ((slave_list_size <= slave_id) || (NULL == slave_list[slave_id]))
   {
+    /* FIXME: Add forwardings for this type of message here.. */
     send_operation_fail_msg (client, op_id, "Slave not found");
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
@@ -3507,7 +3659,7 @@ reghost_free_iterator (void *cls,
     cleanup_focc (focc);
   }
   if (NULL != rhc->sub_op)
-    GNUNET_TESTBED_operation_cancel (rhc->sub_op);
+    GNUNET_TESTBED_operation_done (rhc->sub_op);
   if (NULL != rhc->client)
   GNUNET_SERVER_client_drop (rhc->client);
   GNUNET_free (value);
@@ -3526,6 +3678,8 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct LCFContextQueue *lcfq;
   struct OverlayConnectContext *occ;
+  struct RequestOverlayConnectContext *rocc;
+  struct ForwardedOperationContext *fopc;
   uint32_t id;
 
   shutdown_task_id = GNUNET_SCHEDULER_NO_TASK;
@@ -3533,6 +3687,32 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   (void) GNUNET_CONTAINER_multihashmap_iterate (ss_map, &ss_map_free_iterator,
                                                 NULL);
   GNUNET_CONTAINER_multihashmap_destroy (ss_map);
+  /* cleanup any remaining forwarded operations */
+  while (NULL != (fopc = fopcq_head))
+  {
+    GNUNET_CONTAINER_DLL_remove (fopcq_head, fopcq_tail, fopc);
+    GNUNET_TESTBED_forward_operation_msg_cancel_ (fopc->opc);
+    if (GNUNET_SCHEDULER_NO_TASK != fopc->timeout_task)
+      GNUNET_SCHEDULER_cancel (fopc->timeout_task);
+    GNUNET_SERVER_client_drop (fopc->client);
+    switch (fopc->type)
+    {
+    case OP_PEER_CREATE:
+      GNUNET_free (fopc->cls);
+      break;
+    case OP_PEER_START:
+    case OP_PEER_STOP:
+    case OP_PEER_DESTROY:
+    case OP_PEER_INFO:
+    case OP_OVERLAY_CONNECT:
+    case OP_LINK_CONTROLLERS:
+    case OP_GET_SLAVE_CONFIG:
+      break;
+    case OP_FORWARDED:
+      GNUNET_assert (0);
+    };
+    GNUNET_free (fopc);
+  }
   if (NULL != lcfq_head)
   {
     if (GNUNET_SCHEDULER_NO_TASK != lcf_proc_task_id)
@@ -3551,10 +3731,17 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   }
   while (NULL != (occ = occq_head))
     cleanup_occ (occ);
+  while (NULL != (rocc = roccq_head))
+    cleanup_rocc (rocc);
   /* Clear peer list */
   for (id = 0; id < peer_list_size; id++)
     if (NULL != peer_list[id])
     {
+      /* If destroy flag is set it means that this peer should have been
+         destroyed by a context which we destroy before */
+      GNUNET_break (GNUNET_NO == peer_list[id]->destroy_flag);
+      /* counter should be zero as we free all contexts before */
+      GNUNET_break (0 == peer_list[id]->reference_cnt);
       if (GNUNET_NO == peer_list[id]->is_remote)
       {
        if (GNUNET_YES == peer_list[id]->details.local.is_running)
@@ -3630,7 +3817,6 @@ client_disconnect_cb (void *cls, struct GNUNET_SERVER_Client *client)
   if (client == master_context->client)
   {
     LOG (GNUNET_ERROR_TYPE_DEBUG, "Master client disconnected\n");
-    GNUNET_SERVER_client_drop (client);
     /* should not be needed as we're terminated by failure to read
      * from stdin, but if stdin fails for some reason, this shouldn't
      * hurt for now --- might need to revise this later if we ever