- towards testbed barriers
[oweals/gnunet.git] / src / testbed / gnunet-service-testbed_oc.c
index f5811194b6b91f46f1eba404d348a27c6b1f6367..b98f319a21d080e3ebd29de15d8d8d31e0d0ebdf 100644 (file)
@@ -1,10 +1,10 @@
 /*
   This file is part of GNUnet.
-  (C) 2012 Christian Grothoff (and other contributing authors)
+  (C) 2008--2013 Christian Grothoff (and other contributing authors)
 
   GNUnet is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published
-  by the Free Software Foundation; either version 2, or (at your
+  by the Free Software Foundation; either version 3, or (at your
   option) any later version.
 
   GNUnet is distributed in the hope that it will be useful, but
  */
 
 #include "gnunet-service-testbed.h"
-#include "testbed_api_operations.h"
+
+/**
+ * Redefine LOG with a changed log component string
+ */
+#ifdef LOG
+#undef LOG
+#endif
+#define LOG(kind,...)                                   \
+  GNUNET_log_from (kind, "testbed-OC", __VA_ARGS__)
 
 
 /**
- * Context information for transport try connect
+ * Context information for requesting TRANSPORT to connect to a peer
  */
 struct TryConnectContext
 {
@@ -39,9 +47,14 @@ struct TryConnectContext
   struct GNUNET_PeerIdentity *pid;
 
   /**
-   * The transport handle
+   * The transport handle obtained from cache. Do NOT close/disconnect.
+   */
+  struct GNUNET_TRANSPORT_Handle *th_;
+
+  /**
+   * The GetCacheHandle for the p1th transport handle
    */
-  struct GNUNET_TRANSPORT_Handle *th;
+  struct GSTCacheGetHandle *cgh_th;
 
   /**
    * the try connect handle
@@ -67,54 +80,133 @@ struct TryConnectContext
 
 
 /**
- * Context information for connecting 2 peers in overlay
+ * Types for context information we create for overlay connect requests
+ */
+enum OverlayConnectContextType
+{
+  /**
+   * This type is used if the overlay connection is local i.e. the connection
+   * has to be made between local peers
+   */
+  OCC_TYPE_LOCAL,
+  
+  /**
+   * Type to be used when the first peer is local and the other peer is on a slave
+   * controller started by us
+   */
+  OCC_TYPE_REMOTE_SLAVE,
+
+  /**
+   * Type to be used when the first peer is local and the other peer is on a
+   * controller which is not started by us.
+   */
+  OCC_TYPE_REMOTE_LATERAL
+};
+
+
+/**
+ * Context data for operations on second peer in local overlay connection
+ * contexts
+ */
+struct LocalPeer2Context
+{   
+  /**
+   * The handle for offering the HELLO of the first peer to the second
+   * peer.
+   */
+  struct GNUNET_TRANSPORT_OfferHelloHandle *ohh;
+
+  /**
+   * The transport TryConnectContext
+   */
+  struct TryConnectContext tcc;
+};
+
+
+/**
+ * Context data for operations on second peer in remote overlay connection
+ * contexts
+ */
+struct RemotePeer2Context
+{  
+  /**
+   * Controller of peer 2; If OCC_TYPE_REMOTE_LATERAL is the type of overlay
+   * connection then this can be NULL until the connection to the controller is
+   * established
+   */
+  struct GNUNET_TESTBED_Controller *p2c;
+  
+  /**
+   * Operation context for the suboperation we start to get the identity of the
+   * second peer
+   */
+  struct OperationContext *opc;
+  
+  /**
+   * Notification handle acquire to connect to a remote controller.  Only used
+   * if the type of overlay connection is OCC_TYPE_REMOTE_LATERAL.
+   */
+  struct NeighbourConnectNotification *ncn;
+
+  /**
+   * The neighbour handle.  Only used if the type of overlay connection is
+   * OCC_TYPE_REMOTE_LATERAL.
+   */
+  struct Neighbour *p2n;  
+};
+
+/**
+ * Context information for connecting 2 peers in overlay.
  */
 struct OverlayConnectContext
 {
   /**
-   * The next pointer for maintaining a DLL
+   * The next pointer for maintaining a DLL of all OverlayConnectContexts
    */
   struct OverlayConnectContext *next;
 
   /**
-   * The prev pointer for maintaining a DLL
+   * The prev pointer for maintaining a DLL of all OverlayConnectContexts
    */
   struct OverlayConnectContext *prev;
-  
+
   /**
-   * The client which has requested for overlay connection
+   * The client which has requested for overlay connection. This is used to send
+   * either a success of failure message
    */
   struct GNUNET_SERVER_Client *client;
 
   /**
-   * the peer which has to connect to the other peer
+   * the first peer which is to expect an overlay connection from the second peer.
    */
   struct Peer *peer;
 
   /**
-   * Transport handle of the first peer to get its HELLO
+   * Transport handle of the first peer obtained from cache to get its HELLO. Do
+   * NOT close/disconnect.
    */
-  struct GNUNET_TRANSPORT_Handle *p1th;
+  struct GNUNET_TRANSPORT_Handle *p1th_;
 
   /**
-   * Core handles of the first peer; used to notify when second peer connects to it
+   * The CacheGetHandle for the p1th transport handle
    */
-  struct GNUNET_CORE_Handle *ch;
+  struct GSTCacheGetHandle *cgh_p1th;
 
   /**
-   * HELLO of the other peer
+   * The GetCacheHandle for registering callback to notify CORE level peer
+   * connects and to get our identity.
    */
-  struct GNUNET_MessageHeader *hello;
+  struct GSTCacheGetHandle *cgh_ch;
 
   /**
-   * Get hello handle to acquire HELLO of first peer
+   * HELLO of the first peer.  This should be sent to the second peer.
    */
-  struct GNUNET_TRANSPORT_GetHelloHandle *ghh;
+  struct GNUNET_MessageHeader *hello;
 
   /**
-   * The handle for offering HELLO
+   * Get GetHelloHandle to acquire a HELLO of the first peer
    */
-  struct GNUNET_TRANSPORT_OfferHelloHandle *ohh;
+  struct GNUNET_TRANSPORT_GetHelloHandle *ghh;
 
   /**
    * The error message we send if this overlay connect operation has timed out
@@ -122,19 +214,21 @@ struct OverlayConnectContext
   char *emsg;
 
   /**
-   * Operation context for suboperations
+   * Context information for operations on the second peer
    */
-  struct OperationContext *opc;
+  union {
 
-  /**
-   * Controller of peer 2; NULL if the peer is local
-   */
-  struct GNUNET_TESTBED_Controller *peer2_controller;
+    /**
+     * Context information to be used if the second peer is local
+     */
+    struct LocalPeer2Context local;
 
-  /**
-   * The transport try connect context
-   */
-  struct TryConnectContext tcc;
+    /**
+     * Context information to be used if the second peer is remote
+     */
+    struct RemotePeer2Context remote;
+
+  } p2ctx;
 
   /**
    * The peer identity of the first peer
@@ -168,41 +262,41 @@ struct OverlayConnectContext
   GNUNET_SCHEDULER_TaskIdentifier cleanup_task;
 
   /**
-   * The id of peer A
+   * The type of this context information
    */
-  uint32_t peer_id;
+  enum OverlayConnectContextType type;
 
   /**
-   * The id of peer B
+   * The id of the second peer which is has to connect to the first peer
    */
   uint32_t other_peer_id;
-
 };
 
 
 /**
- * Context information for RequestOverlayConnect
- * operations. RequestOverlayConnect is used when peers A, B reside on different
- * hosts and the host controller for peer B is asked by the host controller of
- * peer A to make peer B connect to peer A
+ * Context information for remote overlay connect operations.  Remote overlay
+ * connections are used when peers A and B reside on different hosts.  In these
+ * operations the host controller for peer B is asked by the host controller of
+ * peer A to make peer B connect to peer A by sending the controller of peer B
+ * the HELLO of peer A.
  */
-struct RequestOverlayConnectContext
+struct RemoteOverlayConnectCtx
 {
   /**
    * the next pointer for DLL
    */
-  struct RequestOverlayConnectContext *next;
+  struct RemoteOverlayConnectCtx *next;
 
   /**
    * the prev pointer for DLL
    */
-  struct RequestOverlayConnectContext *prev;
+  struct RemoteOverlayConnectCtx *prev;
 
   /**
    * The peer handle of peer B
    */
   struct Peer *peer;
-  
+
   /**
    * Peer A's HELLO
    */
@@ -227,12 +321,12 @@ struct RequestOverlayConnectContext
    * Task for offering HELLO of A to B and doing try_connect
    */
   GNUNET_SCHEDULER_TaskIdentifier attempt_connect_task_id;
-  
+
   /**
    * Task to timeout RequestOverlayConnect
    */
   GNUNET_SCHEDULER_TaskIdentifier timeout_rocc_task_id;
-  
+
   /**
    * The id of the operation responsible for creating this context
    */
@@ -254,12 +348,12 @@ static struct OverlayConnectContext *occq_tail;
  * DLL head for RequectOverlayConnectContext DLL - to be used to clean up during
  * shutdown
  */
-static struct RequestOverlayConnectContext *roccq_head;
+static struct RemoteOverlayConnectCtx *roccq_head;
 
 /**
  * DLL tail for RequectOverlayConnectContext DLL
  */
-static struct RequestOverlayConnectContext *roccq_tail;
+static struct RemoteOverlayConnectCtx *roccq_tail;
 
 
 /**
@@ -270,6 +364,7 @@ static struct RequestOverlayConnectContext *roccq_tail;
 void
 GST_cleanup_focc (struct ForwardedOverlayConnectContext *focc)
 {
+  GNUNET_SERVER_client_drop (focc->client);  
   GNUNET_free_non_null (focc->orig_msg);
   GNUNET_free (focc);
 }
@@ -289,14 +384,14 @@ forwarded_overlay_connect_timeout (void *cls,
   struct ForwardedOperationContext *fopc = cls;
   struct RegisteredHostContext *rhc;
   struct ForwardedOverlayConnectContext *focc;
-  
+
   rhc = fopc->cls;
   focc = rhc->focc_dll_head;
   GNUNET_CONTAINER_DLL_remove (rhc->focc_dll_head, rhc->focc_dll_tail, focc);
+  LOG_DEBUG ("Overlay linking between peers %u and %u failed\n", focc->peer1,
+             focc->peer2);
   GST_cleanup_focc (focc);
-  LOG_DEBUG ("Overlay linking between peers %u and %u failed\n",
-            focc->peer1, focc->peer2);
-  GST_forwarded_operation_timeout (cls, tc);
+  GST_forwarded_operation_timeout (fopc, tc);
   if (NULL != rhc->focc_dll_head)
     GST_process_next_focc (rhc);
 }
@@ -317,7 +412,7 @@ forwarded_overlay_connect_listener (void *cls,
   struct ForwardedOperationContext *fopc = cls;
   struct RegisteredHostContext *rhc;
   struct ForwardedOverlayConnectContext *focc;
-  
+
   rhc = fopc->cls;
   GST_forwarded_operation_reply_relay (cls, msg);
   focc = rhc->focc_dll_head;
@@ -338,30 +433,81 @@ GST_process_next_focc (struct RegisteredHostContext *rhc)
 {
   struct ForwardedOperationContext *fopc;
   struct ForwardedOverlayConnectContext *focc;
+  struct Peer *peer;
+  struct Slave *slave;
 
   focc = rhc->focc_dll_head;
   GNUNET_assert (NULL != focc);
-  GNUNET_assert (RHC_OL_CONNECT == rhc->state);
+  GNUNET_assert (RHC_DONE == rhc->state);
+  GNUNET_assert (VALID_PEER_ID (focc->peer1));
+  peer = GST_peer_list[focc->peer1];
+  GNUNET_assert (GNUNET_YES == peer->is_remote);
+  GNUNET_assert (NULL != (slave = peer->details.remote.slave));
   fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-  GNUNET_SERVER_client_keep (rhc->client);
-  fopc->client = rhc->client;  
+  GNUNET_SERVER_client_keep (focc->client);
+  fopc->client = focc->client;
   fopc->operation_id = focc->operation_id;
   fopc->cls = rhc;
   fopc->type = OP_OVERLAY_CONNECT;
   fopc->opc =
-      GNUNET_TESTBED_forward_operation_msg_ (rhc->gateway->controller,
+      GNUNET_TESTBED_forward_operation_msg_ (slave->controller,
                                              focc->operation_id, focc->orig_msg,
                                              &forwarded_overlay_connect_listener,
                                              fopc);
   GNUNET_free (focc->orig_msg);
   focc->orig_msg = NULL;
   fopc->timeout_task =
-      GNUNET_SCHEDULER_add_delayed (TIMEOUT, &forwarded_overlay_connect_timeout,
+      GNUNET_SCHEDULER_add_delayed (GST_timeout, &forwarded_overlay_connect_timeout,
                                     fopc);
   GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
 }
 
 
+/**
+ * Cleans up any used handles in local peer2 context
+ *
+ * @param lp2c the local peer2 context information
+ */
+static void
+cleanup_occ_lp2c (struct LocalPeer2Context *lp2c)
+{
+  if (NULL != lp2c->ohh)
+    GNUNET_TRANSPORT_offer_hello_cancel (lp2c->ohh);
+  if (NULL != lp2c->tcc.cgh_th)
+    GST_cache_get_handle_done (lp2c->tcc.cgh_th);
+  if (NULL != lp2c->tcc.tch)
+    GNUNET_TRANSPORT_try_connect_cancel (lp2c->tcc.tch);
+  if (GNUNET_SCHEDULER_NO_TASK != lp2c->tcc.task)
+    GNUNET_SCHEDULER_cancel (lp2c->tcc.task);
+}
+
+
+/**
+ * Cleans up any used handles in remote peer2 context.  Relinquishes the
+ * remote controller connection if it has been established on-demand.
+ *
+ * @param rp2c the remote peer2 context information
+ */
+static void
+cleanup_occ_rp2c (struct RemotePeer2Context *rp2c)
+{
+  if (NULL != rp2c->opc)
+    GNUNET_TESTBED_forward_operation_msg_cancel_ (rp2c->opc);
+  if (NULL != rp2c->ncn)
+    GST_neighbour_get_connection_cancel (rp2c->ncn);
+  if ( (NULL != rp2c->p2c) && (NULL != rp2c->p2n) )
+    GST_neighbour_release_connection (rp2c->p2n);
+
+}
+
+/**
+ * Condition for checking if given peer is ready to be destroyed
+ *
+ * @param peer the peer to check
+ */
+#define PEER_EXPIRED(peer)                      \
+  ( (GNUNET_YES == peer->destroy_flag) && (0 == peer->reference_cnt) )
+
 /**
  * Cleanup overlay connect context structure
  *
@@ -370,48 +516,44 @@ GST_process_next_focc (struct RegisteredHostContext *rhc)
 static void
 cleanup_occ (struct OverlayConnectContext *occ)
 {
+  struct Peer *peer2;
+
   LOG_DEBUG ("0x%llx: Cleaning up occ\n", occ->op_id);
   GNUNET_free_non_null (occ->emsg);
   GNUNET_free_non_null (occ->hello);
   GNUNET_SERVER_client_drop (occ->client);
-  if (NULL != occ->opc)
-    GNUNET_TESTBED_forward_operation_msg_cancel_ (occ->opc);
   if (GNUNET_SCHEDULER_NO_TASK != occ->send_hello_task)
     GNUNET_SCHEDULER_cancel (occ->send_hello_task);
   if (GNUNET_SCHEDULER_NO_TASK != occ->cleanup_task)
     GNUNET_SCHEDULER_cancel (occ->cleanup_task);
   if (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task)
     GNUNET_SCHEDULER_cancel (occ->timeout_task);
-  if (NULL != occ->ch)
-  {
-    GNUNET_CORE_disconnect (occ->ch);
-    occ->peer->reference_cnt--;
-  }
+  if (NULL != occ->cgh_ch)
+    GST_cache_get_handle_done (occ->cgh_ch);
   if (NULL != occ->ghh)
     GNUNET_TRANSPORT_get_hello_cancel (occ->ghh);
-  if (NULL != occ->ohh)
-    GNUNET_TRANSPORT_offer_hello_cancel (occ->ohh);
-  if (GNUNET_SCHEDULER_NO_TASK != occ->tcc.task)
-    GNUNET_SCHEDULER_cancel (occ->tcc.task);
-  if (NULL != occ->tcc.tch)
-    GNUNET_TRANSPORT_try_connect_cancel (occ->tcc.tch);
-  if (NULL != occ->p1th)
-  {
-    GNUNET_TRANSPORT_disconnect (occ->p1th);
-    occ->peer->reference_cnt--;
-  }
-  if (NULL != occ->tcc.th)
+  if (NULL != occ->cgh_p1th)
+    GST_cache_get_handle_done (occ->cgh_p1th);
+  GNUNET_assert (NULL != GST_peer_list);
+  GNUNET_assert (occ->peer->reference_cnt > 0);  
+  occ->peer->reference_cnt--;
+  if (PEER_EXPIRED (occ->peer))
+    GST_destroy_peer (occ->peer);
+  switch (occ->type)
   {
-    GNUNET_TRANSPORT_disconnect (occ->tcc.th);
-    GST_peer_list[occ->other_peer_id]->reference_cnt--;
+  case OCC_TYPE_LOCAL:
+    peer2 = GST_peer_list[occ->other_peer_id];
+    GNUNET_assert (peer2->reference_cnt > 0);
+    peer2->reference_cnt--;
+    if (PEER_EXPIRED (peer2))
+      GST_destroy_peer (peer2);
+    cleanup_occ_lp2c (&occ->p2ctx.local);
+    break;
+  case OCC_TYPE_REMOTE_SLAVE:
+  case OCC_TYPE_REMOTE_LATERAL:
+    cleanup_occ_rp2c (&occ->p2ctx.remote);
+    break;
   }
-  if ((GNUNET_YES == occ->peer->destroy_flag)
-      && (0 == occ->peer->reference_cnt))
-    GST_destroy_peer (occ->peer);
-  if ((NULL == occ->peer2_controller)
-      && (GNUNET_YES == GST_peer_list[occ->other_peer_id]->destroy_flag)
-        && (0 == GST_peer_list[occ->other_peer_id]->reference_cnt))
-      GST_destroy_peer (GST_peer_list[occ->other_peer_id]);  
   GNUNET_CONTAINER_DLL_remove (occq_head, occq_tail, occ);
   GNUNET_free (occ);
 }
@@ -427,7 +569,7 @@ static void
 do_cleanup_occ (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct OverlayConnectContext *occ = cls;
-  
+
   occ->cleanup_task = GNUNET_SCHEDULER_NO_TASK;
   cleanup_occ (occ);
 }
@@ -445,35 +587,51 @@ timeout_overlay_connect (void *cls,
 {
   struct OverlayConnectContext *occ = cls;
 
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task);
   occ->timeout_task = GNUNET_SCHEDULER_NO_TASK;
-  LOG (GNUNET_ERROR_TYPE_WARNING,
-       "0x%llx: Timeout while connecting peers %u and %u\n",
-       occ->op_id, occ->peer_id, occ->other_peer_id);
+  /* LOG (GNUNET_ERROR_TYPE_WARNING, */
+  /*      "0x%llx: Timeout while connecting peers %u and %u: %s\n", occ->op_id, */
+  /*      occ->peer->id, occ->other_peer_id, occ->emsg); */
   GST_send_operation_fail_msg (occ->client, occ->op_id, occ->emsg);
   cleanup_occ (occ);
 }
 
 
+static void
+send_overlay_connect_success_msg (struct OverlayConnectContext *occ)
+{
+  struct GNUNET_TESTBED_ConnectionEventMessage *msg;
+
+  LOG_DEBUG ("0x%llx: Peers connected - Sending overlay connect success\n",
+             occ->op_id);
+  msg = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
+  msg->header.size =
+      htons (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
+  msg->header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_PEER_CONNECT_EVENT);
+  msg->event_type = htonl (GNUNET_TESTBED_ET_CONNECT);
+  msg->peer1 = htonl (occ->peer->id);
+  msg->peer2 = htonl (occ->other_peer_id);
+  msg->operation_id = GNUNET_htonll (occ->op_id);
+  GST_queue_message (occ->client, &msg->header);
+}
+
+
 /**
  * Function called to notify transport users that another
  * peer connected to us.
  *
  * @param cls closure
  * @param new_peer the peer that connected
- * @param ats performance data
- * @param ats_count number of entries in ats (excluding 0-termination)
  */
 static void
-overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
-                        const struct GNUNET_ATS_Information *ats,
-                        unsigned int ats_count)
+overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer)
 {
   struct OverlayConnectContext *occ = cls;
-  struct GNUNET_TESTBED_ConnectionEventMessage *msg;
+  struct LocalPeer2Context *lp2c;
   char *new_peer_str;
   char *other_peer_str;
 
-  //LOG_DEBUG ("Overlay connect notify\n");
+  LOG_DEBUG ("Overlay connect notify\n");
   if (0 ==
       memcmp (new_peer, &occ->peer_identity,
               sizeof (struct GNUNET_PeerIdentity)))
@@ -484,8 +642,8 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
       memcmp (new_peer, &occ->other_peer_identity,
               sizeof (struct GNUNET_PeerIdentity)))
   {
-    /* LOG_DEBUG ("Unexpected peer %4s connected when expecting peer %4s\n", */
-    /*                new_peer_str, other_peer_str); */
+    LOG_DEBUG ("Unexpected peer %4s connected when expecting peer %4s\n",
+               new_peer_str, other_peer_str);
     GNUNET_free (new_peer_str);
     GNUNET_free (other_peer_str);
     return;
@@ -502,26 +660,19 @@ overlay_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task);
   GNUNET_SCHEDULER_cancel (occ->timeout_task);
   occ->timeout_task = GNUNET_SCHEDULER_NO_TASK;
-  if (GNUNET_SCHEDULER_NO_TASK != occ->tcc.task)
+  if (OCC_TYPE_LOCAL == occ->type)
   {
-    GNUNET_SCHEDULER_cancel (occ->tcc.task);
-    occ->tcc.task = GNUNET_SCHEDULER_NO_TASK;
+    lp2c = &occ->p2ctx.local;
+    if (GNUNET_SCHEDULER_NO_TASK != lp2c->tcc.task)
+    {
+      GNUNET_SCHEDULER_cancel (lp2c->tcc.task);
+      lp2c->tcc.task = GNUNET_SCHEDULER_NO_TASK;
+    }
   }
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = NULL;
-  LOG_DEBUG ("0x%llx: Peers connected - Sending overlay connect success\n",
-             occ->op_id);
-  msg = GNUNET_malloc (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
-  msg->header.size =
-      htons (sizeof (struct GNUNET_TESTBED_ConnectionEventMessage));
-  msg->header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_PEERCONEVENT);
-  msg->event_type = htonl (GNUNET_TESTBED_ET_CONNECT);
-  msg->peer1 = htonl (occ->peer_id);
-  msg->peer2 = htonl (occ->other_peer_id);
-  msg->operation_id = GNUNET_htonll (occ->op_id);
-  GST_queue_message (occ->client, &msg->header);
+  send_overlay_connect_success_msg (occ);
   occ->cleanup_task = GNUNET_SCHEDULER_add_now (&do_cleanup_occ, occ);
-  //cleanup_occ (occ);
 }
 
 
@@ -542,17 +693,18 @@ try_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
  * @param result GNUNET_OK if message was transmitted to transport service
  *               GNUNET_SYSERR if message was not transmitted to transport service
  */
-static void 
+static void
 try_connect_cb (void *cls, const int result)
 {
   struct TryConnectContext *tcc = cls;
 
   tcc->tch = NULL;
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == tcc->task);
-  tcc->task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
-                                            (GNUNET_TIME_UNIT_MILLISECONDS,
-                                            500 + pow(2, ++tcc->retries)),
-                                            &try_connect_task, tcc);
+  tcc->task =
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS,
+                                     500 + pow (2, ++tcc->retries)),
+                                    &try_connect_task, tcc);
 }
 
 
@@ -569,13 +721,15 @@ try_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
   tcc->task = GNUNET_SCHEDULER_NO_TASK;
   if (0 != (GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason))
-    return;  
+    return;
   GNUNET_assert (NULL == tcc->tch);
   GNUNET_assert (NULL != tcc->pid);
-  GNUNET_assert (NULL != tcc->th);
-  LOG_DEBUG ("0x%llx: Trail %u to connect to peer %s\n", tcc->op_id, tcc->retries,
-            GNUNET_i2s(tcc->pid));
-  tcc->tch = GNUNET_TRANSPORT_try_connect (tcc->th, tcc->pid, &try_connect_cb, tcc);
+  GNUNET_assert (NULL != tcc->th_);
+  GNUNET_assert (NULL != tcc->cgh_th);
+  LOG_DEBUG ("0x%llx: Trail %u to connect to peer %4s\n", tcc->op_id,
+             tcc->retries, GNUNET_i2s (tcc->pid));
+  tcc->tch =
+      GNUNET_TRANSPORT_try_connect (tcc->th_, tcc->pid, &try_connect_cb, tcc);
 }
 
 
@@ -602,8 +756,11 @@ static void
 occ_hello_sent_cb (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct OverlayConnectContext *occ = cls;
+  struct LocalPeer2Context *lp2c;
 
-  occ->ohh = NULL;
+  GNUNET_assert (OCC_TYPE_LOCAL == occ->type);
+  lp2c = &occ->p2ctx.local;
+  lp2c->ohh = NULL;
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == occ->send_hello_task);
   if (GNUNET_SCHEDULER_REASON_TIMEOUT == tc->reason)
   {
@@ -617,17 +774,59 @@ occ_hello_sent_cb (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   if (GNUNET_SCHEDULER_REASON_READ_READY != tc->reason)
     return;
   GNUNET_free_non_null (occ->emsg);
-  GNUNET_asprintf (&occ->emsg, "0x%llx: Timeout while try connect",
-                   occ->op_id);
-  occ->tcc.pid =  &occ->peer_identity;
-  occ->tcc.op_id = occ->op_id;
-  occ->tcc.task = GNUNET_SCHEDULER_add_now (&try_connect_task, &occ->tcc);
+  GNUNET_asprintf (&occ->emsg,
+                   "0x%llx: Timeout during TRANSPORT_try_connect() "
+                   "at peer %4s", occ->op_id, 
+                   GNUNET_i2s(&occ->other_peer_identity));
+  lp2c->tcc.pid = &occ->peer_identity;
+  lp2c->tcc.op_id = occ->op_id;
+  lp2c->tcc.task = GNUNET_SCHEDULER_add_now (&try_connect_task, &lp2c->tcc);
 }
 
 
 /**
- * Task to offer HELLO of peer 1 to peer 2 and try to make peer 2 to connect to
- * peer 1.
+ * Sends the HELLO of peer1 to peer2's controller through remote overlay connect
+ * request.
+ *
+ * @param occ the overlay connect context.  Its type must be either
+ *          OCC_TYPE_REMOTE_SLAVE or OCC_TYPE_REMOTE_LATERAL
+ */
+void
+send_hello_thru_rocc (struct OverlayConnectContext *occ)
+{
+  struct GNUNET_TESTBED_RemoteOverlayConnectMessage *msg;
+  char *other_peer_str;
+  uint16_t msize;
+  uint16_t hello_size;
+  GNUNET_assert (OCC_TYPE_LOCAL != occ->type);
+  GNUNET_assert (NULL != occ->hello);
+  other_peer_str = GNUNET_strdup (GNUNET_i2s (&occ->other_peer_identity));  
+  LOG_DEBUG ("0x%llx: Offering HELLO of %s (size: %u) to %s via Remote "
+             "Overlay Request\n", occ->op_id,
+             GNUNET_i2s (&occ->peer_identity), ntohs (occ->hello->size),
+             other_peer_str);
+  GNUNET_free (other_peer_str);
+  hello_size = ntohs (occ->hello->size);
+  msize =
+      sizeof (struct GNUNET_TESTBED_RemoteOverlayConnectMessage) + hello_size;
+  msg = GNUNET_malloc (msize);
+  msg->header.type =
+      htons (GNUNET_MESSAGE_TYPE_TESTBED_REMOTE_OVERLAY_CONNECT);
+  msg->header.size = htons (msize);
+  msg->peer = htonl (occ->other_peer_id);
+  msg->operation_id = GNUNET_htonll (occ->op_id);
+  (void) memcpy (&msg->peer_identity, &occ->peer_identity,
+                 sizeof (struct GNUNET_PeerIdentity));
+  memcpy (msg->hello, occ->hello, hello_size);
+  GNUNET_TESTBED_queue_message_ (occ->p2ctx.remote.p2c, &msg->header);
+}
+
+
+/**
+ * Task to offer HELLO of peer 1 to peer 2.  If peer2 is local it is offered
+ * using its TRANSPORT connection; if remote the HELLO is sent remotely by using
+ * send_hello_thru_rocc()
  *
  * @param cls the OverlayConnectContext
  * @param tc the TaskContext from scheduler
@@ -636,58 +835,69 @@ static void
 send_hello (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct OverlayConnectContext *occ = cls;
+  struct LocalPeer2Context *lp2c;
   char *other_peer_str;
 
   occ->send_hello_task = GNUNET_SCHEDULER_NO_TASK;
   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
     return;
   GNUNET_assert (NULL != occ->hello);
-  other_peer_str = GNUNET_strdup (GNUNET_i2s (&occ->other_peer_identity));
-  if (NULL != occ->peer2_controller)
+  if (OCC_TYPE_LOCAL != occ->type)
   {
-    struct GNUNET_TESTBED_RequestConnectMessage *msg;
-    uint16_t msize;
-    uint16_t hello_size;
-
-    LOG_DEBUG ("0x%llx: Offering HELLO of %s (size: %u) to %s via Remote "
-               "Overlay Request\n",
-               occ->op_id,
-              GNUNET_i2s (&occ->peer_identity), 
-              ntohs (occ->hello->size),
-              other_peer_str);
-    hello_size = ntohs (occ->hello->size);
-    msize = sizeof (struct GNUNET_TESTBED_RequestConnectMessage) + hello_size;
-    msg = GNUNET_malloc (msize);
-    msg->header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_REQUESTCONNECT);
-    msg->header.size = htons (msize);
-    msg->peer = htonl (occ->other_peer_id);
-    msg->operation_id = GNUNET_htonll (occ->op_id);
-    (void) memcpy (&msg->peer_identity, &occ->peer_identity,
-                  sizeof (struct GNUNET_PeerIdentity));
-    memcpy (msg->hello, occ->hello, hello_size);
-    GNUNET_TESTBED_queue_message_ (occ->peer2_controller, &msg->header);
+    send_hello_thru_rocc (occ);
+    return;
   }
-  else
+  lp2c = &occ->p2ctx.local;
+  other_peer_str = GNUNET_strdup (GNUNET_i2s (&occ->other_peer_identity));  
+  LOG_DEBUG ("0x%llx: Offering HELLO of %s to %s\n", occ->op_id,
+             GNUNET_i2s (&occ->peer_identity), other_peer_str);
+  GNUNET_free (other_peer_str);
+  lp2c->ohh =
+      GNUNET_TRANSPORT_offer_hello (lp2c->tcc.th_, occ->hello,
+                                    occ_hello_sent_cb, occ);
+  if (NULL == lp2c->ohh)
   {
-    LOG_DEBUG ("0x%llx: Offering HELLO of %s to %s\n",
-               occ->op_id,
-              GNUNET_i2s (&occ->peer_identity), other_peer_str);
-    occ->ohh = GNUNET_TRANSPORT_offer_hello (occ->tcc.th,
-                                             occ->hello,
-                                             occ_hello_sent_cb,
-                                             occ);
-    if (NULL == occ->ohh)
-    {
-      GNUNET_break (0);
-      occ->send_hello_task =
-          GNUNET_SCHEDULER_add_delayed
-          (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                          100 + GNUNET_CRYPTO_random_u32
-                                          (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
-           &send_hello, occ);
-    }
+    GNUNET_break (0);
+    occ->send_hello_task =
+        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                      (GNUNET_TIME_UNIT_MILLISECONDS,
+                                       100 +
+                                       GNUNET_CRYPTO_random_u32
+                                       (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
+                                      &send_hello, occ);
   }
-  GNUNET_free (other_peer_str);  
+}
+
+
+/**
+ * Callback from cache with needed handles set
+ *
+ * @param cls the closure passed to GST_cache_get_handle_transport()
+ * @param ch the handle to CORE. Can be NULL if it is not requested
+ * @param th the handle to TRANSPORT. Can be NULL if it is not requested
+ * @param ignore_ peer identity which is ignored in this callback
+ */
+static void
+p2_transport_connect_cache_callback (void *cls, struct GNUNET_CORE_Handle *ch,
+                                     struct GNUNET_TRANSPORT_Handle *th,
+                                     const struct GNUNET_PeerIdentity *ignore_)
+{
+  struct OverlayConnectContext *occ = cls;
+
+  GNUNET_assert (OCC_TYPE_LOCAL == occ->type);
+  if (NULL == th)
+  {
+    GNUNET_asprintf (&occ->emsg, "0x%llx: Cannot connect to TRANSPORT of %s",
+                     occ->op_id, GNUNET_i2s (&occ->other_peer_identity));
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
+    occ->timeout_task =
+        GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
+    return;
+  }
+  occ->p2ctx.local.tcc.th_ = th;
+  GNUNET_asprintf (&occ->emsg, "0x%llx: Timeout while offering HELLO to %s",
+                   occ->op_id, GNUNET_i2s (&occ->other_peer_identity));
+  occ->send_hello_task = GNUNET_SCHEDULER_add_now (&send_hello, occ);
 }
 
 
@@ -700,25 +910,22 @@ send_hello (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 static void
 p2_transport_connect (struct OverlayConnectContext *occ)
 {
+  struct Peer *peer2;
+
   GNUNET_assert (NULL == occ->emsg);
   GNUNET_assert (NULL != occ->hello);
   GNUNET_assert (NULL == occ->ghh);
-  GNUNET_assert (NULL == occ->p1th);
-  if (NULL == occ->peer2_controller)
+  GNUNET_assert (NULL == occ->p1th_);
+  GNUNET_assert (NULL == occ->cgh_p1th);
+  if (OCC_TYPE_LOCAL == occ->type)
   {
-    GST_peer_list[occ->other_peer_id]->reference_cnt++;
-    occ->tcc.th =
-       GNUNET_TRANSPORT_connect (GST_peer_list[occ->other_peer_id]->details.local.cfg,
-                                 &occ->other_peer_identity, NULL, NULL, NULL,
-                                 NULL);
-    if (NULL == occ->tcc.th)
-    {
-      GNUNET_asprintf (&occ->emsg, "0x%llx: Cannot connect to TRANSPORT of %s",
-                      occ->op_id, GNUNET_i2s (&occ->other_peer_identity));
-      GNUNET_SCHEDULER_cancel (occ->timeout_task);
-      occ->timeout_task = GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
-      return;
-    }
+    GNUNET_assert (NULL != (peer2 = GST_peer_list[occ->other_peer_id]));    
+    occ->p2ctx.local.tcc.cgh_th =
+        GST_cache_get_handle_transport (occ->other_peer_id,
+                                        peer2->details.local.cfg,
+                                        &p2_transport_connect_cache_callback,
+                                        occ, NULL, NULL, NULL);
+    return;
   }
   GNUNET_asprintf (&occ->emsg, "0x%llx: Timeout while offering HELLO to %s",
                    occ->op_id, GNUNET_i2s (&occ->other_peer_identity));
@@ -767,20 +974,20 @@ hello_update_cb (void *cls, const struct GNUNET_MessageHeader *hello)
                                          &empty);
   if (GNUNET_YES == empty)
   {
-    LOG_DEBUG ("0x%llx: HELLO of %s is empty\n",
-               occ->op_id, GNUNET_i2s (&occ->peer_identity));
+    LOG_DEBUG ("0x%llx: HELLO of %s is empty\n", occ->op_id,
+               GNUNET_i2s (&occ->peer_identity));
     return;
   }
-  LOG_DEBUG ("0x%llx: Received HELLO of %s\n",
-             occ->op_id, GNUNET_i2s (&occ->peer_identity));
+  LOG_DEBUG ("0x%llx: Received HELLO of %s\n", occ->op_id,
+             GNUNET_i2s (&occ->peer_identity));
   occ->hello = GNUNET_malloc (msize);
-  GST_hello_cache_add (&occ->peer_identity, hello);
+  GST_cache_add_hello (occ->peer->id, hello);
   memcpy (occ->hello, hello, msize);
   GNUNET_TRANSPORT_get_hello_cancel (occ->ghh);
   occ->ghh = NULL;
-  GNUNET_TRANSPORT_disconnect (occ->p1th);
-  occ->p1th = NULL;
-  occ->peer->reference_cnt--;
+  GST_cache_get_handle_done (occ->cgh_p1th);
+  occ->cgh_p1th = NULL;
+  occ->p1th_ = NULL;
   GNUNET_free_non_null (occ->emsg);
   occ->emsg = NULL;
   p2_transport_connect (occ);
@@ -788,66 +995,101 @@ hello_update_cb (void *cls, const struct GNUNET_MessageHeader *hello)
 
 
 /**
- * Function called after GNUNET_CORE_connect has succeeded (or failed
- * for good).  Note that the private key of the peer is intentionally
- * not exposed here; if you need it, your process should try to read
- * the private key file directly (which should work if you are
- * authorized...).
+ * Callback from cache with needed handles set
  *
- * @param cls closure
- * @param server handle to the server, NULL if we failed
- * @param my_identity ID of this peer, NULL if we failed
+ * @param cls the closure passed to GST_cache_get_handle_transport()
+ * @param ch the handle to CORE. Can be NULL if it is not requested
+ * @param th the handle to TRANSPORT. Can be NULL if it is not requested
+ * @param ignore_ peer identity which is ignored in this callback
+ */
+static void
+p1_transport_connect_cache_callback (void *cls, struct GNUNET_CORE_Handle *ch,
+                                     struct GNUNET_TRANSPORT_Handle *th,
+                                     const struct GNUNET_PeerIdentity *ignore_)
+{
+  struct OverlayConnectContext *occ = cls;
+
+  GNUNET_free_non_null (occ->emsg);
+  occ->emsg = NULL;
+  if (NULL == th)
+  {
+    GNUNET_asprintf (&occ->emsg, "0x%llx: Cannot connect to TRANSPORT of %s",
+                     occ->op_id, GNUNET_i2s (&occ->peer_identity));
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
+    occ->timeout_task =
+        GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
+    return;
+  }
+  GNUNET_assert (NULL == occ->p1th_);
+  GNUNET_assert (NULL != occ->cgh_p1th);
+  occ->p1th_ = th;
+  GNUNET_asprintf (&occ->emsg,
+                   "0x%llx: Timeout while acquiring HELLO of peer %4s",
+                   occ->op_id, GNUNET_i2s (&occ->peer_identity));
+  occ->ghh = GNUNET_TRANSPORT_get_hello (occ->p1th_, &hello_update_cb, occ);
+}
+
+
+/**
+ * Callback from cache with needed handles set
+ *
+ * @param cls the closure passed to GST_cache_get_handle_transport()
+ * @param ch the handle to CORE. Can be NULL if it is not requested
+ * @param th the handle to TRANSPORT. Can be NULL if it is not requested
+ * @param my_identity the identity of our peer
  */
 static void
-core_startup_cb (void *cls, struct GNUNET_CORE_Handle *server,
-                 const struct GNUNET_PeerIdentity *my_identity)
+occ_cache_get_handle_core_cb (void *cls, struct GNUNET_CORE_Handle *ch,
+                              struct GNUNET_TRANSPORT_Handle *th,
+                              const struct GNUNET_PeerIdentity *my_identity)
 {
   struct OverlayConnectContext *occ = cls;
   const struct GNUNET_MessageHeader *hello;
 
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task);
   GNUNET_free_non_null (occ->emsg);
-  (void) GNUNET_asprintf (&occ->emsg,
-                          "0x%llx: Failed to connect to CORE of peer with"
-                          "id: %u", occ->op_id, occ->peer_id);
-  if ((NULL == server) || (NULL == my_identity))
-    goto error_return;
-  GNUNET_free (occ->emsg);
-  occ->ch = server;
+  if ((NULL == ch) || (NULL == my_identity))
+  {
+    GNUNET_asprintf (&occ->emsg,
+                     "0x%llx: Failed to connect to CORE of peer with "
+                     "id: %u", occ->op_id, occ->peer->id);
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
+    occ->timeout_task =
+        GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
+    return;
+  }
   occ->emsg = NULL;
+  if (GNUNET_YES ==
+      GNUNET_CORE_is_peer_connected_sync (ch, &occ->other_peer_identity))
+  {
+    LOG_DEBUG ("0x%llx: Target peer already connected\n", occ->op_id);
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
+    occ->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+    send_overlay_connect_success_msg (occ);
+    occ->cleanup_task = GNUNET_SCHEDULER_add_now (&do_cleanup_occ, occ);
+    return;
+  }
   memcpy (&occ->peer_identity, my_identity,
           sizeof (struct GNUNET_PeerIdentity));
-  LOG_DEBUG ("0x%llx: Acquiring HELLO of peer %s\n",
-             occ->op_id, GNUNET_i2s (&occ->peer_identity));
+  LOG_DEBUG ("0x%llx: Acquiring HELLO of peer %s\n", occ->op_id,
+             GNUNET_i2s (&occ->peer_identity));
   /* Lookup for HELLO in hello cache */
-  if (NULL != (hello = GST_hello_cache_lookup (&occ->peer_identity)))
+  if (NULL != (hello = GST_cache_lookup_hello (occ->peer->id)))
   {
-    LOG_DEBUG ("0x%llx: HELLO of peer %s found in cache\n",
-               occ->op_id, GNUNET_i2s (&occ->peer_identity));
+    LOG_DEBUG ("0x%llx: HELLO of peer %s found in cache\n", occ->op_id,
+               GNUNET_i2s (&occ->peer_identity));
     occ->hello = GNUNET_copy_message (hello);
     p2_transport_connect (occ);
     return;
   }
-  occ->peer->reference_cnt++;
-  occ->p1th =
-      GNUNET_TRANSPORT_connect (occ->peer->details.local.cfg,
-                                &occ->peer_identity, NULL, NULL, NULL, NULL);
-  if (NULL == occ->p1th)
-  {
-    GNUNET_asprintf (&occ->emsg,
-                     "0x%llx: Cannot connect to TRANSPORT of peer %4s",
-                     occ->op_id, GNUNET_i2s (&occ->peer_identity));
-    goto error_return;
-  }  
   GNUNET_asprintf (&occ->emsg,
-                   "0x%llx: Timeout while acquiring HELLO of peer %4s",
+                   "0x%llx: Timeout while acquiring TRANSPORT of %s from cache",
                    occ->op_id, GNUNET_i2s (&occ->peer_identity));
-  occ->ghh = GNUNET_TRANSPORT_get_hello (occ->p1th, &hello_update_cb, occ);
-  return;
-  
- error_return:
-  GNUNET_SCHEDULER_cancel (occ->timeout_task);
-  occ->timeout_task = GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
-  return;
+  occ->cgh_p1th =
+      GST_cache_get_handle_transport (occ->peer->id,
+                                      occ->peer->details.local.cfg,
+                                      p1_transport_connect_cache_callback, occ,
+                                      NULL, NULL, NULL);
 }
 
 
@@ -863,74 +1105,49 @@ static void
 overlay_connect_get_config (void *cls, const struct GNUNET_MessageHeader *msg)
 {
   struct OverlayConnectContext *occ = cls;
+  struct RemotePeer2Context *rp2c;
   const struct GNUNET_TESTBED_PeerConfigurationInformationMessage *cmsg;
-  const struct GNUNET_CORE_MessageHandler no_handlers[] = {
-    {NULL, 0, 0}
-  };
-
-  occ->opc = NULL;
-  if (GNUNET_MESSAGE_TYPE_TESTBED_PEERCONFIG != ntohs (msg->type))
-    goto error_return;
-  cmsg = (const struct GNUNET_TESTBED_PeerConfigurationInformationMessage *)
-      msg;
+
+  GNUNET_assert (OCC_TYPE_LOCAL != occ->type);
+  rp2c = &occ->p2ctx.remote;
+  rp2c->opc = NULL;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != occ->timeout_task);
+  if (GNUNET_MESSAGE_TYPE_TESTBED_PEER_INFORMATION != ntohs (msg->type))
+  {
+    GNUNET_SCHEDULER_cancel (occ->timeout_task);
+    occ->timeout_task =
+        GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
+  }
+  cmsg =
+      (const struct GNUNET_TESTBED_PeerConfigurationInformationMessage *) msg;
   memcpy (&occ->other_peer_identity, &cmsg->peer_identity,
-         sizeof (struct GNUNET_PeerIdentity));
+          sizeof (struct GNUNET_PeerIdentity));
   GNUNET_free_non_null (occ->emsg);
-  GNUNET_asprintf (&occ->emsg, "0x%llx: Timeout while connecting to CORE",
-                   occ->op_id);
-  occ->peer->reference_cnt++;
-  occ->ch =
-      GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
-                           &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
-                           GNUNET_NO, no_handlers);
-  if (NULL == occ->ch)
-    goto error_return;
+  GNUNET_asprintf (&occ->emsg,
+                   "0x%llx: Timeout while connecting to CORE of peer with "
+                   "id: %u", occ->op_id, occ->peer->id);
+  occ->cgh_ch =
+      GST_cache_get_handle_core (occ->peer->id, occ->peer->details.local.cfg,
+                                 occ_cache_get_handle_core_cb, occ,
+                                 &occ->other_peer_identity,
+                                 &overlay_connect_notify, occ);
   return;
-
- error_return:
-  GNUNET_SCHEDULER_cancel (occ->timeout_task);
-  occ->timeout_task = 
-      GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
 }
 
 
 /**
- * Callback which will be called to after a host registration succeeded or failed
+ * Callback which will be called after a host registration succeeded or failed
  *
  * @param cls the RegisteredHostContext
  * @param emsg the error message; NULL if host registration is successful
  */
-static void 
-registeredhost_registration_completion (void *cls, const char *emsg)
+static void
+host_registration_comp (void *cls, const char *emsg)
 {
   struct RegisteredHostContext *rhc = cls;
-  struct GNUNET_CONFIGURATION_Handle *cfg;
-  uint32_t peer2_host_id;
-
-  /* if (NULL != rhc->focc_dll_head) */
-  /*   TESTBED_process_next_focc (rhc); */
-  peer2_host_id = GNUNET_TESTBED_host_get_id_ (rhc->reg_host);
-  GNUNET_assert (RHC_INIT == rhc->state);
-  GNUNET_assert (NULL == rhc->sub_op);
-  if ((NULL == rhc->gateway2)
-      || ((peer2_host_id < GST_slave_list_size) /* Check if we have the needed config */
-          && (NULL != GST_slave_list[peer2_host_id])))
-  {
-    rhc->state = RHC_LINK;
-    cfg = (NULL == rhc->gateway2) ? our_config : GST_slave_list[peer2_host_id]->cfg;
-    rhc->sub_op =
-        GNUNET_TESTBED_controller_link (rhc,
-                                        rhc->gateway->controller,
-                                        rhc->reg_host,
-                                        rhc->host,
-                                        cfg,
-                                        GNUNET_NO);
-    return;
-  }
-  rhc->state = RHC_GET_CFG;
-  rhc->sub_op =  GNUNET_TESTBED_get_slave_config (rhc,
-                                                  rhc->gateway2->controller,
-                                                  rhc->reg_host);
+  
+  rhc->state = RHC_DONE;
+  GST_process_next_focc (rhc);
 }
 
 
@@ -944,9 +1161,8 @@ registeredhost_registration_completion (void *cls, const char *emsg)
  *         iterate,
  *         GNUNET_NO if not.
  */
-static int 
-reghost_match_iterator (void *cls,
-                        const struct GNUNET_HashCode * key,
+static int
+reghost_match_iterator (void *cls, const struct GNUNET_HashCode *key,
                         void *value)
 {
   struct RegisteredHostContext **rh = cls;
@@ -983,6 +1199,175 @@ hash_hosts (struct GNUNET_TESTBED_Host *reg_host,
 }
 
 
+/**
+ * Checks if the given host is registered at the given slave.
+ *
+ * @param slave the slave where registration has to be checked.  The check is
+ *          actually done through a locally maintained hashmap.  No
+ *          communication with the slave is involved.
+ * @param host the host to register
+ * @return If the given host is not registered already or the registration is
+ *           pending, it returns the registration context.  Any overlay connects
+ *           to be forwarded should be queued in the context so that they can be
+ *           executed when the registration is completed.  If the given host is
+ *           already registered, NULL is returned.
+ */
+static struct RegisteredHostContext *
+register_host (struct Slave *slave, struct GNUNET_TESTBED_Host *host)
+{
+  struct GNUNET_HashCode hash;
+  struct RegisteredHostContext *rhc;
+  
+  rhc = GNUNET_malloc (sizeof (struct RegisteredHostContext));
+  rhc->reg_host = host;
+  rhc->host = GST_host_list[slave->host_id];
+  GNUNET_assert (NULL != rhc->reg_host);
+  GNUNET_assert (NULL != rhc->host);
+  rhc->state = RHC_INIT;
+  hash = hash_hosts (rhc->reg_host, rhc->host);
+  if ((GNUNET_NO ==
+       GNUNET_CONTAINER_multihashmap_contains (slave->reghost_map, &hash)) ||
+      (GNUNET_SYSERR !=
+       GNUNET_CONTAINER_multihashmap_get_multiple (slave->reghost_map,
+                                                   &hash,
+                                                   reghost_match_iterator,
+                                                   &rhc)))
+  {
+    /* create and add a new registerd host context */
+    /* add the focc to its queue */
+    GNUNET_CONTAINER_multihashmap_put (slave->reghost_map, &hash, rhc,
+                                       GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
+    GST_queue_host_registration (slave, host_registration_comp,
+                                 rhc, rhc->reg_host);
+  }
+  else
+  {
+    /* rhc is now set to the existing one from the hash map by
+     * reghost_match_iterator() */
+    /* if queue is empty then ignore creating focc and proceed with normal
+     * forwarding */
+    if (RHC_DONE == rhc->state)
+      return NULL;
+  }
+  return rhc;
+}
+
+
+/**
+ * Forwards the overlay connect request to a slave controller.  Before
+ * forwarding, any hosts which are needed to be known by the slave controller to
+ * execute the overlay connect request are registered at slave.
+ *
+ * @param msg the overlay connect request message to be forwarded
+ * @param client the client to which the status of the forwarded request has to
+ *          be notified
+ */
+static void
+forward_overlay_connect (const struct GNUNET_TESTBED_OverlayConnectMessage *msg,
+                         struct GNUNET_SERVER_Client *client)
+{
+  struct ForwardedOperationContext *fopc;
+  struct Route *route_to_peer2_host;
+  struct Route *route_to_peer1_host;
+  struct Peer *peer;
+  struct RegisteredHostContext *rhc;
+  struct ForwardedOverlayConnectContext *focc;
+  uint64_t op_id;
+  uint32_t peer2_host_id;
+  uint32_t p1;
+  uint32_t p2;
+
+  p1 = ntohl (msg->peer1);
+  p2 = ntohl (msg->peer2);
+  op_id = GNUNET_ntohll (msg->operation_id);
+  peer2_host_id = ntohl (msg->peer2_host_id);
+  GNUNET_assert (VALID_PEER_ID (p1));
+  GNUNET_assert (VALID_HOST_ID (peer2_host_id));
+  peer = GST_peer_list[p1];
+  GNUNET_assert (GNUNET_YES == peer->is_remote);
+  LOG_DEBUG ("0x%llx: Forwarding overlay connect\n", op_id);
+  route_to_peer2_host = GST_find_dest_route (peer2_host_id);
+  route_to_peer1_host = GST_find_dest_route
+      (peer->details.remote.remote_host_id);
+  GNUNET_assert (NULL != route_to_peer1_host);
+  if ((NULL != route_to_peer2_host) && 
+      (route_to_peer1_host->dest == route_to_peer2_host->dest))
+    goto forward;
+  /* Peer2 is either with us OR peer1 and peer2 can be reached through
+     different subtrees OR peer2 is on a subtree unknown to us */
+  if (NULL != (rhc = register_host (peer->details.remote.slave,
+                                    GST_host_list[peer2_host_id])))
+  {
+    LOG_DEBUG ("Queueing forwarding FOCC for connecting peers %u and %u\n", p1, p2);
+    focc = GNUNET_malloc (sizeof (struct ForwardedOverlayConnectContext));
+    focc->peer1 = p1;
+    focc->peer2 = p2;
+    focc->peer2_host_id = peer2_host_id;
+    focc->orig_msg = GNUNET_copy_message (&msg->header);
+    focc->operation_id = op_id;
+    focc->client = client;
+    GNUNET_SERVER_client_keep (client);
+    GNUNET_CONTAINER_DLL_insert_tail (rhc->focc_dll_head, rhc->focc_dll_tail,
+                                      focc);
+    return;
+  }
+
+ forward:
+  LOG_DEBUG ("Forwarding without FOCC for connecting peers %u and %u\n", p1, p2);
+  fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
+  GNUNET_SERVER_client_keep (client);
+  fopc->client = client;
+  fopc->operation_id = op_id;
+  fopc->type = OP_OVERLAY_CONNECT;
+  fopc->opc =
+      GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.
+                                             slave->controller, op_id,
+                                             &msg->header,
+                                             &GST_forwarded_operation_reply_relay,
+                                             fopc);
+  fopc->timeout_task =
+      GNUNET_SCHEDULER_add_delayed (GST_timeout, &GST_forwarded_operation_timeout,
+                                    fopc);
+  GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
+}
+
+
+/**
+ * Callback called when a connection to the controller of peer2 has been
+ * established
+ *
+ * @param cls the overlay connect contexts
+ * @param c handle to the controller connection
+ */
+static void
+p2_controller_connect_cb (void *cls, struct GNUNET_TESTBED_Controller *c)
+{
+  struct OverlayConnectContext *occ = cls;
+  struct RemotePeer2Context *rp2c;
+  struct GNUNET_TESTBED_PeerGetConfigurationMessage cmsg;
+
+  GNUNET_assert (OCC_TYPE_LOCAL != occ->type);
+  rp2c = &occ->p2ctx.remote;
+  rp2c->ncn = NULL;
+  rp2c->p2c = c;
+  cmsg.header.size =
+      htons (sizeof (struct GNUNET_TESTBED_PeerGetConfigurationMessage));
+  cmsg.header.type =
+      htons (GNUNET_MESSAGE_TYPE_TESTBED_GET_PEER_INFORMATION);
+  cmsg.peer_id = htonl (occ->other_peer_id);
+  cmsg.operation_id = GNUNET_htonll (occ->op_id);
+  rp2c->opc =
+      GNUNET_TESTBED_forward_operation_msg_ (rp2c->p2c,
+                                             occ->op_id, &cmsg.header,
+                                             &overlay_connect_get_config,
+                                             occ);
+  GNUNET_free_non_null (occ->emsg);
+  GNUNET_asprintf (&occ->emsg,
+                   "0x%llx: Timeout while getting peer identity of peer "
+                   "with id: %u", occ->op_id, occ->other_peer_id);
+}
+
+
 /**
  * Handler for GNUNET_MESSAGE_TYPE_TESTBED_OLCONNECT messages
  *
@@ -992,22 +1377,20 @@ hash_hosts (struct GNUNET_TESTBED_Host *reg_host,
  */
 void
 GST_handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
-                                const struct GNUNET_MessageHeader *message)
+                            const struct GNUNET_MessageHeader *message)
 {
   const struct GNUNET_TESTBED_OverlayConnectMessage *msg;
-  const struct GNUNET_CORE_MessageHandler no_handlers[] = {
-    {NULL, 0, 0}
-  };
   struct Peer *peer;
+  struct Peer *peer2;
   struct OverlayConnectContext *occ;
-  struct GNUNET_TESTBED_Controller *peer2_controller;
+  struct Neighbour *p2n;
   uint64_t operation_id;
   uint32_t p1;
-  uint32_t p2; 
+  uint32_t p2;
   uint32_t peer2_host_id;
 
-  if (sizeof (struct GNUNET_TESTBED_OverlayConnectMessage)
-      != ntohs (message->size))
+  if (sizeof (struct GNUNET_TESTBED_OverlayConnectMessage) !=
+      ntohs (message->size))
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
@@ -1016,212 +1399,106 @@ GST_handle_overlay_connect (void *cls, struct GNUNET_SERVER_Client *client,
   msg = (const struct GNUNET_TESTBED_OverlayConnectMessage *) message;
   p1 = ntohl (msg->peer1);
   p2 = ntohl (msg->peer2);
-  if ((p1 >= GST_peer_list_size) || (NULL == GST_peer_list[p1]))
+  if (!VALID_PEER_ID (p1))
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
   peer = GST_peer_list[p1];
+  operation_id = GNUNET_ntohll (msg->operation_id);
+  LOG_DEBUG
+      ("Received overlay connect for peers %u and %u with op id: 0x%llx\n", p1,
+       p2, operation_id);
   peer2_host_id = ntohl (msg->peer2_host_id);
-  operation_id = GNUNET_ntohll (msg->operation_id);  
-  LOG_DEBUG ("Received overlay connect for peers %u and %u with op id: 0x%llx\n",
-            p1, p2, operation_id);
   if (GNUNET_YES == peer->is_remote)
   {
-    struct ForwardedOperationContext *fopc;
-    struct Route *route_to_peer2_host;
-    struct Route *route_to_peer1_host;
-
-    LOG_DEBUG ("0x%llx: Forwarding overlay connect\n", operation_id);
-    route_to_peer2_host = NULL;
-    route_to_peer1_host = NULL;
-    route_to_peer2_host = GST_find_dest_route (peer2_host_id);
-    if ((NULL != route_to_peer2_host) 
-        || (peer2_host_id == GST_context->host_id))
+    if (!VALID_HOST_ID (peer2_host_id))
     {
-      /* Peer 2 either below us OR with us */
-      route_to_peer1_host = 
-          GST_find_dest_route
-          (GST_peer_list[p1]->details.remote.remote_host_id); 
-      /* Because we get this message only if we know where peer 1 is */
-      GNUNET_assert (NULL != route_to_peer1_host);
-      if ((peer2_host_id == GST_context->host_id) 
-          || (route_to_peer2_host->dest != route_to_peer1_host->dest))
-      {
-        /* Peer2 is either with us OR peer1 and peer2 can be reached through
-           different gateways */
-        struct GNUNET_HashCode hash;
-        struct RegisteredHostContext *rhc;
-        int skip_focc;
-
-        rhc = GNUNET_malloc (sizeof (struct RegisteredHostContext));
-        if (NULL != route_to_peer2_host)
-          rhc->reg_host = GST_host_list[route_to_peer2_host->dest];
-        else
-          rhc->reg_host = GST_host_list[GST_context->host_id];
-        rhc->host = GST_host_list[route_to_peer1_host->dest];
-        GNUNET_assert (NULL != rhc->reg_host);
-        GNUNET_assert (NULL != rhc->host);
-        rhc->gateway = peer->details.remote.slave;
-        rhc->gateway2 = (NULL == route_to_peer2_host) ? NULL :
-            GST_slave_list[route_to_peer2_host->dest];
-        rhc->state = RHC_INIT;
-        GNUNET_SERVER_client_keep (client);
-        rhc->client = client;
-        hash = hash_hosts (rhc->reg_host, rhc->host);
-        skip_focc = GNUNET_NO;
-        if ((GNUNET_NO == 
-             GNUNET_CONTAINER_multihashmap_contains
-             (peer->details.remote.slave->reghost_map, &hash))
-            || (GNUNET_SYSERR != GNUNET_CONTAINER_multihashmap_get_multiple
-                (peer->details.remote.slave->reghost_map, &hash,
-                 reghost_match_iterator, &rhc)))
-        {
-          /* create and add a new registerd host context */
-          /* add the focc to its queue */
-          GNUNET_CONTAINER_multihashmap_put
-              (peer->details.remote.slave->reghost_map,
-               &hash, rhc, GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE);
-          GNUNET_assert (NULL != GST_host_list[peer2_host_id]);
-          GST_queue_host_registration (peer->details.remote.slave,
-                                           registeredhost_registration_completion,
-                                           rhc,
-                                           GST_host_list[peer2_host_id]);
-        }
-        else {
-          /* rhc is now set to the existing one from the hash map by
-             reghost_match_iterator() */
-          /* if queue is empty then ignore creating focc and proceed with
-             normal forwarding */                
-          if (RHC_OL_CONNECT == rhc->state)
-            skip_focc = GNUNET_YES;
-        }
-        if (GNUNET_NO == skip_focc)
-        {
-          struct ForwardedOverlayConnectContext *focc;
-
-          focc = GNUNET_malloc (sizeof (struct ForwardedOverlayConnectContext));
-          focc->peer1 = p1;
-          focc->peer2 = p2;
-          focc->peer2_host_id = peer2_host_id;
-          focc->orig_msg = GNUNET_copy_message (message);
-          focc->operation_id = operation_id;
-          GNUNET_CONTAINER_DLL_insert_tail (rhc->focc_dll_head,
-                                            rhc->focc_dll_tail,
-                                            focc);
-          GNUNET_SERVER_receive_done (client, GNUNET_OK);
-          return;
-        }
-      }
+      GNUNET_break (0);
+      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+      return;      
     }
-    fopc = GNUNET_malloc (sizeof (struct ForwardedOperationContext));
-    GNUNET_SERVER_client_keep (client);
-    fopc->client = client;
-    fopc->operation_id = operation_id;
-    fopc->type = OP_OVERLAY_CONNECT;
-    fopc->opc = 
-       GNUNET_TESTBED_forward_operation_msg_ (peer->details.remote.slave->controller,
-                                              operation_id, message,
-                                              &GST_forwarded_operation_reply_relay,
-                                              fopc);
-    fopc->timeout_task =
-       GNUNET_SCHEDULER_add_delayed (TIMEOUT, &GST_forwarded_operation_timeout,
-                                     fopc);
-    GNUNET_CONTAINER_DLL_insert_tail (fopcq_head, fopcq_tail, fopc);
+    forward_overlay_connect (msg, client);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-
-  peer2_controller = NULL;
-  if ((p2 >= GST_peer_list_size) || (NULL == GST_peer_list[p2]))
-  {   
-    if ((peer2_host_id >= GST_slave_list_size)
-       || (NULL ==GST_slave_list[peer2_host_id]))
+  p2n = NULL;
+  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));  
+  occ->type = OCC_TYPE_LOCAL;
+  if (!VALID_PEER_ID (p2))       /* May be peer2 is on a another controller */
+  {    
+    if (NULL == (p2n = GST_get_neighbour (peer2_host_id)))
     {
-      LOG (GNUNET_ERROR_TYPE_WARNING,
-           "0x%llx: Configuration of peer2's controller missing for connecting peers"
-           "%u and %u\n", operation_id, p1, p2);      
-      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-      return;
-    }
-    peer2_controller = GST_slave_list[peer2_host_id]->controller;
-    if (NULL == peer2_controller)
-    {
-      GNUNET_break (0);       /* What's going on? */
-      GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-      return;
+      if (!VALID_HOST_ID (peer2_host_id))
+      {
+        GNUNET_break (0);
+        LOG (GNUNET_ERROR_TYPE_WARNING,
+             "0x%llx: Peer %u's host not in our neighbours list\n",
+             operation_id, p2);
+        GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
+        GNUNET_free (occ);
+        return;
+      }
+      p2n = GST_create_neighbour (GST_host_list[peer2_host_id]);
     }
+    occ->type = OCC_TYPE_REMOTE_LATERAL;
+    occ->p2ctx.remote.p2n = p2n;
   }
-  else
+  else if (GNUNET_YES == GST_peer_list[p2]->is_remote)
   {
-    if (GNUNET_YES == GST_peer_list[p2]->is_remote)
-      peer2_controller = GST_peer_list[p2]->details.remote.slave->controller;
+    occ->type = OCC_TYPE_REMOTE_SLAVE;
+    occ->p2ctx.remote.p2c = GST_peer_list[p2]->details.remote.slave->controller;
   }
-  occ = GNUNET_malloc (sizeof (struct OverlayConnectContext));
   GNUNET_CONTAINER_DLL_insert_tail (occq_head, occq_tail, occ);
   GNUNET_SERVER_client_keep (client);
   occ->client = client;
-  occ->peer_id = p1;
   occ->other_peer_id = p2;
-  occ->peer = GST_peer_list[p1];
-  occ->op_id = GNUNET_ntohll (msg->operation_id);
-  occ->peer2_controller = peer2_controller;
-  /* Get the identity of the second peer */
-  if (NULL != occ->peer2_controller)
+  GST_peer_list[p1]->reference_cnt++;
+  occ->peer = GST_peer_list[p1];  
+  occ->op_id = operation_id;
+  GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == occ->timeout_task);
+  occ->timeout_task =
+      GNUNET_SCHEDULER_add_delayed (GST_timeout, &timeout_overlay_connect, occ);
+  switch (occ->type)
   {
-    struct GNUNET_TESTBED_PeerGetConfigurationMessage cmsg;
-
-    cmsg.header.size = 
-       htons (sizeof (struct GNUNET_TESTBED_PeerGetConfigurationMessage));
-    cmsg.header.type = htons (GNUNET_MESSAGE_TYPE_TESTBED_GETPEERCONFIG);
-    cmsg.peer_id = msg->peer2;
-    cmsg.operation_id = msg->operation_id;
-    occ->opc = 
-       GNUNET_TESTBED_forward_operation_msg_ (occ->peer2_controller,
-                                              occ->op_id, &cmsg.header,
-                                              &overlay_connect_get_config,
-                                              occ);
+  case OCC_TYPE_REMOTE_LATERAL:
     GNUNET_asprintf (&occ->emsg,
-                     "0x%llx: Timeout while getting peer identity of peer "
-                     "with id: %u", occ->op_id, occ->other_peer_id);
-    occ->timeout_task =
-       GNUNET_SCHEDULER_add_delayed (TIMEOUT,
-                                     &timeout_overlay_connect, occ);
-    GNUNET_SERVER_receive_done (client, GNUNET_OK);
-    return;
+                     "0x%llx: Timeout while acquiring connection to peer %u's "
+                     "host: %u\n", occ->op_id, occ->other_peer_id, peer2_host_id);
+    occ->p2ctx.remote.ncn = 
+        GST_neighbour_get_connection (p2n, &p2_controller_connect_cb, occ);
+    break;
+  case OCC_TYPE_REMOTE_SLAVE:
+    p2_controller_connect_cb (occ, occ->p2ctx.remote.p2c);
+    break;
+  case OCC_TYPE_LOCAL:
+    peer2 = GST_peer_list[occ->other_peer_id];
+    peer2->reference_cnt++;
+    GNUNET_TESTING_peer_get_identity (peer2->details.local.peer,
+                                      &occ->other_peer_identity);
+    GNUNET_asprintf (&occ->emsg,
+                     "0x%llx: Timeout while connecting to CORE of peer with "
+                     "id: %u", occ->op_id, occ->peer->id);
+    occ->cgh_ch =
+        GST_cache_get_handle_core (occ->peer->id, occ->peer->details.local.cfg,
+                                   occ_cache_get_handle_core_cb, occ,
+                                   &occ->other_peer_identity,
+                                   &overlay_connect_notify, occ);
+    break;
   }
-  GNUNET_TESTING_peer_get_identity (GST_peer_list[occ->other_peer_id]->details.local.peer,
-                                   &occ->other_peer_identity);
-  /* Connect to the core of 1st peer and wait for the 2nd peer to connect */
-  occ->emsg = GNUNET_strdup ("Timeout while connecting to CORE");
-  GNUNET_asprintf (&occ->emsg,
-                   "0x%llx: Timeout while connecting to CORE of peer with "
-                   "id: %u", occ->op_id, occ->peer_id);
-  occ->peer->reference_cnt++;
-  occ->ch =
-      GNUNET_CORE_connect (occ->peer->details.local.cfg, occ, &core_startup_cb,
-                           &overlay_connect_notify, NULL, NULL, GNUNET_NO, NULL,
-                           GNUNET_NO, no_handlers);
-  if (NULL == occ->ch)
-    occ->timeout_task = 
-       GNUNET_SCHEDULER_add_now (&timeout_overlay_connect, occ);
-  else
-    occ->timeout_task =
-       GNUNET_SCHEDULER_add_delayed (TIMEOUT,
-                                     &timeout_overlay_connect, occ);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
 
 
 /**
- * Function to cleanup RequestOverlayConnectContext and any associated tasks
+ * Function to cleanup RemoteOverlayConnectCtx and any associated tasks
  * with it
  *
- * @param rocc the RequestOverlayConnectContext
+ * @param rocc the RemoteOverlayConnectCtx
  */
 static void
-cleanup_rocc (struct RequestOverlayConnectContext *rocc)
+cleanup_rocc (struct RemoteOverlayConnectCtx *rocc)
 {
   LOG_DEBUG ("0x%llx: Cleaning up rocc\n", rocc->op_id);
   if (GNUNET_SCHEDULER_NO_TASK != rocc->attempt_connect_task_id)
@@ -1234,10 +1511,12 @@ cleanup_rocc (struct RequestOverlayConnectContext *rocc)
     GNUNET_TRANSPORT_try_connect_cancel (rocc->tcc.tch);
   if (GNUNET_SCHEDULER_NO_TASK != rocc->tcc.task)
     GNUNET_SCHEDULER_cancel (rocc->tcc.task);
-  GNUNET_TRANSPORT_disconnect (rocc->tcc.th);
+  //GNUNET_TRANSPORT_disconnect (rocc->tcc.th_);
+  GST_cache_get_handle_done (rocc->tcc.cgh_th);
+  GNUNET_assert (rocc->peer->reference_cnt > 0);
   rocc->peer->reference_cnt--;
-  if ((GNUNET_YES == rocc->peer->destroy_flag)
-      && (0 == rocc->peer->reference_cnt))
+  if ((GNUNET_YES == rocc->peer->destroy_flag) &&
+      (0 == rocc->peer->reference_cnt))
     GST_destroy_peer (rocc->peer);
   GNUNET_free_non_null (rocc->hello);
   GNUNET_CONTAINER_DLL_remove (roccq_head, roccq_tail, rocc);
@@ -1248,14 +1527,14 @@ cleanup_rocc (struct RequestOverlayConnectContext *rocc)
 /**
  * Task to timeout rocc and cleanit up
  *
- * @param cls the RequestOverlayConnectContext
+ * @param cls the RemoteOverlayConnectCtx
  * @param tc the TaskContext from scheduler
  */
 static void
 timeout_rocc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct RequestOverlayConnectContext *rocc = cls;
-  
+  struct RemoteOverlayConnectCtx *rocc = cls;
+
   GNUNET_assert (rocc->timeout_rocc_task_id != GNUNET_SCHEDULER_NO_TASK);
   rocc->timeout_rocc_task_id = GNUNET_SCHEDULER_NO_TASK;
   LOG_DEBUG ("0x%llx: rocc timed out\n", rocc->op_id);
@@ -1267,21 +1546,19 @@ timeout_rocc_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
  * Function called to notify transport users that another
  * peer connected to us.
  *
- * @param cls closure
+ * @param cls the RemoteOverlayConnectContext
  * @param new_peer the peer that connected
- * @param ats performance data
- * @param ats_count number of entries in ats (excluding 0-termination)
  */
-static void 
-transport_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
-                          const struct GNUNET_ATS_Information * ats,
-                          uint32_t ats_count)
+static void
+cache_transport_peer_connect_notify (void *cls,
+                                     const struct GNUNET_PeerIdentity *new_peer)
 {
-  struct RequestOverlayConnectContext *rocc = cls;
+  struct RemoteOverlayConnectCtx *rocc = cls;
 
   LOG_DEBUG ("0x%llx: Request Overlay connect notify\n", rocc->op_id);
-  if (0 != memcmp (new_peer, &rocc->a_id, sizeof (struct GNUNET_PeerIdentity)))
-    return;
+  GNUNET_assert (0 ==
+                 memcmp (new_peer, &rocc->a_id,
+                         sizeof (struct GNUNET_PeerIdentity)));
   LOG_DEBUG ("0x%llx: Peer %4s connected\n", rocc->op_id,
              GNUNET_i2s (&rocc->a_id));
   cleanup_rocc (rocc);
@@ -1290,9 +1567,9 @@ transport_connect_notify (void *cls, const struct GNUNET_PeerIdentity *new_peer,
 
 /**
  * Task to offer the HELLO message to the peer and ask it to connect to the peer
- * whose identity is in RequestOverlayConnectContext
+ * whose identity is in RemoteOverlayConnectCtx
  *
- * @param cls the RequestOverlayConnectContext
+ * @param cls the RemoteOverlayConnectCtx
  * @param tc the TaskContext from scheduler
  */
 static void
@@ -1310,8 +1587,8 @@ attempt_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
 static void
 rocc_hello_sent_cb (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct RequestOverlayConnectContext *rocc = cls;
-  
+  struct RemoteOverlayConnectCtx *rocc = cls;
+
   rocc->ohh = NULL;
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == rocc->attempt_connect_task_id);
   LOG_DEBUG ("0x%llx: HELLO of peer %4s sent to local peer with id: %u\n",
@@ -1320,41 +1597,81 @@ rocc_hello_sent_cb (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   {
     GNUNET_break (0);
     rocc->attempt_connect_task_id =
-        GNUNET_SCHEDULER_add_now (&attempt_connect_task,
-                                  rocc);
+        GNUNET_SCHEDULER_add_now (&attempt_connect_task, rocc);
     return;
   }
   if (GNUNET_SCHEDULER_REASON_READ_READY != tc->reason)
+  {
+    GNUNET_break (0);
     return;
+  }
   rocc->tcc.task = GNUNET_SCHEDULER_add_now (&try_connect_task, &rocc->tcc);
 }
 
 
 /**
  * Task to offer the HELLO message to the peer and ask it to connect to the peer
- * whose identity is in RequestOverlayConnectContext
+ * whose identity is in RemoteOverlayConnectCtx
  *
- * @param cls the RequestOverlayConnectContext
+ * @param cls the RemoteOverlayConnectCtx
  * @param tc the TaskContext from scheduler
  */
 static void
 attempt_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
-  struct RequestOverlayConnectContext *rocc = cls;
+  struct RemoteOverlayConnectCtx *rocc = cls;
 
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != rocc->attempt_connect_task_id);
   rocc->attempt_connect_task_id = GNUNET_SCHEDULER_NO_TASK;
   LOG_DEBUG ("0x%llx: Offering HELLO of peer %4s to local peer with id: %u\n",
              rocc->op_id, GNUNET_i2s (&rocc->a_id), rocc->peer->id);
-  rocc->ohh = GNUNET_TRANSPORT_offer_hello (rocc->tcc.th, rocc->hello,
-                                            rocc_hello_sent_cb, rocc);
+  rocc->ohh =
+      GNUNET_TRANSPORT_offer_hello (rocc->tcc.th_, rocc->hello,
+                                    rocc_hello_sent_cb, rocc);
   if (NULL == rocc->ohh)
-    rocc->attempt_connect_task_id = 
-        GNUNET_SCHEDULER_add_delayed 
-        (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
-                                        100 + GNUNET_CRYPTO_random_u32
-                                        (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
-         &attempt_connect_task, rocc);
+    rocc->attempt_connect_task_id =
+        GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                      (GNUNET_TIME_UNIT_MILLISECONDS,
+                                       100 +
+                                       GNUNET_CRYPTO_random_u32
+                                       (GNUNET_CRYPTO_QUALITY_WEAK, 500)),
+                                      &attempt_connect_task, rocc);
+}
+
+
+/**
+ * Callback from cache with needed handles set
+ *
+ * @param cls the closure passed to GST_cache_get_handle_transport()
+ * @param ch the handle to CORE. Can be NULL if it is not requested
+ * @param th the handle to TRANSPORT. Can be NULL if it is not requested
+ * @param ignore_ peer identity which is ignored in this callback
+ */
+static void
+rocc_cache_get_handle_transport_cb (void *cls, struct GNUNET_CORE_Handle *ch,
+                                    struct GNUNET_TRANSPORT_Handle *th,
+                                    const struct GNUNET_PeerIdentity *ignore_)
+{
+  struct RemoteOverlayConnectCtx *rocc = cls;
+
+  if (NULL == th)
+  {
+    rocc->timeout_rocc_task_id =
+        GNUNET_SCHEDULER_add_now (&timeout_rocc_task, rocc);
+    return;
+  }
+  rocc->tcc.th_ = th;
+  rocc->tcc.pid = &rocc->a_id;
+  if (GNUNET_YES ==
+      GNUNET_TRANSPORT_check_neighbour_connected (rocc->tcc.th_, rocc->tcc.pid))
+  {
+    LOG_DEBUG ("0x%llx: Target peer %4s already connected to local peer: %u\n",
+               rocc->op_id, GNUNET_i2s (&rocc->a_id), rocc->peer->id);
+    cleanup_rocc (rocc);
+    return;
+  }
+  rocc->attempt_connect_task_id =
+      GNUNET_SCHEDULER_add_now (&attempt_connect_task, rocc);
 }
 
 
@@ -1366,42 +1683,45 @@ attempt_connect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
  * @param message the actual message
  */
 void
-GST_handle_overlay_request_connect (void *cls,
-                                        struct GNUNET_SERVER_Client *client, 
-                                        const struct GNUNET_MessageHeader
-                                        *message)
+GST_handle_remote_overlay_connect (void *cls,
+                                   struct GNUNET_SERVER_Client *client,
+                                   const struct GNUNET_MessageHeader *message)
 {
-  const struct GNUNET_TESTBED_RequestConnectMessage *msg;
-  struct RequestOverlayConnectContext *rocc;
+  const struct GNUNET_TESTBED_RemoteOverlayConnectMessage *msg;
+  struct RemoteOverlayConnectCtx *rocc;
   struct Peer *peer;
+  struct GNUNET_PeerIdentity pid;
+  static char pid_str[16];
   uint32_t peer_id;
   uint16_t msize;
   uint16_t hsize;
-  
+
   msize = ntohs (message->size);
-  if (sizeof (struct GNUNET_TESTBED_RequestConnectMessage) >= msize)
+  if (sizeof (struct GNUNET_TESTBED_RemoteOverlayConnectMessage) >= msize)
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
-  }  
-  msg = (const struct GNUNET_TESTBED_RequestConnectMessage *) message;
-  if ((NULL == msg->hello) || 
-      (GNUNET_MESSAGE_TYPE_HELLO != ntohs (msg->hello->type)))
+  }
+  msg = (const struct GNUNET_TESTBED_RemoteOverlayConnectMessage *) message;
+  if ((NULL == msg->hello) ||
+      ((GNUNET_MESSAGE_TYPE_HELLO != ntohs (msg->hello->type))))
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
   hsize = ntohs (msg->hello->size);
-  if ((sizeof (struct GNUNET_TESTBED_RequestConnectMessage) + hsize) != msize)
+  if ((sizeof (struct GNUNET_TESTBED_RemoteOverlayConnectMessage) + hsize) !=
+      msize)
   {
     GNUNET_break (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
     return;
   }
   peer_id = ntohl (msg->peer);
-  if ((peer_id >= GST_peer_list_size) || (NULL == (peer = GST_peer_list[peer_id])))
+  if ((peer_id >= GST_peer_list_size) ||
+      (NULL == (peer = GST_peer_list[peer_id])))
   {
     GNUNET_break_op (0);
     GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
@@ -1410,43 +1730,35 @@ GST_handle_overlay_request_connect (void *cls,
   if (GNUNET_YES == peer->is_remote)
   {
     struct GNUNET_MessageHeader *msg2;
-    
+
     msg2 = GNUNET_copy_message (message);
     GNUNET_TESTBED_queue_message_ (peer->details.remote.slave->controller,
                                    msg2);
     GNUNET_SERVER_receive_done (client, GNUNET_OK);
     return;
   }
-  rocc = GNUNET_malloc (sizeof (struct RequestOverlayConnectContext));
+  rocc = GNUNET_malloc (sizeof (struct RemoteOverlayConnectCtx));
   rocc->op_id = GNUNET_ntohll (msg->operation_id);
   GNUNET_CONTAINER_DLL_insert_tail (roccq_head, roccq_tail, rocc);
   memcpy (&rocc->a_id, &msg->peer_identity,
           sizeof (struct GNUNET_PeerIdentity));
-  LOG_DEBUG ("Received request for overlay connection with op_id: 0x%llx "
-             "from local peer %u to peer %4s with hello size: %u\n",
-             rocc->op_id,
-            peer_id,
-             GNUNET_i2s (&rocc->a_id),
-            hsize);
+  GNUNET_TESTING_peer_get_identity (peer->details.local.peer, &pid);
+  (void) strncpy (pid_str, GNUNET_i2s (&pid), 15);
+  LOG_DEBUG ("0x%llx: Remote overlay connect %4s to peer %4s with hello size: %u\n",
+             rocc->op_id, pid_str, GNUNET_i2s (&rocc->a_id), hsize);
   rocc->peer = peer;
   rocc->peer->reference_cnt++;
-  rocc->tcc.op_id = rocc->op_id;
-  rocc->tcc.th = GNUNET_TRANSPORT_connect (rocc->peer->details.local.cfg, NULL, rocc,
-                                           NULL, &transport_connect_notify, NULL);
-  if (NULL == rocc->tcc.th)
-  {
-    GNUNET_break (0);
-    GNUNET_free (rocc);
-    GNUNET_SERVER_receive_done (client, GNUNET_SYSERR);
-    return;
-  }
-  rocc->tcc.pid = &rocc->a_id;
   rocc->hello = GNUNET_malloc (hsize);
   memcpy (rocc->hello, msg->hello, hsize);
-  rocc->attempt_connect_task_id =
-      GNUNET_SCHEDULER_add_now (&attempt_connect_task, rocc);
+  rocc->tcc.op_id = rocc->op_id;
+  rocc->tcc.cgh_th =
+      GST_cache_get_handle_transport (peer_id, rocc->peer->details.local.cfg,
+                                      &rocc_cache_get_handle_transport_cb, rocc,
+                                      &rocc->a_id,
+                                      &cache_transport_peer_connect_notify,
+                                      rocc);
   rocc->timeout_rocc_task_id =
-      GNUNET_SCHEDULER_add_delayed (TIMEOUT, &timeout_rocc_task, rocc);
+      GNUNET_SCHEDULER_add_delayed (GST_timeout, &timeout_rocc_task, rocc);
   GNUNET_SERVER_receive_done (client, GNUNET_OK);
 }
 
@@ -1470,7 +1782,7 @@ GST_free_occq ()
 void
 GST_free_roccq ()
 {
-  struct RequestOverlayConnectContext *rocc;
+  struct RemoteOverlayConnectCtx *rocc;
 
   while (NULL != (rocc = roccq_head))
     cleanup_rocc (rocc);