- distribute peers equally among island nodes on SuperMUC
[oweals/gnunet.git] / src / testbed / testbed_api_testbed.c
index 194d54a5872401f1cbb62a7d9e75650cec6d1f0f..c4cfe59b114da6f0d3ab77f5cf616eab82689531 100644 (file)
@@ -1,6 +1,6 @@
 /*
   This file is part of GNUnet
-  (C) 2008--2012 Christian Grothoff (and other contributing authors)
+  (C) 2008--2013 Christian Grothoff (and other contributing authors)
 
   GNUnet is free software; you can redistribute it and/or modify
   it under the terms of the GNU General Public License as published
@@ -26,6 +26,7 @@
  */
 
 #include "platform.h"
+#include "gnunet_util_lib.h"
 #include "gnunet_testbed_service.h"
 #include "testbed_api_peers.h"
 #include "testbed_api_hosts.h"
 #define LOG(kind,...)                                           \
   GNUNET_log_from (kind, "testbed-api-testbed", __VA_ARGS__)
 
+/**
+ * Debug logging shortcut
+ */
+#define DEBUG(...)                              \
+  LOG (GNUNET_ERROR_TYPE_DEBUG, __VA_ARGS__)
+
+/**
+ * The default setup timeout in seconds
+ */
+#define DEFAULT_SETUP_TIMEOUT 300
+
+/**
+ * Testbed Run Handle
+ */
+struct RunContext;
 
 /**
- * DLL of operations
+ * Context information for the operation we start
  */
-struct DLLOperation
+struct RunContextOperation
 {
   /**
    * The testbed operation handle
@@ -58,15 +74,6 @@ struct DLLOperation
    */
   void *cls;
 
-  /**
-   * The next pointer for DLL
-   */
-  struct DLLOperation *next;
-
-  /**
-   * The prev pointer for DLL
-   */
-  struct DLLOperation *prev;
 };
 
 
@@ -80,11 +87,23 @@ enum State
    */
   RC_INIT = 0,
 
+#if ENABLE_LL
+  /**
+   * Island level controllers are started and linked
+   */
+  RC_ISLANDS_LINKED,
+#endif
+
   /**
    * Controllers on given hosts started and linked
    */
   RC_LINKED,
 
+  /**
+   * Peers are created
+   */
+  RC_PEERS_CREATED,
+
   /**
    * The testbed run is ready and the master callback can be called now. At this
    * time the peers are all started and if a topology is provided in the
@@ -92,17 +111,61 @@ enum State
    */
   RC_READY,
 
+  /* /\** */
+  /*  * Peers are stopped */
+  /*  *\/ */
+  /* RC_PEERS_STOPPED, */
+
+  /* /\** */
+  /*  * Peers are destroyed */
+  /*  *\/ */
+  /* RC_PEERS_DESTROYED */
+
+  /**
+   * All peers shutdown (stopped and destroyed)
+   */
+  RC_PEERS_SHUTDOWN
+};
+
+
+/**
+ * Context for host compability checks
+ */
+struct CompatibilityCheckContext
+{
+  /**
+   * The run context
+   */
+  struct RunContext *rc;
+
   /**
-   * Peers are stopped
+   * Handle for the compability check
    */
-  RC_PEERS_STOPPED,
+  struct GNUNET_TESTBED_HostHabitableCheckHandle *h;
 
   /**
-   * Peers are destroyed
+   * Index of the host in the run context's hosts array
    */
-  RC_PEERS_DESTROYED
+  unsigned int index;
 };
 
+#if ENABLE_LL
+/**
+ * Structure to represent an island of SuperMUC's nodes
+ */
+struct Island
+{
+  /**
+   * Array of nodes in this island
+   */
+  struct GNUNET_TESTBED_Host **hosts;
+
+  /**
+   * Number of nodes in the above array
+   */
+  unsigned int nhosts;
+};
+#endif
 
 /**
  * Testbed Run Handle
@@ -150,31 +213,33 @@ struct RunContext
    * TestMaster callback to call when testbed initialization is done
    */
   GNUNET_TESTBED_TestMaster test_master;
-  
+
   /**
    * The closure for the TestMaster callback
    */
   void *test_master_cls;
 
   /**
-   * The head element of DLL operations
+   * A hashmap for operations started by us
    */
-  struct DLLOperation *dll_op_head;
+  struct GNUNET_CONTAINER_MultiHashMap32 *rcop_map;
 
   /**
-   * The tail element of DLL operations
+   * An array of hosts loaded from the hostkeys file
    */
-  struct DLLOperation *dll_op_tail;
+  struct GNUNET_TESTBED_Host **hosts;
 
+#if ENABLE_LL
   /**
-   * An array of hosts loaded from the hostkeys file
+   * Array of SuperMUC islands
    */
-  struct GNUNET_TESTBED_Host **hosts;
+  struct Island **islands;
+#endif
 
   /**
-   * The handle for whether a host is habitable or not
+   * Array of compatibility check contexts
    */
-  struct GNUNET_TESTBED_HostHabitableCheckHandle **hc_handles;
+  struct CompatibilityCheckContext *hclist;
 
   /**
    * Array of peers which we create
@@ -188,7 +253,8 @@ struct RunContext
   struct GNUNET_TESTBED_Operation *topology_operation;
 
   /**
-   * The file containing topology data. Only used if the topology is set to 'FROM_FILE'
+   * The file containing topology data. Only used if the topology is set to
+   * 'FROM_FILE'
    */
   char *topo_file;
 
@@ -197,6 +263,11 @@ struct RunContext
    */
   struct GNUNET_TESTBED_HostRegistrationHandle *reg_handle;
 
+  /**
+   * Profiling start time
+   */
+  struct GNUNET_TIME_Absolute pstart_time;
+
   /**
    * Host registration task
    */
@@ -207,6 +278,16 @@ struct RunContext
    */
   GNUNET_SCHEDULER_TaskIdentifier shutdown_run_task;
 
+  /**
+   * Task to be run of a timeout
+   */
+  GNUNET_SCHEDULER_TaskIdentifier timeout_task;
+
+  /**
+   * Task run upon shutdown interrupts
+   */
+  GNUNET_SCHEDULER_TaskIdentifier interrupt_task;
+
   /**
    * The event mask for the controller
    */
@@ -249,12 +330,6 @@ struct RunContext
    */
   unsigned int num_peers;
 
-  /**
-   * counter to count overlay connect attempts. This counter includes both
-   * successful and failed overlay connects
-   */
-  unsigned int oc_count;
-
   /**
    * Expected overlay connects. Should be zero if no topology is relavant
    */
@@ -264,72 +339,159 @@ struct RunContext
    * Number of random links to established
    */
   unsigned int random_links;
-  
+
+  /**
+   * the number of overlay link connection attempts that succeeded
+   */
+  unsigned int links_succeeded;
+
+  /**
+   * the number of overlay link connection attempts that failed
+   */
+  unsigned int links_failed;
+
+#if ENABLE_LL
+  /**
+   * Number of SuperMUC islands we are running on
+   */
+  unsigned int nislands;
+#endif
 };
 
 
 /**
- * Task for starting peers
+ * Return a 32-bit key from a pointer
  *
- * @param cls the RunHandle
- * @param tc the task context from scheduler
+ * @param rcop the pointer
+ * @return 32-bit key
  */
-static void
-start_peers_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+static uint32_t
+rcop_key (void *rcop)
+{  
+  return * ((uint32_t *) &rcop);
+}
+
+
+/**
+ * Context information used for finding a pointer in the rcop_map
+ */
+struct SearchContext
 {
-  struct RunContext *rc = cls;
-  struct DLLOperation *dll_op;
-  unsigned int peer;
+  /**
+   * The operation pointer to look for
+   */
+  struct GNUNET_TESTBED_Operation *query;
 
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Starting Peers\n");
-  for (peer = 0; peer < rc->num_peers; peer++)
+  /**
+   * The Run context operation which has the operation being queried
+   */
+  struct RunContextOperation *result;
+};
+
+
+/**
+ * Iterator for searching over the elements matching a given query
+ *
+ * @param cls the SearchContext
+ * @param key the 32-bit key
+ * @param value the RunContextOperation element
+ * @return GNUNET_YES to continue iteration; GNUNET_NO to cancel it
+ */
+static int
+search_iterator (void *cls, uint32_t key, void *value)
+{
+  struct RunContextOperation *rcop = value;
+  struct SearchContext *sc = cls;
+
+  GNUNET_assert (NULL != rcop);
+  if (sc->query == rcop->op)
   {
-    dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-    dll_op->op = GNUNET_TESTBED_peer_start (NULL, rc->peers[peer], NULL, NULL);
-    dll_op->cls = rc->peers[peer];
-    GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail, dll_op);
+    GNUNET_assert (NULL == sc->result);
+    sc->result = rcop;
+    return GNUNET_NO;
   }
-  rc->peer_count = 0;
+  return GNUNET_YES;
 }
 
 
 /**
- * Functions of this signature are called when a peer has been successfully
- * created
+ * Initiate a search for the given operation in the rcop_map
  *
- * @param cls the closure from GNUNET_TESTBED_peer_create()
- * @param peer the handle for the created peer; NULL on any error during
- *          creation
- * @param emsg NULL if peer is not NULL; else MAY contain the error description
+ * @param rc the RunContext whose rcop_map will be searched for the given
+ *          operation
+ * @param op the given operation to search for
+ * @return the matching RunContextOperation if found; NULL if not
+ */
+static struct RunContextOperation *
+search_rcop (struct RunContext *rc, struct GNUNET_TESTBED_Operation *op)
+{
+  struct SearchContext sc;
+  
+  sc.query = op;
+  sc.result = NULL;
+  if (GNUNET_SYSERR == 
+      GNUNET_CONTAINER_multihashmap32_get_multiple (rc->rcop_map,
+                                                    rcop_key (op),
+                                                    &search_iterator,
+                                                    &sc))
+  {
+    GNUNET_assert (NULL != sc.result);
+    return sc.result;
+  }
+  return NULL;
+}
+
+
+/**
+ * Insert an RunContextOperation into the rcop_map of the given RunContext
+ *
+ * @param rc the RunContext into whose map is to be used for insertion
+ * @param rcop the RunContextOperation to insert
  */
 static void
-peer_create_cb (void *cls, struct GNUNET_TESTBED_Peer *peer, const char *emsg)
+insert_rcop (struct RunContext *rc, struct RunContextOperation *rcop)
 {
-  struct DLLOperation *dll_op = cls;
-  struct RunContext *rc;
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap32_put (rc->rcop_map,
+                                                      rcop_key (rcop->op), rcop,
+                                                      GNUNET_CONTAINER_MULTIHASHMAPOPTION_MULTIPLE));
+}
 
-  GNUNET_assert (NULL != dll_op);
-  rc = dll_op->rc;
-  GNUNET_assert (NULL != rc);
-  GNUNET_CONTAINER_DLL_remove (rc->dll_op_head, rc->dll_op_tail, dll_op);
-  GNUNET_TESTBED_operation_done (dll_op->op);
-  GNUNET_free (dll_op);
-  if (NULL == peer)
+
+/**
+ * Remove a RunContextOperation from the rcop_map of the given RunContext
+ *
+ * @param rc the RunContext from whose map the given RunContextOperaton has to
+ *          be removed
+ * @param rcop the RunContextOperation
+ */
+static void
+remove_rcop (struct RunContext *rc, struct RunContextOperation *rcop)
+{
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap32_remove (rc->rcop_map,
+                                                         rcop_key (rcop->op),
+                                                         rcop));
+}
+
+#if ENABLE_LL
+static void
+cleanup_islands (struct RunContext *rc)
+{
+  struct Island *island;
+  unsigned int cnt;
+
+  GNUNET_assert (NULL != rc->islands);
+  for (cnt = 0; cnt < rc->nislands; cnt++)
   {
-    if (NULL != emsg)
-      LOG (GNUNET_ERROR_TYPE_WARNING, "Error while creating a peer: %s\n",
-           emsg);
-    /* FIXME: GNUNET_TESTBED_shutdown_run()? */
-    return;
+    island = rc->islands[cnt];
+    GNUNET_free (island->hosts);
+    GNUNET_free (island);
   }
-  rc->peers[rc->peer_count] = peer;
-  rc->peer_count++;
-  if (rc->peer_count < rc->num_peers)
-    return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "%u peers created successfully\n", rc->num_peers);
-  GNUNET_SCHEDULER_add_now (&start_peers_task, rc);
+  GNUNET_free (rc->islands);
+  rc->islands = NULL;
 }
-
+#endif
 
 /**
  * Assuming all peers have been destroyed cleanup run handle
@@ -341,31 +503,27 @@ static void
 cleanup_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct RunContext *rc = cls;
-  struct DLLOperation *dll_op;
-  unsigned int hid;
+  unsigned int cnt;
 
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == rc->register_hosts_task);
   GNUNET_assert (NULL == rc->reg_handle);
   GNUNET_assert (NULL == rc->peers);
-  GNUNET_assert (NULL == rc->hc_handles);
-  GNUNET_assert (RC_PEERS_DESTROYED == rc->state);
-  if (NULL != rc->dll_op_head)
-  {                             /* cancel our pending operations */
-    while (NULL != (dll_op = rc->dll_op_head))
-    {
-      GNUNET_TESTBED_operation_done (dll_op->op);
-      GNUNET_CONTAINER_DLL_remove (rc->dll_op_head, rc->dll_op_tail, dll_op);
-      GNUNET_free (dll_op);
-    }
-  }
+  GNUNET_assert (NULL == rc->hclist);
+  GNUNET_assert (RC_PEERS_SHUTDOWN == rc->state);
+  GNUNET_assert (0 == GNUNET_CONTAINER_multihashmap32_size (rc->rcop_map));
+  GNUNET_CONTAINER_multihashmap32_destroy (rc->rcop_map);
   if (NULL != rc->c)
     GNUNET_TESTBED_controller_disconnect (rc->c);
   if (NULL != rc->cproc)
     GNUNET_TESTBED_controller_stop (rc->cproc);
   if (NULL != rc->h)
     GNUNET_TESTBED_host_destroy (rc->h);
-  for (hid = 0; hid < rc->num_hosts; hid++)
-        GNUNET_TESTBED_host_destroy (rc->hosts[hid]);
+#if ENABLE_LL
+  if (NULL != rc->islands)
+    cleanup_islands (rc);
+#endif
+  for (cnt = 0; cnt < rc->num_hosts; cnt++)
+    GNUNET_TESTBED_host_destroy (rc->hosts[cnt]);
   GNUNET_free_non_null (rc->hosts);
   if (NULL != rc->cfg)
     GNUNET_CONFIGURATION_destroy (rc->cfg);
@@ -376,28 +534,81 @@ cleanup_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
 
 /**
- * Stops the testbed run and releases any used resources
+ * Iterator for cleaning up elements from rcop_map 
  *
- * @param cls the tesbed run handle
- * @param tc the task context from scheduler
+ * @param cls the RunContext
+ * @param key the 32-bit key
+ * @param value the RunContextOperation element
+ * @return always GNUNET_YES
  */
-static void
-shutdown_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+static int
+rcop_cleanup_iterator (void *cls, uint32_t key, void *value)
+{
+  struct RunContext *rc = cls;
+  struct RunContextOperation *rcop = value;
+
+  GNUNET_assert (rc == rcop->rc);
+  remove_rcop (rc, rcop);
+  GNUNET_TESTBED_operation_done (rcop->op);
+  GNUNET_free (rcop);
+  return GNUNET_YES;
+}
 
 
 /**
- * Function to shutdown now
+ * Frees memory, closes pending operations, cancels actives tasks of the given
+ * RunContext 
  *
  * @param rc the RunContext
  */
 static void
-shutdown_now (struct RunContext *rc)
+cleanup (struct RunContext *rc)
 {
-  if (GNUNET_YES == rc->shutdown)
-    return;
-  if (GNUNET_SCHEDULER_NO_TASK != rc->shutdown_run_task)
-    GNUNET_SCHEDULER_cancel (rc->shutdown_run_task);
-  rc->shutdown_run_task = GNUNET_SCHEDULER_add_now (&shutdown_run, rc);
+  struct CompatibilityCheckContext *hc;
+  unsigned int nhost;
+
+  if (NULL != rc->hclist)
+  {
+    for (nhost = 0; nhost < rc->num_hosts; nhost++)
+    {
+      hc = &rc->hclist[nhost];
+      if (NULL != hc->h)
+        GNUNET_TESTBED_is_host_habitable_cancel (hc->h);
+    }
+    GNUNET_free (rc->hclist);
+    rc->hclist = NULL;
+  }
+  /* Stop register hosts task if it is running */
+  if (GNUNET_SCHEDULER_NO_TASK != rc->register_hosts_task)
+  {
+    GNUNET_SCHEDULER_cancel (rc->register_hosts_task);
+    rc->register_hosts_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (GNUNET_SCHEDULER_NO_TASK != rc->timeout_task)
+  {
+    GNUNET_SCHEDULER_cancel (rc->timeout_task);
+    rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (GNUNET_SCHEDULER_NO_TASK != rc->interrupt_task)
+  {
+    GNUNET_SCHEDULER_cancel (rc->interrupt_task);
+    rc->interrupt_task = GNUNET_SCHEDULER_NO_TASK;
+  }
+  if (NULL != rc->reg_handle)
+  {
+    GNUNET_TESTBED_cancel_registration (rc->reg_handle);
+    rc->reg_handle = NULL;
+  }
+  if (NULL != rc->topology_operation)
+  {
+    GNUNET_TESTBED_operation_done (rc->topology_operation);
+    rc->topology_operation = NULL;
+  }
+  /* cancel any exiting operations */
+  GNUNET_assert (GNUNET_SYSERR != 
+                 GNUNET_CONTAINER_multihashmap32_iterate (rc->rcop_map,
+                                                          &rcop_cleanup_iterator,
+                                                          rc));
 }
 
 
@@ -411,117 +622,187 @@ static void
 shutdown_run (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct RunContext *rc = cls;
-  struct DLLOperation *dll_op;
-  int all_peers_destroyed;
-  unsigned int peer;
-  unsigned int nhost;
+  struct RunContextOperation *rcop;
 
   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK != rc->shutdown_run_task);
   rc->shutdown_run_task = GNUNET_SCHEDULER_NO_TASK;
   GNUNET_assert (GNUNET_NO == rc->shutdown);
   rc->shutdown = GNUNET_YES;
-  if (NULL != rc->hc_handles)
-  {
-    for (nhost = 0; nhost < rc->num_hosts; nhost++)
-      if (NULL != rc->hc_handles[nhost])
-        GNUNET_TESTBED_is_host_habitable_cancel (rc->hc_handles[nhost]);
-    GNUNET_free (rc->hc_handles);
-    rc->hc_handles = NULL;
-  }
-  /* Stop register hosts task if it is running */
-  if (GNUNET_SCHEDULER_NO_TASK != rc->register_hosts_task)
-  {
-    GNUNET_SCHEDULER_cancel (rc->register_hosts_task);
-    rc->register_hosts_task = GNUNET_SCHEDULER_NO_TASK;
-  }
-  if (NULL != rc->reg_handle)
-  {
-    GNUNET_TESTBED_cancel_registration (rc->reg_handle);
-    rc->reg_handle = NULL;
-  }
+  cleanup (rc);
   if (NULL != rc->c)
   {
     if (NULL != rc->peers)
     {
-      if (NULL != rc->topology_operation)
-      {
-        GNUNET_TESTBED_operation_done (rc->topology_operation);
-        rc->topology_operation = NULL;
-      }
-      if (RC_INIT == rc->state)
-        rc->state = RC_READY;   /* Even though we haven't called the master callback */
-      rc->peer_count = 0;
-      /* Check if some peers are stopped */
-      for (peer = 0; peer < rc->num_peers; peer++)
-      {
-        if (NULL == rc->peers[peer])
-          continue;
-        if (PS_STOPPED != rc->peers[peer]->state)
-          break;
-      }
-      if (peer == rc->num_peers)
-      {
-        /* All peers are stopped */
-        rc->state = RC_PEERS_STOPPED;
-        all_peers_destroyed = GNUNET_YES;
-        for (peer = 0; peer < rc->num_peers; peer++)
-        {
-          if (NULL == rc->peers[peer])
-            continue;
-          all_peers_destroyed = GNUNET_NO;
-          dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-          dll_op->op = GNUNET_TESTBED_peer_destroy (rc->peers[peer]);
-          GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail,
-                                            dll_op);
-        }
-        if (all_peers_destroyed == GNUNET_NO)
-          return;
-      }
-      /* Some peers are stopped */
-      for (peer = 0; peer < rc->num_peers; peer++)
-      {
-        if ((NULL == rc->peers[peer]) || (PS_STARTED != rc->peers[peer]->state))
-        {
-          rc->peer_count++;
-          continue;
-        }
-        LOG (GNUNET_ERROR_TYPE_DEBUG, "Stopping peer %u\n", peer);
-        dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-        dll_op->op = GNUNET_TESTBED_peer_stop (rc->peers[peer], NULL, NULL);
-        dll_op->cls = rc->peers[peer];
-        GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail,
-                                          dll_op);
-      }
-      if (rc->peer_count != rc->num_peers)
-        return;
-      GNUNET_free (rc->peers);
-      rc->peers = NULL;
-    }    
+      rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+      rcop->rc = rc;
+      rcop->op = GNUNET_TESTBED_shutdown_peers (rc->c, rcop, NULL, NULL);
+      GNUNET_assert (NULL != rcop->op);
+      DEBUG ("Shutting down peers\n");
+      rc->pstart_time = GNUNET_TIME_absolute_get ();
+      insert_rcop (rc, rcop);
+      return;
+    }
   }
-  rc->state = RC_PEERS_DESTROYED;       /* No peers are present so we consider the
-                                         * state where all peers are destroyed  */
+  rc->state = RC_PEERS_SHUTDOWN;       /* No peers are present so we consider the
+                                        * state where all peers are SHUTDOWN  */
   GNUNET_SCHEDULER_add_now (&cleanup_task, rc);
 }
 
 
 /**
- * Task to call master task
+ * Function to shutdown now
  *
- * @param cls the run context
- * @param tc the task context
+ * @param rc the RunContext
  */
 static void
-call_master (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+shutdown_now (struct RunContext *rc)
+{
+  if (GNUNET_YES == rc->shutdown)
+    return;
+  if (GNUNET_SCHEDULER_NO_TASK != rc->shutdown_run_task)
+    GNUNET_SCHEDULER_cancel (rc->shutdown_run_task);
+  GNUNET_SCHEDULER_shutdown (); /* Trigger shutdown in programs using this API */
+  rc->shutdown_run_task = GNUNET_SCHEDULER_add_now (&shutdown_run, rc);
+}
+
+
+/**
+ * Task run upon any interrupt.  Common ones are SIGINT & SIGTERM.
+ *
+ * @param cls the RunContext which has to be acted upon
+ * @param tc the scheduler task context
+ */
+static void
+interrupt (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct RunContext *rc = cls;
-  
-  if (NULL != rc->topology_operation)
+
+  rc->interrupt_task = GNUNET_SCHEDULER_NO_TASK;
+  shutdown_now (rc);
+}
+
+
+/**
+ * Function to return the string representation of the duration between current
+ * time and `pstart_time' in `RunContext'
+ *
+ * @param rc the RunContext
+ * @return the representation string; this is NOT reentrant
+ */
+static const char *
+prof_time (struct RunContext *rc)
+{
+  struct GNUNET_TIME_Relative ptime;
+
+  ptime = GNUNET_TIME_absolute_get_duration (rc->pstart_time);
+  return GNUNET_STRINGS_relative_time_to_string (ptime, GNUNET_YES);
+}
+
+
+/**
+ * Task for starting peers
+ *
+ * @param cls the RunHandle
+ * @param tc the task context from scheduler
+ */
+static void
+start_peers_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct RunContext *rc = cls;
+  struct RunContextOperation *rcop;
+  unsigned int peer;
+
+  DEBUG ("Starting Peers\n");
+  rc->pstart_time = GNUNET_TIME_absolute_get ();
+  for (peer = 0; peer < rc->num_peers; peer++)
   {
-    GNUNET_TESTBED_operation_done (rc->topology_operation);
-    rc->topology_operation = NULL;
+    rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+    rcop->rc = rc;
+    rcop->op  = GNUNET_TESTBED_peer_start (NULL, rc->peers[peer], NULL, NULL);
+    GNUNET_assert (NULL != rcop->op);
+    rcop->cls = rc->peers[peer];
+    insert_rcop (rc, rcop);
+  }
+  rc->peer_count = 0;
+}
+
+
+/**
+ * Functions of this signature are called when a peer has been successfully
+ * created
+ *
+ * @param cls the closure from GNUNET_TESTBED_peer_create()
+ * @param peer the handle for the created peer; NULL on any error during
+ *          creation
+ * @param emsg NULL if peer is not NULL; else MAY contain the error description
+ */
+static void
+peer_create_cb (void *cls, struct GNUNET_TESTBED_Peer *peer, const char *emsg)
+{
+  struct RunContextOperation *rcop = cls;
+  struct RunContext *rc;
+
+  GNUNET_assert (NULL != rcop);
+  GNUNET_assert (NULL != (rc = rcop->rc));
+  remove_rcop (rc, rcop);
+  GNUNET_TESTBED_operation_done (rcop->op);
+  GNUNET_free (rcop);
+  if (NULL == peer)
+  {
+    if (NULL != emsg)
+      LOG (GNUNET_ERROR_TYPE_ERROR, "Error while creating a peer: %s\n",
+           emsg);
+    shutdown_now (rc);
+    return;
   }
+  rc->peers[rc->peer_count] = peer;
+  rc->peer_count++;
+  if (rc->peer_count < rc->num_peers)
+    return;
+  DEBUG ("%u peers created in %s\n", rc->num_peers, prof_time (rc));
+  rc->state = RC_PEERS_CREATED;
+  GNUNET_SCHEDULER_add_now (&start_peers_task, rc);
+}
+
+
+/**
+ * call test master callback
+ *
+ * @param rc the RunContext
+ */
+static void
+call_master (struct RunContext *rc)
+{
+  GNUNET_SCHEDULER_cancel (rc->timeout_task);
+  rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
   if (NULL != rc->test_master)
-    rc->test_master (rc->test_master_cls, rc->num_peers, rc->peers);
+    rc->test_master (rc->test_master_cls, rc->num_peers, rc->peers,
+                     rc->links_succeeded, rc->links_failed);
+}
+
+
+/**
+ * Callbacks of this type are called when topology configuration is completed
+ *
+ * @param cls the operation closure given to
+ *          GNUNET_TESTBED_overlay_configure_topology_va() and
+ *          GNUNET_TESTBED_overlay_configure() calls
+ * @param nsuccess the number of successful overlay connects
+ * @param nfailures the number of overlay connects which failed
+ */
+static void
+topology_completion_callback (void *cls, unsigned int nsuccess,
+                              unsigned int nfailures)
+{
+  struct RunContext *rc = cls;
+
+  DEBUG ("Overlay topology generated in %s\n", prof_time (rc));
+  GNUNET_TESTBED_operation_done (rc->topology_operation);
+  rc->topology_operation = NULL;
+  rc->links_succeeded = nsuccess;
+  rc->links_failed = nfailures;
+  rc->state = RC_READY;
+  call_master (rc);
 }
 
 
@@ -533,24 +814,55 @@ call_master (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 static void
 create_peers (struct RunContext *rc)
 {
-  struct DLLOperation *dll_op;
+  struct RunContextOperation *rcop;
+  struct GNUNET_TESTBED_Host *host;
   unsigned int peer;
-
+#if ENABLE_LL
+  struct Island *island;
+  unsigned int icnt;
+  unsigned int hcnt;
+
+  island = NULL;
+  icnt = 0;
+  hcnt = 0;
+#endif
+  DEBUG ("Creating peers\n");
+  rc->pstart_time = GNUNET_TIME_absolute_get ();
   rc->peers =
       GNUNET_malloc (sizeof (struct GNUNET_TESTBED_Peer *) * rc->num_peers);
   GNUNET_assert (NULL != rc->c);
   rc->peer_count = 0;
   for (peer = 0; peer < rc->num_peers; peer++)
   {
-    dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-    dll_op->rc = rc;
-    dll_op->op =
-        GNUNET_TESTBED_peer_create (rc->c, 
-                                    (0 == rc->num_hosts)
-                                    ? rc->h : rc->hosts[peer % rc->num_hosts],
-                                    rc->cfg,
-                                    peer_create_cb, dll_op);
-    GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail, dll_op);
+    rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+    rcop->rc = rc;
+#if ENABLE_LL
+    if (0 != rc->nislands)
+    {
+      island = rc->islands[icnt];
+      if (hcnt == island->nhosts)
+      {
+        icnt++;
+        if (icnt == rc->nislands)
+          icnt = 0;
+        island = rc->islands[icnt];
+        hcnt = 0;
+      }
+      GNUNET_assert (icnt < rc->nislands);
+      GNUNET_assert (hcnt < island->nhosts);
+      GNUNET_assert (NULL != island->hosts[hcnt]);
+      host = island->hosts[hcnt];
+      hcnt++;
+    }
+    else
+      host = rc->h;
+#else
+    host = (0 == rc->num_hosts) ? rc->h : rc->hosts[peer % rc->num_hosts];
+#endif
+    rcop->op = GNUNET_TESTBED_peer_create (rc->c, host, rc->cfg,
+                                           &peer_create_cb, rcop);
+    GNUNET_assert (NULL != rcop->op);
+    insert_rcop (rc, rcop);
   }
 }
 
@@ -566,32 +878,60 @@ static void
 event_cb (void *cls, const struct GNUNET_TESTBED_EventInformation *event)
 {
   struct RunContext *rc = cls;
-  struct DLLOperation *dll_op;
-  unsigned int peer_id;
+  struct RunContextOperation *rcop;
 
   if (RC_INIT == rc->state)
   {
     switch (event->type)
     {
     case GNUNET_TESTBED_ET_OPERATION_FINISHED:
-      dll_op = event->details.operation_finished.op_cls;
+      rcop = event->op_cls;
       if (NULL != event->details.operation_finished.emsg)
       {
-        LOG (GNUNET_ERROR_TYPE_ERROR,
-             _("Linking controllers failed. Exiting"));
+        LOG (GNUNET_ERROR_TYPE_ERROR, _("Linking controllers failed. Exiting"));
         shutdown_now (rc);
       }
       else
         rc->reg_hosts++;
-      GNUNET_assert (event->details.operation_finished.operation == dll_op->op);
-      GNUNET_CONTAINER_DLL_remove (rc->dll_op_head, rc->dll_op_tail, dll_op);
-      GNUNET_TESTBED_operation_done (dll_op->op);
-      GNUNET_free (dll_op);
-      if (rc->reg_hosts == rc->num_hosts)
+      GNUNET_assert (event->op == rcop->op);
+      remove_rcop (rc, rcop);
+      GNUNET_TESTBED_operation_done (rcop->op);
+      GNUNET_free (rcop);
+#if !ENABLE_LL
+      if (rc->reg_hosts != rc->num_hosts)
+        return;
+      rc->state = RC_LINKED;
+      create_peers (rc);
+#else
+      if (rc->reg_hosts != rc->nislands)
+        return;
+      struct Island *island;
+      struct GNUNET_TESTBED_Host *host;
+      unsigned int cnt;
+      unsigned int cnt2;
+      rc->state = RC_ISLANDS_LINKED;
+      rc->reg_hosts = 0;
+      for (cnt = 0; cnt < rc->nislands; cnt++)
       {
-        rc->state = RC_LINKED;
-        create_peers (rc);
+        island = rc->islands[cnt];
+        for (cnt2 = 1; cnt2 < island->nhosts; cnt2++)
+        {
+          host = island->hosts[cnt2];
+          rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+          rcop->rc = rc;
+          rcop->op =
+                GNUNET_TESTBED_controller_link (rcop, rc->c, host,
+                                                island->hosts[0], GNUNET_YES);
+          GNUNET_assert (NULL != rcop->op);
+          insert_rcop (rc, rcop);
+          rc->reg_hosts++;
+        }
       }
+      if (0 != rc->reg_hosts)
+        return;      
+      rc->state = RC_LINKED;
+      create_peers (rc);
+#endif
       return;
     default:
       GNUNET_break (0);
@@ -599,63 +939,62 @@ event_cb (void *cls, const struct GNUNET_TESTBED_EventInformation *event)
       return;
     }
   }
-  if (NULL != rc->topology_operation)
+#if ENABLE_LL
+  if (RC_ISLANDS_LINKED == rc->state)
   {
     switch (event->type)
     {
     case GNUNET_TESTBED_ET_OPERATION_FINISHED:
-    case GNUNET_TESTBED_ET_CONNECT:
-      rc->oc_count++;
-      break;
+      rcop = event->op_cls;
+      if (NULL != event->details.operation_finished.emsg)
+      {
+        LOG (GNUNET_ERROR_TYPE_ERROR,
+             "Linking 2nd level controllers failed. Exiting");
+        shutdown_now (rc);
+      }
+      else
+        rc->reg_hosts--;
+      GNUNET_assert (event->op == rcop->op);
+      remove_rcop (rc, rcop);
+      GNUNET_TESTBED_operation_done (rcop->op);
+      GNUNET_free (rcop);
+      if (0 != rc->reg_hosts)
+        return;
+      rc->state = RC_LINKED;
+      create_peers (rc);
+      return;
     default:
       GNUNET_break (0);
       shutdown_now (rc);
       return;
     }
-    if (rc->oc_count == rc->num_oc)
-    {
-      rc->state = RC_READY;
-      GNUNET_SCHEDULER_add_continuation (&call_master, rc,
-                                         GNUNET_SCHEDULER_REASON_PREREQ_DONE);
-    }
-    return;
   }
-  for (dll_op = rc->dll_op_head; NULL != dll_op; dll_op = dll_op->next)
-  {
-    if ((GNUNET_TESTBED_ET_OPERATION_FINISHED == event->type) &&
-        (event->details.operation_finished.operation == dll_op->op))
-      break;
-    if ((GNUNET_TESTBED_ET_PEER_STOP == event->type) &&
-        (event->details.peer_stop.peer == dll_op->cls))
-      break;
-  }
-  if (NULL == dll_op)
+#endif
+  if (GNUNET_TESTBED_ET_OPERATION_FINISHED != event->type)
     goto call_cc;
-  GNUNET_CONTAINER_DLL_remove (rc->dll_op_head, rc->dll_op_tail, dll_op);
-  GNUNET_TESTBED_operation_done (dll_op->op);
-  GNUNET_free (dll_op);
-  rc->peer_count++;
-  if (rc->peer_count < rc->num_peers)
+  if (NULL == (rcop = search_rcop (rc, event->op)))
+    goto call_cc;
+  remove_rcop (rc, rcop);
+  GNUNET_TESTBED_operation_done (rcop->op);
+  GNUNET_free (rcop);
+  if ( (GNUNET_NO == rc->shutdown)
+       && (NULL != event->details.operation_finished.emsg) )
+  {
+    LOG (GNUNET_ERROR_TYPE_ERROR, "A operation has failed with error: %s\n",
+         event->details.operation_finished.emsg);
+    shutdown_now (rc);
     return;
+  }
+  GNUNET_assert (GNUNET_YES == rc->shutdown);
   switch (rc->state)
   {
   case RC_LINKED:
+  case RC_PEERS_CREATED:
   case RC_READY:
-    rc->state = RC_PEERS_STOPPED;
-    rc->peer_count = 0;
-    for (peer_id = 0; peer_id < rc->num_peers; peer_id++)
-    {
-      dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-      dll_op->op = GNUNET_TESTBED_peer_destroy (rc->peers[peer_id]);
-      GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail,
-                                        dll_op);
-    }
-    break;
-  case RC_PEERS_STOPPED:
-    rc->state = RC_PEERS_DESTROYED;
-    GNUNET_free (rc->peers);
+    rc->state = RC_PEERS_SHUTDOWN;
+    GNUNET_free_non_null (rc->peers);
     rc->peers = NULL;
-    LOG (GNUNET_ERROR_TYPE_DEBUG, "All peers successfully destroyed\n");
+    DEBUG ("Peers shut down in %s\n", prof_time (rc));
     GNUNET_SCHEDULER_add_now (&cleanup_task, rc);
     break;
   default:
@@ -668,30 +1007,26 @@ call_cc:
     rc->cc (rc->cc_cls, event);
   if (GNUNET_TESTBED_ET_PEER_START != event->type)
     return;
-  for (dll_op = rc->dll_op_head; NULL != dll_op; dll_op = dll_op->next)
-    if ((NULL != dll_op->cls) &&
-        (event->details.peer_start.peer == dll_op->cls))
-      break;
-  if (NULL == dll_op)           /* Not our operation */
+  if (NULL == (rcop = search_rcop (rc, event->op))) /* Not our operation */
     return;
-  GNUNET_CONTAINER_DLL_remove (rc->dll_op_head, rc->dll_op_tail, dll_op);
-  GNUNET_TESTBED_operation_done (dll_op->op);
-  GNUNET_free (dll_op);
+  remove_rcop (rc, rcop);
+  GNUNET_TESTBED_operation_done (rcop->op);
+  GNUNET_free (rcop);
   rc->peer_count++;
   if (rc->peer_count < rc->num_peers)
     return;
-  LOG (GNUNET_ERROR_TYPE_DEBUG, "Peers started successfully\n");
+  DEBUG ("%u peers started in %s\n", rc->num_peers, prof_time (rc));
   if (GNUNET_TESTBED_TOPOLOGY_NONE != rc->topology)
   {
-    if ( (GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI == rc->topology)
-         || (GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING == rc->topology)
-         || (GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD == rc->topology))
+    if ((GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI == rc->topology) ||
+        (GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING == rc->topology) ||
+        (GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD == rc->topology))
     {
       rc->topology_operation =
-          GNUNET_TESTBED_overlay_configure_topology (NULL,
-                                                     rc->num_peers,
-                                                     rc->peers,
-                                                     &rc->num_oc,
+          GNUNET_TESTBED_overlay_configure_topology (NULL, rc->num_peers,
+                                                     rc->peers, &rc->num_oc,
+                                                     &topology_completion_callback,
+                                                     rc,
                                                      rc->topology,
                                                      rc->random_links,
                                                      GNUNET_TESTBED_TOPOLOGY_OPTION_END);
@@ -700,31 +1035,34 @@ call_cc:
     {
       GNUNET_assert (NULL != rc->topo_file);
       rc->topology_operation =
-          GNUNET_TESTBED_overlay_configure_topology (NULL,
-                                                     rc->num_peers,
-                                                     rc->peers,
-                                                     &rc->num_oc,
+          GNUNET_TESTBED_overlay_configure_topology (NULL, rc->num_peers,
+                                                     rc->peers, &rc->num_oc,
+                                                     &topology_completion_callback,
+                                                     rc,
                                                      rc->topology,
                                                      rc->topo_file,
                                                      GNUNET_TESTBED_TOPOLOGY_OPTION_END);
     }
     else
       rc->topology_operation =
-          GNUNET_TESTBED_overlay_configure_topology (NULL,
-                                                     rc->num_peers,
-                                                     rc->peers,
-                                                     &rc->num_oc,
+          GNUNET_TESTBED_overlay_configure_topology (NULL, rc->num_peers,
+                                                     rc->peers, &rc->num_oc,
+                                                     &topology_completion_callback,
+                                                     rc,
                                                      rc->topology,
                                                      GNUNET_TESTBED_TOPOLOGY_OPTION_END);
     if (NULL == rc->topology_operation)
       LOG (GNUNET_ERROR_TYPE_WARNING,
            "Not generating topology. Check number of peers\n");
     else
+    {
+      DEBUG ("Creating overlay topology\n");
+      rc->pstart_time = GNUNET_TIME_absolute_get ();
       return;
+    }
   }
   rc->state = RC_READY;
-  GNUNET_SCHEDULER_add_continuation (&call_master, rc,
-                                     GNUNET_SCHEDULER_REASON_PREREQ_DONE);
+  call_master (rc);
 }
 
 
@@ -771,34 +1109,47 @@ static void
 register_hosts (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct RunContext *rc = cls;
-  struct DLLOperation *dll_op;
-  unsigned int slave;
+  struct RunContextOperation *rcop;
+  unsigned int cnt;
 
   rc->register_hosts_task = GNUNET_SCHEDULER_NO_TASK;
   if (rc->reg_hosts == rc->num_hosts)
   {
-    LOG (GNUNET_ERROR_TYPE_DEBUG,
-         "All hosts successfully registered\n");
-    /* Start slaves */
-    for (slave = 0; slave < rc->num_hosts; slave++)
+    DEBUG ("All hosts successfully registered\n");
+    /* Start cnts */
+#if !ENABLE_LL
+    for (cnt = 0; cnt < rc->num_hosts; cnt++)
     {
-      dll_op = GNUNET_malloc (sizeof (struct DLLOperation));
-      dll_op->rc = rc;
-      dll_op->op = GNUNET_TESTBED_controller_link (dll_op,
-                                                   rc->c,
-                                                   rc->hosts[slave],
-                                                   rc->h,
-                                                   rc->cfg,
-                                                   GNUNET_YES);
-      GNUNET_CONTAINER_DLL_insert_tail (rc->dll_op_head, rc->dll_op_tail, dll_op);
+      rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+      rcop->rc = rc;
+      rcop->op =
+          GNUNET_TESTBED_controller_link (rcop, rc->c, rc->hosts[cnt],
+                                          rc->h, GNUNET_YES);
+      GNUNET_assert (NULL != rcop->op);
+      insert_rcop (rc, rcop);
     }
+#else
+    struct Island *island;
+    for (cnt = 0; cnt < rc->nislands; cnt++)
+    {
+      island = rc->islands[cnt];
+      GNUNET_assert (0 < island->nhosts);
+      rcop = GNUNET_malloc (sizeof (struct RunContextOperation));
+      rcop->rc = rc;
+      rcop->op =
+          GNUNET_TESTBED_controller_link (rcop, rc->c, island->hosts[0],
+                                          rc->h, GNUNET_YES);
+      GNUNET_assert (NULL != rcop->op);
+      insert_rcop (rc, rcop);
+    }
+#endif
     rc->reg_hosts = 0;
     return;
   }
-  rc->reg_handle = GNUNET_TESTBED_register_host (rc->c,
-                                                 rc->hosts[rc->reg_hosts++],
-                                                 host_registration_completion,
-                                                 rc);
+  rc->reg_handle =
+      GNUNET_TESTBED_register_host (rc->c, rc->hosts[rc->reg_hosts],
+                                    host_registration_completion, rc);
+  rc->reg_hosts++;
 }
 
 
@@ -820,29 +1171,36 @@ controller_status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
 
   if (status != GNUNET_OK)
   {
-    switch (rc->state)
+    GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                _("Controller crash detected. Shutting down.\n"));
+    rc->cproc = NULL;
+    cleanup (rc);
+    if (NULL != rc->peers)
     {
-    case RC_INIT:
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Testbed startup failed\n");
-      return;
-    default:
-      rc->cproc = NULL;
-      shutdown_now (rc);      
-      return;
+      GNUNET_free (rc->peers);
+      rc->peers = NULL;
     }
+    if (GNUNET_YES == rc->shutdown)
+    {
+      rc->state = RC_PEERS_SHUTDOWN;
+      GNUNET_SCHEDULER_add_now (&cleanup_task, rc);
+    }
+    else
+      shutdown_now (rc);
+    return;
   }
   GNUNET_CONFIGURATION_destroy (rc->cfg);
   rc->cfg = GNUNET_CONFIGURATION_dup (cfg);
   event_mask = rc->event_mask;
-  event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP);
   event_mask |= (1LL << GNUNET_TESTBED_ET_OPERATION_FINISHED);
   event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_START);
   if (rc->topology < GNUNET_TESTBED_TOPOLOGY_NONE)
     event_mask |= GNUNET_TESTBED_ET_CONNECT;
   rc->c =
-      GNUNET_TESTBED_controller_connect (rc->cfg, rc->h, event_mask, &event_cb, rc);
+      GNUNET_TESTBED_controller_connect (rc->h, event_mask, &event_cb, rc);
   if (0 < rc->num_hosts)
   {
+    rc->reg_hosts = 0;
     rc->register_hosts_task = GNUNET_SCHEDULER_add_now (&register_hosts, rc);
     return;
   }
@@ -851,6 +1209,49 @@ controller_status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
 }
 
 
+#if ENABLE_LL
+#define ISLANDNAME_SIZE 4
+static void
+parse_islands (struct RunContext *rc)
+{
+  char island_id[ISLANDNAME_SIZE];
+  struct GNUNET_TESTBED_Host *host;
+  struct Island *island;
+  const char *hostname;
+  unsigned int nhost;
+
+  DEBUG ("Parsing for islands\n");
+  memset (island_id, 0, ISLANDNAME_SIZE);
+  island = NULL;
+  for (nhost = 0; nhost < rc->num_hosts; nhost++)
+  {
+    host = rc->hosts[nhost];
+    hostname = GNUNET_TESTBED_host_get_hostname (host);
+    GNUNET_assert (NULL != hostname);
+    if (NULL == island)
+    {
+      strncpy (island_id, hostname, ISLANDNAME_SIZE - 1);
+      island = GNUNET_malloc (sizeof (struct Island));
+    }
+    if (0 == strncmp (island_id, hostname, ISLANDNAME_SIZE - 1))
+    {      
+      GNUNET_array_append (island->hosts, island->nhosts, host);
+      continue;
+    }
+    DEBUG ("Adding island `%s' with %u hosts\n", island_id, island->nhosts);
+    GNUNET_array_append (rc->islands, rc->nislands, island);
+    island = NULL;
+  }
+  if (NULL != island)
+  {
+    DEBUG ("Adding island `%s' with %u hosts\n", island_id, island->nhosts);
+    GNUNET_array_append (rc->islands, rc->nislands, island);
+  }
+  DEBUG ("Total islands parsed: %u\n", rc->nislands);
+}
+#endif
+
+
 /**
  * Callback function invoked for each interface found.
  *
@@ -864,32 +1265,28 @@ controller_status_cb (void *cls, const struct GNUNET_CONFIGURATION_Handle *cfg,
  * @return GNUNET_OK to continue iteration, GNUNET_SYSERR to abort
  */
 static int
-netint_proc (void *cls, const char *name,
-             int isDefault,
-             const struct sockaddr *addr,
-             const struct sockaddr *broadcast_addr,
-             const struct sockaddr *netmask,
-             socklen_t addrlen)
+netint_proc (void *cls, const char *name, int isDefault,
+             const struct sockaddr *addr, const struct sockaddr *broadcast_addr,
+             const struct sockaddr *netmask, socklen_t addrlen)
 {
-   struct RunContext *rc = cls;
-   char hostip[NI_MAXHOST];
-   char *buf;
-
-   if (sizeof (struct sockaddr_in) != addrlen)
-     return GNUNET_OK;          /* Only consider IPv4 for now */
-   if (0 != getnameinfo(addr, addrlen,
-                        hostip, NI_MAXHOST,
-                        NULL, 0, NI_NUMERICHOST))
-     GNUNET_log_strerror(GNUNET_ERROR_TYPE_WARNING, "getnameinfo");
-   if (NULL == rc->trusted_ip)
-   {
-     rc->trusted_ip = GNUNET_strdup (hostip);
-     return GNUNET_YES;
-   }
-   (void) GNUNET_asprintf (&buf, "%s; %s", rc->trusted_ip, hostip);
-   GNUNET_free (rc->trusted_ip);
-   rc->trusted_ip = buf;
-   return GNUNET_YES;
+  struct RunContext *rc = cls;
+  char hostip[NI_MAXHOST];
+  char *buf;
+
+  if (sizeof (struct sockaddr_in) != addrlen)
+    return GNUNET_OK;           /* Only consider IPv4 for now */
+  if (0 !=
+      getnameinfo (addr, addrlen, hostip, NI_MAXHOST, NULL, 0, NI_NUMERICHOST))
+    GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "getnameinfo");
+  if (NULL == rc->trusted_ip)
+  {
+    rc->trusted_ip = GNUNET_strdup (hostip);
+    return GNUNET_YES;
+  }
+  (void) GNUNET_asprintf (&buf, "%s; %s", rc->trusted_ip, hostip);
+  GNUNET_free (rc->trusted_ip);
+  rc->trusted_ip = buf;
+  return GNUNET_YES;
 }
 
 
@@ -903,48 +1300,58 @@ netint_proc (void *cls, const char *name,
  *          given to GNUNET_TESTBED_is_host_habitable() is NULL
  * @param status GNUNET_YES if it is habitable; GNUNET_NO if not
  */
-static void 
-host_habitable_cb (void *cls, const struct GNUNET_TESTBED_Host *host, int status)
+static void
+host_habitable_cb (void *cls, const struct GNUNET_TESTBED_Host *host,
+                   int status)
 {
-  struct RunContext *rc = cls;
+  struct CompatibilityCheckContext *hc = cls;
+  struct RunContext *rc;
+  struct GNUNET_TESTBED_Host **old_hosts;
   unsigned int nhost;
-  
-  for (nhost = 0; nhost < rc->num_hosts; nhost++)
-  {
-    if (host == rc->hosts[nhost])
-      break;
-  }
-  GNUNET_assert (nhost != rc->num_hosts);
-  rc->hc_handles[nhost] = NULL;
+
+  GNUNET_assert (NULL != (rc = hc->rc));
+  nhost = hc->index;
+  GNUNET_assert (nhost <= rc->num_hosts);
+  GNUNET_assert (host == rc->hosts[nhost]);
+  hc->h = NULL;
   if (GNUNET_NO == status)
   {
     if ((NULL != host) && (NULL != GNUNET_TESTBED_host_get_hostname (host)))
       LOG (GNUNET_ERROR_TYPE_ERROR, _("Host %s cannot start testbed\n"),
            GNUNET_TESTBED_host_get_hostname (host));
     else
-      LOG (GNUNET_ERROR_TYPE_ERROR, _("Testbed cannot be started on localhost\n"));
+      LOG (GNUNET_ERROR_TYPE_ERROR,
+           _("Testbed cannot be started on localhost\n"));
     shutdown_now (rc);
     return;
   }
   rc->reg_hosts++;
   if (rc->reg_hosts < rc->num_hosts)
     return;
-  GNUNET_free (rc->hc_handles);
-  rc->hc_handles = NULL;
+  GNUNET_free (rc->hclist);
+  rc->hclist = NULL;
   rc->h = rc->hosts[0];
   rc->num_hosts--;
   if (0 < rc->num_hosts)
-    rc->hosts = &rc->hosts[1];
+  {
+    old_hosts = rc->hosts;
+    rc->hosts =
+        GNUNET_malloc (sizeof (struct GNUNET_TESTBED_Host *) * rc->num_hosts);
+    memcpy (rc->hosts, &old_hosts[1],
+            (sizeof (struct GNUNET_TESTBED_Host *) * rc->num_hosts));
+    GNUNET_free (old_hosts);
+  }
   else
   {
     GNUNET_free (rc->hosts);
     rc->hosts = NULL;
   }
+  parse_islands (rc);
   GNUNET_OS_network_interfaces_list (netint_proc, rc);
   if (NULL == rc->trusted_ip)
     rc->trusted_ip = GNUNET_strdup ("127.0.0.1");
   rc->cproc =
-      GNUNET_TESTBED_controller_start (rc->trusted_ip, rc->h, rc->cfg,
+      GNUNET_TESTBED_controller_start (rc->trusted_ip, rc->h,
                                        &controller_status_cb, rc);
   GNUNET_free (rc->trusted_ip);
   rc->trusted_ip = NULL;
@@ -956,6 +1363,26 @@ host_habitable_cb (void *cls, const struct GNUNET_TESTBED_Host *host, int status
 }
 
 
+/**
+ * Task run upon timeout while setting up the testbed
+ *
+ * @param cls the RunContext
+ * @param tc the task context
+ */
+static void
+timeout_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct RunContext *rc = cls;
+  
+  rc->timeout_task = GNUNET_SCHEDULER_NO_TASK;
+  LOG (GNUNET_ERROR_TYPE_ERROR, _("Shutting down testbed due to timeout while setup.\n"));
+  shutdown_now (rc);
+   if (NULL != rc->test_master)
+     rc->test_master (rc->test_master_cls, 0, NULL, 0, 0);
+   rc->test_master = NULL;
+}
+
+
 /**
  * Convenience method for running a testbed with
  * a single call.  Underlay and overlay topology
@@ -993,16 +1420,30 @@ GNUNET_TESTBED_run (const char *host_filename,
 {
   struct RunContext *rc;
   char *topology;
+  struct CompatibilityCheckContext *hc;      
+  struct GNUNET_TIME_Relative timeout;
   unsigned long long random_links;
   unsigned int hid;
   unsigned int nhost;
-  
+
   GNUNET_assert (num_peers > 0);
   rc = GNUNET_malloc (sizeof (struct RunContext));
+  rc->cfg = GNUNET_CONFIGURATION_dup (cfg);
+#if ENABLE_LL
+  rc->num_hosts = GNUNET_TESTBED_hosts_load_from_loadleveler (rc->cfg,
+                                                              &rc->hosts);
+  if (0 == rc->num_hosts)
+  {
+    LOG (GNUNET_ERROR_TYPE_WARNING,
+           _("No hosts loaded from LoadLeveler. Need at least one host\n"));
+    goto error_cleanup;
+  }
+#else
   if (NULL != host_filename)
   {
     rc->num_hosts =
-        GNUNET_TESTBED_hosts_load_from_file (host_filename, &rc->hosts);
+        GNUNET_TESTBED_hosts_load_from_file (host_filename, rc->cfg,
+                                             &rc->hosts);
     if (0 == rc->num_hosts)
     {
       LOG (GNUNET_ERROR_TYPE_WARNING,
@@ -1011,8 +1452,8 @@ GNUNET_TESTBED_run (const char *host_filename,
     }
   }
   else
-    rc->h = GNUNET_TESTBED_host_create (NULL, NULL, 0);
-  rc->cfg = GNUNET_CONFIGURATION_dup (cfg);
+    rc->h = GNUNET_TESTBED_host_create (NULL, NULL, rc->cfg, 0);
+#endif
   rc->num_peers = num_peers;
   rc->event_mask = event_mask;
   rc->cc = cc;
@@ -1020,17 +1461,17 @@ GNUNET_TESTBED_run (const char *host_filename,
   rc->test_master = test_master;
   rc->test_master_cls = test_master_cls;
   rc->state = RC_INIT;
-  rc->topology = GNUNET_TESTBED_TOPOLOGY_NONE;  
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (rc->cfg, "testbed",
-                                                          "OVERLAY_TOPOLOGY",
-                                                          &topology))
+  rc->topology = GNUNET_TESTBED_TOPOLOGY_NONE;
+  if (GNUNET_OK ==
+      GNUNET_CONFIGURATION_get_value_string (rc->cfg, "testbed",
+                                             "OVERLAY_TOPOLOGY", &topology))
   {
-    if (GNUNET_NO == GNUNET_TESTBED_topology_get_ (&rc->topology,
-                                                    topology))
+    if (GNUNET_NO == GNUNET_TESTBED_topology_get_ (&rc->topology, topology))
     {
-      GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR,
-                                "testbed", "OVERLAY_TOPLOGY",
-                                _("Specified topology must be supported by testbed"));
+      GNUNET_log_config_invalid (GNUNET_ERROR_TYPE_ERROR, "testbed",
+                                 "OVERLAY_TOPLOGY",
+                                 _
+                                 ("Specified topology must be supported by testbed"));
     }
     GNUNET_free (topology);
   }
@@ -1039,69 +1480,89 @@ GNUNET_TESTBED_run (const char *host_filename,
   case GNUNET_TESTBED_TOPOLOGY_ERDOS_RENYI:
   case GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD_RING:
   case GNUNET_TESTBED_TOPOLOGY_SMALL_WORLD:
-    if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (rc->cfg, "testbed",
-                                                            "OVERLAY_RANDOM_LINKS",
-                                                            &random_links))
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_number (rc->cfg, "testbed",
+                                               "OVERLAY_RANDOM_LINKS",
+                                               &random_links))
     {
       /* OVERLAY option RANDOM & SMALL_WORLD_RING requires OVERLAY_RANDOM_LINKS
-         option to be set to the number of random links to be established  */
-      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
-                                "testbed", "OVERLAY_RANDOM_LINKS");
+       * option to be set to the number of random links to be established  */
+      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "testbed",
+                                 "OVERLAY_RANDOM_LINKS");
       goto error_cleanup;
     }
     if (random_links > UINT32_MAX)
     {
-      GNUNET_break (0);       /* Too big number */
+      GNUNET_break (0);         /* Too big number */
       goto error_cleanup;
     }
     rc->random_links = (unsigned int) random_links;
     break;
   case GNUNET_TESTBED_TOPOLOGY_FROM_FILE:
-    if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (rc->cfg, "testbed",
-                                                            "OVERLAY_TOPOLOGY_FILE",
-                                                            &rc->topo_file))
+    if (GNUNET_OK !=
+        GNUNET_CONFIGURATION_get_value_string (rc->cfg, "testbed",
+                                               "OVERLAY_TOPOLOGY_FILE",
+                                               &rc->topo_file))
     {
-      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR,
-                                "testbed", "OVERLAY_TOPOLOGY_FILE");
+      GNUNET_log_config_missing (GNUNET_ERROR_TYPE_ERROR, "testbed",
+                                 "OVERLAY_TOPOLOGY_FILE");
       goto error_cleanup;
     }
-    break;
-  default:   
-    /* Do nothing */
+  default:
+    /* Warn if OVERLAY_RANDOM_LINKS is present that it will be ignored */
+    if (GNUNET_YES ==
+        GNUNET_CONFIGURATION_have_value (rc->cfg, "testbed",
+                                         "OVERLAY_RANDOM_LINKS"))
+      LOG (GNUNET_ERROR_TYPE_WARNING,
+           "Ignoring value of `OVERLAY_RANDOM_LINKS' in given configuration\n");
     break;
   }
-  if (NULL != host_filename)
+  if (0 != rc->num_hosts)
   {
-    rc->hc_handles = GNUNET_malloc (sizeof (struct
-                                            GNUNET_TESTBED_HostHabitableCheckHandle *) 
-                                    * rc->num_hosts);
-    for (nhost = 0; nhost < rc->num_hosts; nhost++) 
-    {    
-      if (NULL == (rc->hc_handles[nhost] = 
-                   GNUNET_TESTBED_is_host_habitable (rc->hosts[nhost], rc->cfg,
-                                                     &host_habitable_cb,
-                                                     rc)))
+    rc->hclist = GNUNET_malloc (sizeof (struct CompatibilityCheckContext)
+                                * rc->num_hosts);
+    for (nhost = 0; nhost < rc->num_hosts; nhost++)
+    {
+      hc = &rc->hclist[nhost];
+      hc->index = nhost;
+      hc->rc = rc;
+      hc->h = GNUNET_TESTBED_is_host_habitable (rc->hosts[nhost], rc->cfg,
+                                                &host_habitable_cb, hc);
+      if (NULL == hc->h)
       {
         GNUNET_break (0);
         for (nhost = 0; nhost < rc->num_hosts; nhost++)
-          if (NULL != rc->hc_handles[nhost])
-            GNUNET_TESTBED_is_host_habitable_cancel (rc->hc_handles[nhost]);
-        GNUNET_free (rc->hc_handles);
-        rc->hc_handles = NULL;
+        {
+          hc = &rc->hclist[nhost];
+          if (NULL != hc->h)
+            GNUNET_TESTBED_is_host_habitable_cancel (hc->h);
+        }
+        GNUNET_free (rc->hclist);
+        rc->hclist = NULL;
         goto error_cleanup;
       }
     }
   }
   else
     rc->cproc =
-      GNUNET_TESTBED_controller_start ("127.0.0.1", rc->h, rc->cfg,
-                                       &controller_status_cb, rc);
-  rc->shutdown_run_task =
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
-                                    &shutdown_run, rc);
+        GNUNET_TESTBED_controller_start ("127.0.0.1", rc->h,
+                                         &controller_status_cb, rc);
+  if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED",
+                                                        "SETUP_TIMEOUT",
+                                                        &timeout))
+  {
+    timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
+                                             DEFAULT_SETUP_TIMEOUT);
+  }
+  rc->rcop_map = GNUNET_CONTAINER_multihashmap32_create (256);
+  rc->timeout_task =
+      GNUNET_SCHEDULER_add_delayed (timeout, &timeout_task, rc);
+  rc->interrupt_task =
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &interrupt,
+                                    rc);
   return;
 
- error_cleanup:
+error_cleanup:
   if (NULL != rc->h)
     GNUNET_TESTBED_host_destroy (rc->h);
   if (NULL != rc->hosts)
@@ -1111,6 +1572,8 @@ GNUNET_TESTBED_run (const char *host_filename,
         GNUNET_TESTBED_host_destroy (rc->hosts[hid]);
     GNUNET_free (rc->hosts);
   }
+  if (NULL != rc->cfg)
+    GNUNET_CONFIGURATION_destroy (rc->cfg);
   GNUNET_free (rc);
 }