fix
[oweals/gnunet.git] / src / testing / testing_group.c
index 78100e446a0d2f4c6cc6251bb745a204c847d494..a09460844476a3a4402250982459b6fdd1a2a5b6 100644 (file)
 #include "platform.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
+#include "gnunet_core_service.h"
 
 #define VERBOSE_TESTING GNUNET_NO
 
+#define VERBOSE_TOPOLOGY GNUNET_YES
+
 #define DEBUG_CHURN GNUNET_NO
 
 /**
  */
 #define HIGH_PORT 56000
 
-#define MAX_OUTSTANDING_CONNECTIONS 50
+#define MAX_OUTSTANDING_CONNECTIONS 40
+
+#define MAX_CONCURRENT_HOSTKEYS 10
 
-#define MAX_CONCURRENT_HOSTKEYS 16
+#define MAX_CONCURRENT_STARTING 10
 
-#define MAX_CONCURRENT_STARTING 50
+#define MAX_CONCURRENT_SHUTDOWN 10
 
-#define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
+#define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 200)
 
-#define CONNECT_ATTEMPTS 8
+#define CONNECT_ATTEMPTS 21
 
 /**
  * Prototype of a function called whenever two peers would be connected
  * in a certain topology.
  */
-typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg, 
-                                                 unsigned int first,
-                                                 unsigned int second);
+typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
+                                                            GNUNET_TESTING_PeerGroup
+                                                            * pg,
+                                                            unsigned int
+                                                            first,
+                                                            unsigned int
+                                                            second);
 
 
 /**
@@ -91,7 +100,7 @@ struct ChurnContext
    * Number of peers that still need to be stopped
    */
   unsigned int num_to_stop;
+
   /**
    * Number of peers that failed to start
    */
@@ -138,17 +147,28 @@ struct ShutdownContext
   /**
    * Total peers to wait for
    */
-  int total_peers;
+  unsigned int total_peers;
 
   /**
    * Number of peers successfully shut down
    */
-  int peers_down;
+  unsigned int peers_down;
 
   /**
    * Number of peers failed to shut down
    */
-  int peers_failed;
+  unsigned int peers_failed;
+
+  /**
+   * Number of peers we have started shutting
+   * down.  If too many, wait on them.
+   */
+  unsigned int outstanding;
+
+  /**
+   * Timeout for shutdown.
+   */
+  struct GNUNET_TIME_Relative timeout;
 
   /**
    * Callback to call when all peers either
@@ -162,6 +182,39 @@ struct ShutdownContext
   void *cb_cls;
 };
 
+/**
+ * Individual shutdown context for a particular peer.
+ */
+struct PeerShutdownContext
+{
+  /**
+   * Pointer to the high level shutdown context.
+   */
+  struct ShutdownContext *shutdown_ctx;
+
+  /**
+   * The daemon handle for the peer to shut down.
+   */
+  struct GNUNET_TESTING_Daemon *daemon;
+};
+
+/**
+ * Individual shutdown context for a particular peer.
+ */
+struct PeerRestartContext
+{
+  /**
+   * Pointer to the high level restart context.
+   */
+  struct ChurnRestartContext *churn_restart_ctx;
+
+  /**
+   * The daemon handle for the peer to shut down.
+   */
+  struct GNUNET_TESTING_Daemon *daemon;
+};
+
+
 struct CreateTopologyContext
 {
 
@@ -228,6 +281,36 @@ struct InternalStartContext
    * Hostname, where to start the peer
    */
   const char *hostname;
+
+  /**
+   * Username to use when connecting to the
+   * host via ssh.
+   */
+  const char *username;
+
+  /**
+   * Port to use for ssh.
+   */
+  uint16_t sshport;
+
+};
+
+struct ChurnRestartContext
+{
+  /**
+   * Number of restarts currently in flight.
+   */
+  unsigned int outstanding;
+
+  /**
+   * Handle to the underlying churn context.
+   */
+  struct ChurnContext *churn_ctx;
+
+  /**
+   * How long to allow the operation to take.
+   */
+  struct GNUNET_TIME_Relative timeout;
 };
 
 /**
@@ -288,7 +371,7 @@ struct PeerData
 
 
 /**
- * Data we keep per host.
+ * Linked list of per-host data.
  */
 struct HostData
 {
@@ -297,6 +380,16 @@ struct HostData
    */
   char *hostname;
 
+  /**
+   * SSH username to use when connecting to this host.
+   */
+  char *username;
+
+  /**
+   * SSH port to use when connecting to this host.
+   */
+  uint16_t sshport;
+
   /**
    * Lowest port that we have not yet used
    * for GNUnet.
@@ -304,17 +397,93 @@ struct HostData
   uint16_t minport;
 };
 
+struct TopologyIterateContext
+{
+  /**
+   * Callback for notifying of two connected peers.
+   */
+  GNUNET_TESTING_NotifyTopology topology_cb;
+
+  /**
+   * Closure for topology_cb
+   */
+  void *cls;
+
+  /**
+   * Number of peers currently connected to.
+   */
+  unsigned int connected;
+
+  /**
+   * Number of peers we have finished iterating.
+   */
+  unsigned int completed;
+
+  /**
+   * Number of peers total.
+   */
+  unsigned int total;
+};
+
+struct StatsIterateContext
+{
+  /**
+   * Continuation to call once all stats information has been retrieved.
+   */
+  GNUNET_STATISTICS_Callback cont;
+
+  /**
+   * Proc function to call on each value received.
+   */
+  GNUNET_TESTING_STATISTICS_Iterator proc;
+
+  /**
+   * Closure for topology_cb
+   */
+  void *cls;
+
+  /**
+   * Number of peers currently connected to.
+   */
+  unsigned int connected;
+
+  /**
+   * Number of peers we have finished iterating.
+   */
+  unsigned int completed;
+
+  /**
+   * Number of peers total.
+   */
+  unsigned int total;
+};
+
+struct CoreContext
+{
+  void *iter_context;
+  struct GNUNET_TESTING_Daemon *daemon;
+};
+
+struct StatsCoreContext
+{
+  void *iter_context;
+  struct GNUNET_TESTING_Daemon *daemon;
+  /**
+   * Handle to the statistics service.
+   */
+  struct GNUNET_STATISTICS_Handle *stats_handle;
+
+  /**
+   * Handle for getting statistics.
+   */
+  struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
+};
 
 /**
  * Handle to a group of GNUnet peers.
  */
 struct GNUNET_TESTING_PeerGroup
 {
-  /**
-   * Our scheduler.
-   */
-  struct GNUNET_SCHEDULER_Handle *sched;
-
   /**
    * Configuration template.
    */
@@ -341,11 +510,15 @@ struct GNUNET_TESTING_PeerGroup
   void *notify_connection_cls;
 
   /**
-   * NULL-terminated array of information about
-   * hosts.
+   * Array of information about hosts.
    */
   struct HostData *hosts;
 
+  /**
+   * Number of hosts (size of HostData)
+   */
+  unsigned int num_hosts;
+
   /**
    * Array of "total" peers.
    */
@@ -375,19 +548,57 @@ struct GNUNET_TESTING_PeerGroup
 struct UpdateContext
 {
   struct GNUNET_CONFIGURATION_Handle *ret;
+  const struct GNUNET_CONFIGURATION_Handle *orig;
   const char *hostname;
   unsigned int nport;
   unsigned int upnum;
+  unsigned int fdnum;
 };
 
+struct ConnectTopologyContext
+{
+  /**
+   * How many connections are left to create.
+   */
+  unsigned int remaining_connections;
+
+  /**
+   * Handle to group of peers.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
+  /**
+   * Temp value set for each iteration.
+   */
+  struct PeerData *first;
+
+  /**
+   * Notification that all peers are connected.
+   */
+  GNUNET_TESTING_NotifyCompletion notify_connections_done;
+
+  /**
+   * Closure for notify.
+   */
+  void *notify_cls;
+};
 
 struct ConnectContext
 {
+  /**
+   * Peer to connect second to.
+   */
   struct GNUNET_TESTING_Daemon *first;
 
+  /**
+   * Peer to connect first to.
+   */
   struct GNUNET_TESTING_Daemon *second;
 
-  struct GNUNET_TESTING_PeerGroup *pg;
+  /**
+   * Higher level topology connection context.
+   */
+  struct ConnectTopologyContext *ct_ctx;
 };
 
 /**
@@ -397,11 +608,10 @@ struct ConnectContext
  * @param hash set to uid (extended with zeros)
  */
 static void
-hash_from_uid (uint32_t uid,
-               GNUNET_HashCode *hash)
+hash_from_uid (uint32_t uid, GNUNET_HashCode * hash)
 {
-  memset (hash, 0, sizeof(GNUNET_HashCode));
-  *((uint32_t*)hash) = uid;
+  memset (hash, 0, sizeof (GNUNET_HashCode));
+  *((uint32_t *) hash) = uid;
 }
 
 /**
@@ -411,9 +621,9 @@ hash_from_uid (uint32_t uid,
  * @param hash set to uid (extended with zeros)
  */
 static void
-uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
+uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 {
-  memcpy (uid, hash, sizeof(uint32_t));
+  memcpy (uid, hash, sizeof (uint32_t));
 }
 
 /**
@@ -432,77 +642,77 @@ static int outstanding_connects;
  *         known topology, GNUNET_NO if not
  */
 int
-GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
+GNUNET_TESTING_topology_get (enum GNUNET_TESTING_Topology *topology,
+                             const char *topology_string)
 {
   /**
    * Strings representing topologies in enum
    */
-  static const char * topology_strings[] =
-    {
+  static const char *topology_strings[] = {
       /**
        * A clique (everyone connected to everyone else).
        */
-      "CLIQUE",
-      
+    "CLIQUE",
+
       /**
        * Small-world network (2d torus plus random links).
        */
-      "SMALL_WORLD",
-      
+    "SMALL_WORLD",
+
       /**
        * Small-world network (ring plus random links).
        */
-      "SMALL_WORLD_RING",
-      
+    "SMALL_WORLD_RING",
+
       /**
        * Ring topology.
        */
-      "RING",
-      
+    "RING",
+
       /**
        * 2-d torus.
        */
-      "2D_TORUS",
-      
+    "2D_TORUS",
+
       /**
        * Random graph.
        */
-      "ERDOS_RENYI",
-      
+    "ERDOS_RENYI",
+
       /**
        * Certain percentage of peers are unable to communicate directly
        * replicating NAT conditions
        */
-      "INTERNAT",
-      
+    "INTERNAT",
+
       /**
        * Scale free topology.
        */
-      "SCALE_FREE",
-      
+    "SCALE_FREE",
+
       /**
        * Straight line topology.
        */
-      "LINE",
-      
+    "LINE",
+
       /**
        * All peers are disconnected.
        */
-      "NONE",
+    "NONE",
 
-      NULL
-    };
+    NULL
+  };
 
   int curr = 0;
   if (topology_string == NULL)
     return GNUNET_NO;
   while (topology_strings[curr] != NULL)
     {
-      if (strcasecmp(topology_strings[curr], topology_string) == 0)
-       {
-         *topology = curr;
-         return GNUNET_YES;
-       }
+      if (strcasecmp (topology_strings[curr], topology_string) == 0)
+        {
+          *topology = curr;
+          return GNUNET_YES;
+        }
       curr++;
     }
   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
@@ -520,57 +730,64 @@ GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topol
  *         topology option, GNUNET_NO if not
  */
 int
-GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
-                                   char * topology_string)
+GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption
+                                    *topology_option,
+                                    const char *topology_string)
 {
   /**
    * Options for connecting a topology as strings.
    */
-  static const char * topology_option_strings[] =
-    {
+  static const char *topology_option_strings[] = {
       /**
        * Try to connect all peers specified in the topology.
        */
-      "CONNECT_ALL",
-      
+    "CONNECT_ALL",
+
       /**
        * Choose a random subset of connections to create.
        */
-      "CONNECT_RANDOM_SUBSET",
-      
+    "CONNECT_RANDOM_SUBSET",
+
       /**
        * Create at least X connections for each peer.
        */
-      "CONNECT_MINIMUM",
-      
+    "CONNECT_MINIMUM",
+
       /**
        * Using a depth first search, create one connection
        * per peer.  If any are missed (graph disconnected)
        * start over at those peers until all have at least one
        * connection.
        */
-      "CONNECT_DFS",
-      
+    "CONNECT_DFS",
+
+      /**
+       * Find the N closest peers to each allowed peer in the
+       * topology and make sure a connection to those peers
+       * exists in the connect topology.
+       */
+    "CONNECT_CLOSEST",
+
       /**
        * No options specified.
        */
-      "CONNECT_NONE",
+    "CONNECT_NONE",
 
-      NULL
-    };
+    NULL
+  };
   int curr = 0;
 
   if (topology_string == NULL)
     return GNUNET_NO;
   while (NULL != topology_option_strings[curr])
     {
-      if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
-       {
-         *topology_option = curr;
-         return GNUNET_YES;
-       }
+      if (strcasecmp (topology_option_strings[curr], topology_string) == 0)
+        {
+          *topology_option = curr;
+          return GNUNET_YES;
+        }
       curr++;
-    } 
+    }
   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
   return GNUNET_NO;
 }
@@ -593,24 +810,53 @@ update_config (void *cls,
   unsigned int ival;
   char cval[12];
   char uval[128];
+  char *single_variable;
+  char *per_host_variable;
+  unsigned long long num_per_host;
 
   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
     {
-      if (ival != 0)
-       {
-         GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
-         value = cval;
-       }
+      GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
+      if ((ival != 0)
+          && (GNUNET_YES !=
+              GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
+                                                    single_variable)))
+        {
+          GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
+          value = cval;
+        }
+
+      GNUNET_free (single_variable);
     }
 
   if (0 == strcmp (option, "UNIXPATH"))
     {
-      GNUNET_snprintf (uval, 
-                      sizeof (uval),
-                      "/tmp/test-service-%s-%u", 
-                      section,
-                      ctx->upnum++);
-      value = uval;
+      GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
+      GNUNET_asprintf (&per_host_variable, "num_%s_per_host", section);
+      if (GNUNET_YES !=
+          GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
+                                                single_variable))
+        {
+          GNUNET_snprintf (uval,
+                           sizeof (uval),
+                           "/tmp/test-service-%s-%u", section, ctx->upnum++);
+          value = uval;
+        }
+      else if ((GNUNET_YES ==
+               GNUNET_CONFIGURATION_get_value_number (ctx->orig, "testing",
+                                                      per_host_variable,
+                                                      &num_per_host)) && (num_per_host > 0))
+
+        {
+          GNUNET_snprintf (uval,
+                           sizeof (uval),
+                           "/tmp/test-service-%s-%u",
+                           section, ctx->fdnum % num_per_host);
+          value = uval;
+        }
+      GNUNET_free (single_variable);
+      GNUNET_free (per_host_variable);
+
     }
 
   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
@@ -633,14 +879,15 @@ update_config (void *cls,
  *             port numbers that were used
  * @param upnum number to make unix domain socket names unique
  * @param hostname hostname of the controlling host, to allow control connections from
+ * @param fdnum number used to offset the unix domain socket for grouped processes
+ *              (such as statistics or peerinfo, which can be shared among others)
  *
  * @return new configuration, NULL on error
  */
 static struct GNUNET_CONFIGURATION_Handle *
-make_config (const struct GNUNET_CONFIGURATION_Handle *cfg, 
-            uint16_t * port,
-            uint32_t * upnum,
-            const char *hostname)
+make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
+             uint16_t * port,
+             uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 {
   struct UpdateContext uc;
   uint16_t orig;
@@ -650,8 +897,10 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
   orig = *port;
   uc.nport = *port;
   uc.upnum = *upnum;
+  uc.fdnum = *fdnum;
   uc.ret = GNUNET_CONFIGURATION_create ();
   uc.hostname = hostname;
+  uc.orig = cfg;
 
   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
   if (uc.nport >= HIGH_PORT)
@@ -661,12 +910,25 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
       return NULL;
     }
 
-  if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
+  if (GNUNET_CONFIGURATION_get_value_string
+      (cfg, "testing", "control_host", &control_host) == GNUNET_OK)
     {
-      GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
-      GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
-      GNUNET_free_non_null(control_host);
-      GNUNET_free(allowed_hosts);
+      if (hostname != NULL)
+        GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1; %s;", control_host,
+                         hostname);
+      else
+        GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", control_host);
+
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
+                                             "ACCEPT_FROM", allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
+                                             "ACCEPT_FROM", allowed_hosts);
+      GNUNET_free_non_null (control_host);
+      GNUNET_free (allowed_hosts);
     }
 
 
@@ -674,13 +936,27 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
    * otherwise gnunet-arm is unable to connect to it in some instances */
   if (hostname != NULL)
     {
-      GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
-      GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
-      GNUNET_free(allowed_hosts);
+      GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
+                                             "BINDTO", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
+                                             "BINDTO", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_free (allowed_hosts);
+    }
+  else
+    {
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
+                                             "BINDTO", "127.0.0.1");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
+                                             "BINDTO", "127.0.0.1");
     }
 
   *port = (uint16_t) uc.nport;
   *upnum = uc.upnum;
+  uc.fdnum++;
+  *fdnum = uc.fdnum;
   return uc.ret;
 }
 
@@ -696,8 +972,9 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
  *         technically should only be 0 or 2
  *
  */
-static int
-add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+add_actual_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                        unsigned int first, unsigned int second)
 {
   int added;
   int add_first;
@@ -706,17 +983,21 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].connect_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].connect_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
@@ -724,14 +1005,28 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].connect_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].connect_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[second].num_connections++;
       added++;
     }
@@ -753,8 +1048,9 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  *         for being sure doesn't bother me!
  *
  */
-static int
-add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+add_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                         unsigned int first, unsigned int second)
 {
   int added;
 #if OLD
@@ -769,17 +1065,21 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].allowed_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].allowed_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
@@ -805,9 +1105,16 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].allowed_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 #if OLD
-      new_first = GNUNET_malloc(sizeof(struct PeerConnection));
+      new_first = GNUNET_malloc (sizeof (struct PeerConnection));
       new_first->daemon = pg->peers[second].daemon;
       new_first->next = pg->peers[first].connected_peers;
       pg->peers[first].connected_peers = new_first;
@@ -818,9 +1125,16 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].allowed_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 #if OLD
-      new_second = GNUNET_malloc(sizeof(struct PeerConnection));
+      new_second = GNUNET_malloc (sizeof (struct PeerConnection));
       new_second->daemon = pg->peers[first].daemon;
       new_second->next = pg->peers[second].connected_peers;
       pg->peers[second].connected_peers = new_second;
@@ -843,8 +1157,9 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  * @return the number of connections added (can be 0, 1 or 2)
  *
  */
-static int
-blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+blacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                       unsigned int first, unsigned int second)
 {
   int added;
   int add_first;
@@ -852,17 +1167,23 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->
+                                              peers[first].blacklisted_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->
+                                              peers[second].blacklisted_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
@@ -870,14 +1191,28 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].blacklisted_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].blacklisted_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[second].num_connections++;
       added++;
     }
@@ -895,8 +1230,9 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
  * @return the number of connections removed (can be 0, 1 or 2)
  *
  */
-static int
-unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+unblacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                         unsigned int first, unsigned int second)
 {
   int removed;
   int remove_first;
@@ -904,22 +1240,42 @@ unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
-  remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
-  remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
+  remove_first =
+    GNUNET_CONTAINER_multihashmap_contains (pg->
+                                            peers[first].blacklisted_peers,
+                                            &hash_second);
+  remove_second =
+    GNUNET_CONTAINER_multihashmap_contains (pg->
+                                            peers[second].blacklisted_peers,
+                                            &hash_first);
 
   removed = 0;
   if (remove_first)
     {
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->
+                                                           peers
+                                                           [first].blacklisted_peers,
+                                                           &hash_second,
+                                                           pg->
+                                                           peers
+                                                           [second].daemon));
       removed++;
     }
 
   if (remove_second)
     {
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->
+                                                           peers
+                                                           [second].blacklisted_peers,
+                                                           &hash_first,
+                                                           pg->
+                                                           peers
+                                                           [first].daemon));
       removed++;
     }
 
@@ -941,8 +1297,9 @@ unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  *
  * @return the number of connections created
  */
-static int
-create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_scale_free (struct GNUNET_TESTING_PeerGroup *pg,
+                   GNUNET_TESTING_ConnectionProcessor proc)
 {
 
   unsigned int total_connections;
@@ -952,19 +1309,23 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
   double random;
   double probability;
 
-  GNUNET_assert(pg->total > 1);
+  GNUNET_assert (pg->total > 1);
 
   /* Add a connection between the first two nodes */
-  total_connections = proc(pg, 0, 1);
+  total_connections = proc (pg, 0, 1);
 
   for (outer_count = 1; outer_count < pg->total; outer_count++)
     {
       previous_total_connections = total_connections;
       for (i = 0; i < outer_count; i++)
         {
-          probability = pg->peers[i].num_connections / (double)previous_total_connections;
-          random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
+          probability =
+            pg->peers[i].num_connections /
+            (double) previous_total_connections;
+          random =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX)) / ((double) UINT64_MAX);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Considering connecting peer %d to peer %d\n",
@@ -974,10 +1335,9 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
             {
 #if VERBOSE_TESTING
               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                          "Connecting peer %d to peer %d\n",
-                          outer_count, i);
+                          "Connecting peer %d to peer %d\n", outer_count, i);
 #endif
-              total_connections += proc(pg, outer_count, i);
+              total_connections += proc (pg, outer_count, i);
             }
         }
     }
@@ -996,8 +1356,9 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
  * @return the number of connections that were set up
  *
  */
-int
-create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_small_world_ring (struct GNUNET_TESTING_PeerGroup *pg,
+                         GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i, j;
   int nodeToConnect;
@@ -1012,32 +1373,30 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
   unsigned int useAnd;
   int connect_attempts;
 
-  logNModifier = 0.5; /* FIXME: default value? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "LOGNMODIFIER",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &logNModifier) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "LOGNMODIFIER",
-                   "TESTING");
+  logNModifier = 0.5;           /* FIXME: default value? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "LOGNMODIFIER",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &logNModifier) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "LOGNMODIFIER", "TESTING");
       GNUNET_free (p_string);
     }
-  percentage = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PERCENTAGE",
-                   "TESTING");
+  percentage = 0.5;             /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
   natLog = log (pg->total);
@@ -1068,25 +1427,26 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
 
       for (j = 0; j < connsPerPeer / 2; j++)
         {
-          random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                     UINT64_MAX) / ( (double) UINT64_MAX));
+          random =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX) / ((double) UINT64_MAX));
           if (random < percentage)
             {
               /* Connect to uniformly selected random peer */
               randomPeer =
                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                   pg->total);
+                                          pg->total);
               while ((((randomPeer < max) && (randomPeer > min))
                       && (useAnd == 0)) || (((randomPeer > min)
                                              || (randomPeer < max))
                                             && (useAnd == 1)))
                 {
                   randomPeer =
-                      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                                         pg->total);
+                    GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                              pg->total);
                 }
-              smallWorldConnections +=
-                proc (pg, i, randomPeer);
+              smallWorldConnections += proc (pg, i, randomPeer);
             }
           else
             {
@@ -1095,8 +1455,7 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
                 {
                   nodeToConnect = nodeToConnect - pg->total;
                 }
-              connect_attempts +=
-                proc (pg, i, nodeToConnect);
+              connect_attempts += proc (pg, i, nodeToConnect);
             }
         }
 
@@ -1118,8 +1477,9 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
  * @return the number of connections that were set up
  *
  */
-static int
-create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg,
+                       GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int outer_count, inner_count;
   unsigned int cutoff;
@@ -1127,18 +1487,17 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
   double nat_percentage;
   char *p_string;
 
-  nat_percentage = 0.6; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "NATPERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &nat_percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "NATPERCENTAGE",
-                   "TESTING");
+  nat_percentage = 0.6;         /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &nat_percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
 
@@ -1160,7 +1519,7 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
                           "Connecting peer %d to peer %d\n",
                           outer_count, inner_count);
 #endif
-              connect_attempts += proc(pg, outer_count, inner_count);
+              connect_attempts += proc (pg, outer_count, inner_count);
             }
         }
     }
@@ -1180,8 +1539,9 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
  * @return the number of connections that were set up
  *
  */
-static int
-create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_small_world (struct GNUNET_TESTING_PeerGroup *pg,
+                    GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i, j, k;
   unsigned int square;
@@ -1197,38 +1557,45 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
   unsigned int distance;
   double probability, random, percentage;
   unsigned int smallWorldConnections;
+  unsigned int small_world_it;
   char *p_string;
   int connect_attempts;
   square = floor (sqrt (pg->total));
   rows = square;
   cols = square;
 
-  percentage = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PERCENTAGE",
-                   "TESTING");
+  percentage = 0.5;             /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
-  probability = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PROBABILITY",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &probability) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PROBABILITY",
-                   "TESTING");
+  if (percentage < 0.0)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
+                  "PERCENTAGE", "TESTING", percentage);
+      percentage = 0.5;
+    }
+  probability = 0.5;            /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PROBABILITY",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &probability) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PROBABILITY", "TESTING");
       GNUNET_free (p_string);
     }
   if (square * square != pg->total)
@@ -1244,9 +1611,10 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
         }
     }
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
-                  rows, cols);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _
+              ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
+              rows, cols);
 #endif
 
   connect_attempts = 0;
@@ -1279,11 +1647,16 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
 #if VERBOSE_TESTING > 2
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("natural log of %d is %d, will run %d iterations\n"),
-             pg->total, natLog, (int) (natLog * percentage));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
+              pg->total, natLog, (int) (natLog * percentage));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Total connections added thus far: %u!\n"), connect_attempts);
 #endif
   smallWorldConnections = 0;
-  for (i = 0; i < (int) (natLog * percentage); i++)
+  small_world_it = (unsigned int) (natLog * percentage);
+  if (small_world_it < 1)
+    small_world_it = 1;
+  GNUNET_assert (small_world_it > 0 && small_world_it < (unsigned int) -1);
+  for (i = 0; i < small_world_it; i++)
     {
       for (j = 0; j < pg->total; j++)
         {
@@ -1296,14 +1669,18 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
               node2Row = k / cols;
               node2Col = k - (node2Row * cols);
               /* Simple Cartesian distance */
-              distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
+              distance =
+                abs (node1Row - node2Row) + abs (node1Col - node2Col);
               if (distance > 1)
                 {
                   /* Calculate probability as 1 over the square of the distance */
                   probability = 1.0 / (distance * distance);
                   /* Choose a random value between 0 and 1 */
-                 random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                             UINT64_MAX)) / ( (double) UINT64_MAX);
+                  random =
+                    ((double)
+                     GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                               UINT64_MAX)) /
+                    ((double) UINT64_MAX);
                   /* If random < probability, then connect the two nodes */
                   if (random < probability)
                     smallWorldConnections += proc (pg, j, k);
@@ -1314,9 +1691,9 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
     }
   connect_attempts += smallWorldConnections;
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Total connections added for small world: %d!\n"),
-                      smallWorldConnections);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Total connections added for small world: %d!\n"),
+              smallWorldConnections);
 #endif
   return connect_attempts;
 }
@@ -1332,8 +1709,9 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
  * @return the number of connections that were set up
  *
  */
-static int
-create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg,
+                    GNUNET_TESTING_ConnectionProcessor proc)
 {
   double temp_rand;
   unsigned int outer_count;
@@ -1342,18 +1720,17 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
   double probability;
   char *p_string;
 
-  probability = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PROBABILITY",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &probability) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PROBABILITY",
-                   "TESTING");
+  probability = 0.5;            /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PROBABILITY",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &probability) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PROBABILITY", "TESTING");
       GNUNET_free (p_string);
     }
   connect_attempts = 0;
@@ -1362,8 +1739,10 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
       for (inner_count = outer_count + 1; inner_count < pg->total;
            inner_count++)
         {
-          temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                        UINT64_MAX)) / ( (double) UINT64_MAX);
+          temp_rand =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX)) / ((double) UINT64_MAX);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("rand is %f probability is %f\n"), temp_rand,
@@ -1381,7 +1760,9 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
 
 /**
  * Create a topology given a peer group (set of running peers)
- * and a connection processor.
+ * and a connection processor.  This particular function creates
+ * the connections for a 2d-torus, plus additional "closest"
+ * connections per peer.
  *
  * @param pg the peergroup to create the topology on
  * @param proc the connection processor to call to actually set
@@ -1390,8 +1771,9 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
  * @return the number of connections that were set up
  *
  */
-static int
-create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg,
+                 GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i;
   unsigned int square;
@@ -1420,9 +1802,10 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
         }
     }
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
-                  rows, cols);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _
+              ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
+              rows, cols);
 #endif
   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
    * to the node to its right and above.  Once this is over, we'll have our torus!
@@ -1439,11 +1822,10 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
       else
         nodeToConnect = i - cols + 1;
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      i, nodeToConnect);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", i, nodeToConnect);
 #endif
-      connect_attempts += proc(pg, i, nodeToConnect);
+      connect_attempts += proc (pg, i, nodeToConnect);
 
       /* Second connect to the node immediately above */
       if (i < cols)
@@ -1455,10 +1837,9 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
         {
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      i, nodeToConnect);
+                      "Connecting peer %d to peer %d\n", i, nodeToConnect);
 #endif
-          connect_attempts += proc(pg, i, nodeToConnect);
+          connect_attempts += proc (pg, i, nodeToConnect);
         }
 
     }
@@ -1478,8 +1859,9 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
  * @return the number of connections that were set up
  *
  */
-static int
-create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_clique (struct GNUNET_TESTING_PeerGroup *pg,
+               GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int outer_count;
   unsigned int inner_count;
@@ -1497,7 +1879,7 @@ create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionPro
                       "Connecting peer %d to peer %d\n",
                       outer_count, inner_count);
 #endif
-          connect_attempts += proc(pg, outer_count, inner_count);
+          connect_attempts += proc (pg, outer_count, inner_count);
         }
     }
 
@@ -1515,8 +1897,9 @@ create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionPro
  * @return the number of connections that were set up
  *
  */
-static int
-create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_line (struct GNUNET_TESTING_PeerGroup *pg,
+             GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int count;
   int connect_attempts;
@@ -1527,11 +1910,10 @@ create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      count, count + 1);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", count, count + 1);
 #endif
-      connect_attempts += proc(pg, count, count + 1);
+      connect_attempts += proc (pg, count, count + 1);
     }
 
   return connect_attempts;
@@ -1548,8 +1930,9 @@ create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
  * @return the number of connections that were set up
  *
  */
-static int
-create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_ring (struct GNUNET_TESTING_PeerGroup *pg,
+             GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int count;
   int connect_attempts;
@@ -1560,15 +1943,14 @@ create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      count, count + 1);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", count, count + 1);
 #endif
-      connect_attempts += proc(pg, count, count + 1);
+      connect_attempts += proc (pg, count, count + 1);
     }
 
   /* Connect the last peer to the first peer */
-  connect_attempts += proc(pg, pg->total - 1, 0);
+  connect_attempts += proc (pg, pg->total - 1, 0);
 
   return connect_attempts;
 }
@@ -1592,9 +1974,7 @@ create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
  *       "fixing" now.
  */
 static int
-friend_file_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+friend_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   FILE *temp_friend_handle = cls;
   struct GNUNET_TESTING_Daemon *peer = value;
@@ -1602,8 +1982,8 @@ friend_file_iterator (void *cls,
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
-  GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
-  fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
+  GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
+  fprintf (temp_friend_handle, "%s\n", (char *) &peer_enc);
 
   return GNUNET_YES;
 }
@@ -1631,9 +2011,7 @@ struct BlacklistContext
  * @return GNUNET_YES to continue iteration
  */
 static int
-blacklist_file_iterator (void *cls,
-                         const GNUNET_HashCode * key,
-                         void *value)
+blacklist_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct BlacklistContext *blacklist_ctx = cls;
   //FILE *temp_blacklist_handle = cls;
@@ -1642,8 +2020,9 @@ blacklist_file_iterator (void *cls,
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
-  GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
-  fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
+  GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
+  fprintf (blacklist_ctx->temp_file_handle, "%s:%s\n",
+           blacklist_ctx->transport, (char *) &peer_enc);
 
   return GNUNET_YES;
 }
@@ -1661,64 +2040,78 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
   FILE *temp_friend_handle;
   unsigned int pg_iter;
   char *temp_service_path;
-  pid_t *pidarr;
+  struct GNUNET_OS_Process **procarr;
   char *arg;
-  char * mytemp;
+  char *mytemp;
   enum GNUNET_OS_ProcessStatusType type;
   unsigned long return_code;
   int count;
   int ret;
   int max_wait = 10;
 
-  pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
+  procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      mytemp = GNUNET_DISK_mktemp("friends");
-      GNUNET_assert(mytemp != NULL);
+      mytemp = GNUNET_DISK_mktemp ("friends");
+      GNUNET_assert (mytemp != NULL);
       temp_friend_handle = fopen (mytemp, "wt");
-      GNUNET_assert(temp_friend_handle != NULL);
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
-      fclose(temp_friend_handle);
+      GNUNET_assert (temp_friend_handle != NULL);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].allowed_peers,
+                                             &friend_file_iterator,
+                                             temp_friend_handle);
+      fclose (temp_friend_handle);
 
       if (GNUNET_OK !=
-         GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
-       {
+          GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
+                                                 daemon->cfg, "PATHS",
+                                                 "SERVICEHOME",
+                                                 &temp_service_path))
+        {
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                     _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
-                     "SERVICEHOME",
-                     "PATHS");
+                      _
+                      ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
+                      "SERVICEHOME", "PATHS");
           if (UNLINK (mytemp) != 0)
-            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
-         GNUNET_free (mytemp);
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+                                      mytemp);
+          GNUNET_free (mytemp);
           break;
         }
 
-      if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
+      if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
         {
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
-                                         "mv", mytemp, arg, NULL);
+          procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
+                                                      "mv", mytemp, arg,
+                                                      NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
-      else /* Remote, scp the file to the correct place */
+      else                      /* Remote, scp the file to the correct place */
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
-            GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
+            GNUNET_asprintf (&arg, "%s@%s:%s/friends",
+                             pg->peers[pg_iter].daemon->username,
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
           else
-            GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
-                                         "scp", mytemp, arg, NULL);
+            GNUNET_asprintf (&arg, "%s:%s/friends",
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
+          procarr[pg_iter] =
+            GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
+                                     NULL);
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Copying file with command scp %s %s\n"), mytemp, arg);
+                      _("Copying file with command scp %s %s\n"), mytemp,
+                      arg);
 #endif
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
@@ -1735,22 +2128,25 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
-          if (pidarr[pg_iter] != 0) /* Check for already completed! */
+          if (procarr[pg_iter] != NULL) /* Check for already completed! */
             {
-              if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
+              if (GNUNET_OS_process_status
+                  (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
                 {
                   ret = GNUNET_SYSERR;
                 }
-              else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
+              else if ((type != GNUNET_OS_PROCESS_EXITED)
+                       || (return_code != 0))
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
-                  pidarr[pg_iter] = 0;
+                  GNUNET_OS_process_close (procarr[pg_iter]);
+                  procarr[pg_iter] = NULL;
 #if VERBOSE_TESTING
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("File %d copied\n"), pg_iter);
+                  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                              _("File %d copied\n"), pg_iter);
 #endif
                 }
             }
@@ -1758,16 +2154,16 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
       count++;
       if (ret == GNUNET_SYSERR)
         {
-         /* FIXME: why sleep here? -CG */
-          sleep(1);
+          /* FIXME: why sleep here? -CG */
+          sleep (1);
         }
     }
 
 #if VERBOSE_TESTING
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                _("Finished copying all friend files!\n"));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Finished copying all friend files!\n"));
 #endif
-  GNUNET_free(pidarr);
+  GNUNET_free (procarr);
   return ret;
 }
 
@@ -1781,13 +2177,14 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
  * @param transports space delimited list of transports to blacklist
  */
 static int
-create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
+create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
+                                 const char *transports)
 {
   FILE *temp_file_handle;
   static struct BlacklistContext blacklist_ctx;
   unsigned int pg_iter;
   char *temp_service_path;
-  pid_t *pidarr;
+  struct GNUNET_OS_Process **procarr;
   char *arg;
   char *mytemp;
   enum GNUNET_OS_ProcessStatusType type;
@@ -1800,77 +2197,93 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
   char *pos;
   char *temp_transports;
 
-  pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
+  procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      mytemp = GNUNET_DISK_mktemp("blacklist");
-      GNUNET_assert(mytemp != NULL);
+      mytemp = GNUNET_DISK_mktemp ("blacklist");
+      GNUNET_assert (mytemp != NULL);
       temp_file_handle = fopen (mytemp, "wt");
-      GNUNET_assert(temp_file_handle != NULL);
-      temp_transports = GNUNET_strdup(transports);
+      GNUNET_assert (temp_file_handle != NULL);
+      temp_transports = GNUNET_strdup (transports);
       blacklist_ctx.temp_file_handle = temp_file_handle;
-      transport_len = strlen(temp_transports) + 1;
+      transport_len = strlen (temp_transports) + 1;
       pos = NULL;
 
       for (i = 0; i < transport_len; i++)
-      {
-        if ((temp_transports[i] == ' ') && (pos == NULL))
-          continue; /* At start of string (whitespace) */
-        else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
         {
-          temp_transports[i] = '\0';
-          blacklist_ctx.transport = pos;
-          GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
-          pos = NULL;
-        } /* At beginning of actual string */
-        else if (pos == NULL)
-        {
-          pos = &temp_transports[i];
+          if ((temp_transports[i] == ' ') && (pos == NULL))
+            continue;           /* At start of string (whitespace) */
+          else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
+            {
+              temp_transports[i] = '\0';
+              blacklist_ctx.transport = pos;
+              GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                                     peers
+                                                     [pg_iter].blacklisted_peers,
+                                                     &blacklist_file_iterator,
+                                                     &blacklist_ctx);
+              pos = NULL;
+            }                   /* At beginning of actual string */
+          else if (pos == NULL)
+            {
+              pos = &temp_transports[i];
+            }
         }
-      }
 
       GNUNET_free (temp_transports);
-      fclose(temp_file_handle);
+      fclose (temp_file_handle);
 
       if (GNUNET_OK !=
-          GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
+          GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
+                                                 daemon->cfg, "PATHS",
+                                                 "SERVICEHOME",
+                                                 &temp_service_path))
         {
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                      _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
-                      "SERVICEHOME",
-                      "PATHS");
+                      _
+                      ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
+                      "SERVICEHOME", "PATHS");
           if (UNLINK (mytemp) != 0)
-            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+                                      mytemp);
           GNUNET_free (mytemp);
           break;
         }
 
-      if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
+      if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
         {
           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
-                                         "mv", mytemp, arg, NULL);
+          procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
+                                                      "mv", mytemp, arg,
+                                                      NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
-      else /* Remote, scp the file to the correct place */
+      else                      /* Remote, scp the file to the correct place */
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
-            GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
+            GNUNET_asprintf (&arg, "%s@%s:%s/blacklist",
+                             pg->peers[pg_iter].daemon->username,
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
           else
-            GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
-                                         "scp", mytemp, arg, NULL);
+            GNUNET_asprintf (&arg, "%s:%s/blacklist",
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
+          procarr[pg_iter] =
+            GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
+                                     NULL);
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Copying file with command scp %s %s\n"), mytemp, arg);
+                      _("Copying file with command scp %s %s\n"), mytemp,
+                      arg);
 #endif
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
@@ -1887,22 +2300,25 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
-          if (pidarr[pg_iter] != 0) /* Check for already completed! */
+          if (procarr[pg_iter] != NULL) /* Check for already completed! */
             {
-              if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
+              if (GNUNET_OS_process_status
+                  (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
                 {
                   ret = GNUNET_SYSERR;
                 }
-              else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
+              else if ((type != GNUNET_OS_PROCESS_EXITED)
+                       || (return_code != 0))
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
-                  pidarr[pg_iter] = 0;
+                  GNUNET_OS_process_close (procarr[pg_iter]);
+                  procarr[pg_iter] = NULL;
 #if VERBOSE_TESTING
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("File %d copied\n"), pg_iter);
+                  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                              _("File %d copied\n"), pg_iter);
 #endif
                 }
             }
@@ -1910,16 +2326,16 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
       count++;
       if (ret == GNUNET_SYSERR)
         {
-         /* FIXME: why sleep here? -CG */
-          sleep(1);
+          /* FIXME: why sleep here? -CG */
+          sleep (1);
         }
     }
 
 #if VERBOSE_TESTING
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                _("Finished copying all blacklist files!\n"));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Finished copying all blacklist files!\n"));
 #endif
-  GNUNET_free(pidarr);
+  GNUNET_free (procarr);
   return ret;
 }
 
@@ -1928,24 +2344,44 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
  * Internal notification of a connection, kept so that we can ensure some connections
  * happen instead of flooding all testing daemons with requests to connect.
  */
-static void internal_connect_notify (void *cls,
-                                     const struct GNUNET_PeerIdentity *first,
-                                     const struct GNUNET_PeerIdentity *second,
-                                     uint32_t distance,
-                                     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
-                                     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
-                                     struct GNUNET_TESTING_Daemon *first_daemon,
-                                     struct GNUNET_TESTING_Daemon *second_daemon,
-                                     const char *emsg)
-{
-  struct GNUNET_TESTING_PeerGroup *pg = cls;
+static void
+internal_connect_notify (void *cls,
+                         const struct GNUNET_PeerIdentity *first,
+                         const struct GNUNET_PeerIdentity *second,
+                         uint32_t distance,
+                         const struct GNUNET_CONFIGURATION_Handle *first_cfg,
+                         const struct GNUNET_CONFIGURATION_Handle *second_cfg,
+                         struct GNUNET_TESTING_Daemon *first_daemon,
+                         struct GNUNET_TESTING_Daemon *second_daemon,
+                         const char *emsg)
+{
+  struct ConnectTopologyContext *ct_ctx = cls;
+  struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
   outstanding_connects--;
+  ct_ctx->remaining_connections--;
+  if (ct_ctx->remaining_connections == 0)
+    {
+      if (ct_ctx->notify_connections_done != NULL)
+        ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+      GNUNET_free (ct_ctx);
+    }
 
-  pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
+  if (pg->notify_connection != NULL)
+    pg->notify_connection (pg->notify_connection_cls, first, second, distance,
+                           first_cfg, second_cfg, first_daemon, second_daemon,
+                           emsg);
 }
 
 
-static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+/**
+ * Either delay a connection (because there are too many outstanding)
+ * or schedule it for right now.
+ *
+ * @param cls a connection context
+ * @param tc the task runtime context
+ */
+static void
+schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct ConnectContext *connect_context = cls;
 
@@ -1955,16 +2391,20 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
     {
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Delaying connect, we have too many outstanding connections!\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
 #endif
-      GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_connect, connect_context);
     }
   else
     {
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
 #endif
       outstanding_connects++;
       GNUNET_TESTING_daemons_connect (connect_context->first,
@@ -1972,8 +2412,8 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
                                       CONNECT_TIMEOUT,
                                       CONNECT_ATTEMPTS,
                                       &internal_connect_notify,
-                                      connect_context->pg);
-      GNUNET_free(connect_context);
+                                      connect_context->ct_ctx);
+      GNUNET_free (connect_context);
     }
 }
 
@@ -1989,19 +2429,18 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
  * @return GNUNET_YES to continue iteration
  */
 static int
-connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
-  struct PeerData *first = cls;
+  struct ConnectTopologyContext *ct_ctx = cls;
+  struct PeerData *first = ct_ctx->first;
   struct GNUNET_TESTING_Daemon *second = value;
   struct ConnectContext *connect_context;
 
-  connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
-  connect_context->pg = first->pg;
+  connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
   connect_context->first = first->daemon;
   connect_context->second = second;
-  GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
+  connect_context->ct_ctx = ct_ctx;
+  GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
 
   return GNUNET_YES;
 }
@@ -2018,13 +2457,14 @@ connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
-copy_topology_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+copy_topology_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct PeerData *first = cls;
 
-  GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (first->connect_peers, key,
+                                                    value,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
   return GNUNET_YES;
 }
@@ -2045,7 +2485,11 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
   total = 0;
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
+      ret =
+        GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                               peers[pg_iter].allowed_peers,
+                                               &copy_topology_iterator,
+                                               &pg->peers[pg_iter]);
       if (GNUNET_SYSERR == ret)
         return GNUNET_SYSERR;
 
@@ -2061,37 +2505,64 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * of each peer in the peer group
  *
  * @param pg the peer group we are dealing with
+ * @param notify_callback callback to notify when finished
+ * @param notify_cls closure for notify callback
+ *
  * @return the number of connections that will be attempted
  */
 static int
-connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
+connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
+                  GNUNET_TESTING_NotifyCompletion notify_callback,
+                  void *notify_cls)
 {
   unsigned int pg_iter;
   int ret;
-  int total;
+  unsigned int total;
+  struct ConnectTopologyContext *ct_ctx;
 #if OLD
   struct PeerConnection *connection_iter;
   struct ConnectContext *connect_context;
 #endif
 
   total = 0;
+  ct_ctx = GNUNET_malloc (sizeof (struct ConnectTopologyContext));
+  ct_ctx->notify_connections_done = notify_callback;
+  ct_ctx->notify_cls = notify_cls;
+  ct_ctx->pg = pg;
+
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
-      if (GNUNET_SYSERR == ret)
-        return GNUNET_SYSERR;
+      total +=
+        GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
+    }
 
+  if (total == 0)
+    {
+      GNUNET_free (ct_ctx);
+      return total;
+    }
+  ct_ctx->remaining_connections = total;
+  total = 0;
+
+  for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
+    {
+      ct_ctx->first = &pg->peers[pg_iter];
+      ret =
+        GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                               peers[pg_iter].connect_peers,
+                                               &connect_iterator, ct_ctx);
+      GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
       total = total + ret;
 
 #if OLD
-      connection_iter = ;
+      connection_iter = FIXME;
       while (connection_iter != NULL)
         {
-          connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
+          connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
           connect_context->pg = pg;
-          connect_context->first = ;
+          connect_context->first = FIXME;
           connect_context->second = connection_iter->daemon;
-          GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
+          GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
           connection_iter = connection_iter->next;
         }
 #endif
@@ -2107,12 +2578,13 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * by the topology.  This will only have an effect once peers
  * are started if the FRIENDS_ONLY option is set in the base
  * config.  Also takes an optional restrict topology which
- * disallows connections based on a particular transport
+ * disallows connections based on particular transports
  * UNLESS they are specified in the restricted topology.
  *
  * @param pg the peer group struct representing the running peers
  * @param topology which topology to connect the peers in
- * @param restrict_topology allow only direct TCP connections in this topology
+ * @param restrict_topology disallow restrict_transports transport
+ *                          connections to peers NOT in this topology
  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
  * @param restrict_transports space delimited list of transports to blacklist
  *                            to create restricted topology
@@ -2120,25 +2592,22 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * @return the maximum number of connections were all allowed peers
  *         connected to each other
  */
-int
+unsigned int
 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                 enum GNUNET_TESTING_Topology topology,
-                                enum GNUNET_TESTING_Topology restrict_topology,
-                                char *restrict_transports)
+                                enum GNUNET_TESTING_Topology
+                                restrict_topology,
+                                const char *restrict_transports)
 {
   int ret;
-  int num_connections;
+  unsigned int num_connections;
   int unblacklisted_connections;
 
-  GNUNET_assert (pg->notify_connection != NULL);
-  ret = GNUNET_OK;
-
   switch (topology)
     {
     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating clique topology\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating clique topology\n"));
 #endif
       num_connections = create_clique (pg, &add_allowed_connections);
       break;
@@ -2147,7 +2616,8 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) topology\n"));
 #endif
-      num_connections = create_small_world_ring (pg, &add_allowed_connections);
+      num_connections =
+        create_small_world_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TESTING
@@ -2158,15 +2628,13 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating ring topology\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating ring topology\n"));
 #endif
       num_connections = create_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating 2d torus topology\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating 2d torus topology\n"));
 #endif
       num_connections = create_2d_torus (pg, &add_allowed_connections);
       break;
@@ -2179,8 +2647,7 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating InterNAT topology\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating InterNAT topology\n"));
 #endif
       num_connections = create_nated_internet (pg, &add_allowed_connections);
       break;
@@ -2201,7 +2668,8 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
     case GNUNET_TESTING_TOPOLOGY_NONE:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating no allowed topology (all peers can connect at core level)\n"));
+                  _
+                  ("Creating no allowed topology (all peers can connect at core level)\n"));
 #endif
       num_connections = 0;
       break;
@@ -2210,12 +2678,10 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     }
 
-  if (num_connections < 0)
-    return GNUNET_SYSERR;
-
-  if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
+  if (GNUNET_YES ==
+      GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
     {
-      ret = create_and_copy_friend_files(pg);
+      ret = create_and_copy_friend_files (pg);
       if (ret != GNUNET_OK)
         {
 #if VERBOSE_TESTING
@@ -2227,8 +2693,8 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       else
         {
 #if VERBOSE_TESTING
-              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                          _("Friend files created/copied successfully!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Friend files created/copied successfully!\n"));
 #endif
         }
     }
@@ -2236,6 +2702,7 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
   /* Use the create clique method to initially set all connections as blacklisted. */
   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
     create_clique (pg, &blacklist_connections);
+
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
   switch (restrict_topology)
@@ -2245,21 +2712,25 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but clique topology\n"));
 #endif
-      unblacklisted_connections = create_clique (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_clique (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but small world (ring) topology\n"));
 #endif
-      unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_small_world_ring (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Blacklisting all but small world (2d-torus) topology\n"));
+                  _
+                  ("Blacklisting all but small world (2d-torus) topology\n"));
 #endif
-      unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_small_world (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TESTING
@@ -2273,28 +2744,32 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but 2d torus topology\n"));
 #endif
-      unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_2d_torus (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Erdos-Renyi topology\n"));
 #endif
-      unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_erdos_renyi (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but InterNAT topology\n"));
 #endif
-      unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_nated_internet (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Scale Free topology\n"));
 #endif
-      unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_scale_free (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_LINE:
 #if VERBOSE_TESTING
@@ -2306,31 +2781,32 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
     case GNUNET_TESTING_TOPOLOGY_NONE:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating no blacklist topology (all peers can connect at transport level)\n"));
+                  _
+                  ("Creating no blacklist topology (all peers can connect at transport level)\n"));
 #endif
     default:
       break;
     }
 
   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
-  {
-    ret = create_and_copy_blacklist_files(pg, restrict_transports);
-    if (ret != GNUNET_OK)
-      {
+    {
+      ret = create_and_copy_blacklist_files (pg, restrict_transports);
+      if (ret != GNUNET_OK)
+        {
 #if VERBOSE_TESTING
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    _("Failed during blacklist file copying!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Failed during blacklist file copying!\n"));
 #endif
-        return GNUNET_SYSERR;
-      }
-    else
-      {
+          return 0;
+        }
+      else
+        {
 #if VERBOSE_TESTING
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    _("Blacklist files created/copied successfully!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Blacklist files created/copied successfully!\n"));
 #endif
-      }
-  }
+        }
+    }
   return num_connections;
 }
 
@@ -2434,25 +2910,35 @@ struct DFSContext
  * @return GNUNET_YES to continue iteration
  */
 static int
-random_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+random_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct RandomContext *random_ctx = cls;
   double random_number;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
-  random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                    UINT64_MAX)) / ( (double) UINT64_MAX);
+  random_number =
+    ((double)
+     GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                               UINT64_MAX)) / ((double) UINT64_MAX);
   if (random_number < random_ctx->percentage)
-  {
-    GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  }
+    {
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (random_ctx->
+                                                        first->connect_peers_working_set,
+                                                        key, value,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+    }
   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
-  uid_from_hash(key, &second_pos);
-  hash_from_uid(random_ctx->first_uid, &first_hash);
-  GNUNET_assert(random_ctx->pg->total > second_pos);
-  GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
+  uid_from_hash (key, &second_pos);
+  hash_from_uid (random_ctx->first_uid, &first_hash);
+  GNUNET_assert (random_ctx->pg->total > second_pos);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (random_ctx->
+                                                       pg->peers
+                                                       [second_pos].connect_peers,
+                                                       &first_hash,
+                                                       random_ctx->
+                                                       first->daemon));
 
   return GNUNET_YES;
 }
@@ -2467,35 +2953,48 @@ random_connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
-minimum_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+minimum_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct MinimumContext *min_ctx = cls;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
   unsigned int i;
 
-  if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
-  {
-    for (i = 0; i < min_ctx->num_to_add; i++)
-    {
-      if (min_ctx->pg_array[i] == min_ctx->current)
-      {
-        GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        uid_from_hash(key, &second_pos);
-        hash_from_uid(min_ctx->first_uid, &first_hash);
-        GNUNET_assert(min_ctx->pg->total > second_pos);
-        GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
-        GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
-      }
-    }
-    min_ctx->current++;
-    return GNUNET_YES;
-  }
+  if (GNUNET_CONTAINER_multihashmap_size
+      (min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
+    {
+      for (i = 0; i < min_ctx->num_to_add; i++)
+        {
+          if (min_ctx->pg_array[i] == min_ctx->current)
+            {
+              GNUNET_assert (GNUNET_OK ==
+                             GNUNET_CONTAINER_multihashmap_put
+                             (min_ctx->first->connect_peers_working_set, key,
+                              value,
+                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+              uid_from_hash (key, &second_pos);
+              hash_from_uid (min_ctx->first_uid, &first_hash);
+              GNUNET_assert (min_ctx->pg->total > second_pos);
+              GNUNET_assert (GNUNET_OK ==
+                             GNUNET_CONTAINER_multihashmap_put (min_ctx->
+                                                                pg->peers
+                                                                [second_pos].connect_peers_working_set,
+                                                                &first_hash,
+                                                                min_ctx->first->
+                                                                daemon,
+                                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+              /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
+              GNUNET_assert (GNUNET_YES ==
+                             GNUNET_CONTAINER_multihashmap_remove
+                             (min_ctx->pg->peers[second_pos].connect_peers,
+                              &first_hash, min_ctx->first->daemon));
+            }
+        }
+      min_ctx->current++;
+      return GNUNET_YES;
+    }
   else
-    return GNUNET_NO; /* We can stop iterating, we have enough peers! */
+    return GNUNET_NO;           /* We can stop iterating, we have enough peers! */
 
 }
 
@@ -2510,22 +3009,37 @@ minimum_connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
-dfs_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+dfs_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct DFSContext *dfs_ctx = cls;
   GNUNET_HashCode first_hash;
 
   if (dfs_ctx->current == dfs_ctx->chosen)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-      uid_from_hash(key, &dfs_ctx->second_uid);
-      hash_from_uid(dfs_ctx->first_uid, &first_hash);
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
+                                                        first->connect_peers_working_set,
+                                                        key, value,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      uid_from_hash (key, &dfs_ctx->second_uid);
+      hash_from_uid (dfs_ctx->first_uid, &first_hash);
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
+                                                        pg->peers
+                                                        [dfs_ctx->second_uid].connect_peers_working_set,
+                                                        &first_hash,
+                                                        dfs_ctx->
+                                                        first->daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (dfs_ctx->
+                                                           pg->peers
+                                                           [dfs_ctx->second_uid].connect_peers,
+                                                           &first_hash,
+                                                           dfs_ctx->
+                                                           first->daemon));
       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
-      return GNUNET_NO; /* We have found our peer, don't iterate more */
+      return GNUNET_NO;         /* We have found our peer, don't iterate more */
     }
 
   dfs_ctx->current++;
@@ -2541,7 +3055,8 @@ dfs_connect_iterator (void *cls,
  * @param percentage what percent of total connections to make
  */
 void
-choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
+choose_random_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                           double percentage)
 {
   struct RandomContext random_ctx;
   uint32_t pg_iter;
@@ -2552,12 +3067,17 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage
       random_ctx.first = &pg->peers[pg_iter];
       random_ctx.percentage = percentage;
       random_ctx.pg = pg;
-      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (pg->total);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
+                                             &random_connect_iterator,
+                                             &random_ctx);
       /* Now remove the old connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the random set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
     }
 }
 
@@ -2569,43 +3089,48 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage
  * @param num how many connections at least should each peer have (if possible)?
  */
 static void
-choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
+choose_minimum (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
 {
   struct MinimumContext minimum_ctx;
   uint32_t pg_iter;
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
-   {
-     pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
-   }
+    {
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (num);
+    }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       minimum_ctx.first_uid = pg_iter;
-      minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, 
-                                                         GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
+      minimum_ctx.pg_array =
+        GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK,
+                                      GNUNET_CONTAINER_multihashmap_size
+                                      (pg->peers[pg_iter].connect_peers));
       minimum_ctx.first = &pg->peers[pg_iter];
       minimum_ctx.pg = pg;
       minimum_ctx.num_to_add = num;
       minimum_ctx.current = 0;
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
-                                           &minimum_connect_iterator, 
-                                           &minimum_ctx);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
+                                             &minimum_connect_iterator,
+                                             &minimum_ctx);
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the working set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
     }
 
 }
 
 
 static unsigned int
-count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
+count_workingset_connections (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int count;
   unsigned int pg_iter;
@@ -2614,14 +3139,18 @@ count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
+      count +=
+        GNUNET_CONTAINER_multihashmap_size (pg->
+                                            peers
+                                            [pg_iter].connect_peers_working_set);
     }
 
   return count;
 }
 
 
-static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
+static unsigned int
+count_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int count;
   unsigned int pg_iter;
@@ -2630,12 +3159,116 @@ static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *p
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
+      count +=
+        GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
     }
 
   return count;
 }
 
+
+struct FindClosestContext
+{
+  /**
+   * The currently known closest peer.
+   */
+  struct GNUNET_TESTING_Daemon *closest;
+
+  /**
+   * The info for the peer we are adding connections for.
+   */
+  struct PeerData *curr_peer;
+
+  /**
+   * The distance (bits) between the current
+   * peer and the currently known closest.
+   */
+  unsigned int closest_dist;
+
+  /**
+   * The offset of the closest known peer in
+   * the peer group.
+   */
+  unsigned int closest_num;
+};
+
+/**
+ * Iterator over hash map entries of the allowed
+ * peer connections.  Find the closest, not already
+ * connected peer and return it.
+ *
+ * @param cls closure (struct FindClosestContext)
+ * @param key current key code (hash of offset in pg)
+ * @param value value in the hash map - a GNUNET_TESTING_Daemon
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int
+find_closest_peers (void *cls, const GNUNET_HashCode * key, void *value)
+{
+  struct FindClosestContext *closest_ctx = cls;
+  struct GNUNET_TESTING_Daemon *daemon = value;
+
+  if (((closest_ctx->closest == NULL) ||
+       (GNUNET_CRYPTO_hash_matching_bits
+        (&daemon->id.hashPubKey,
+         &closest_ctx->curr_peer->daemon->id.hashPubKey) >
+        closest_ctx->closest_dist))
+      && (GNUNET_YES !=
+          GNUNET_CONTAINER_multihashmap_contains (closest_ctx->
+                                                  curr_peer->connect_peers,
+                                                  key)))
+    {
+      closest_ctx->closest_dist =
+        GNUNET_CRYPTO_hash_matching_bits (&daemon->id.hashPubKey,
+                                          &closest_ctx->curr_peer->daemon->
+                                          id.hashPubKey);
+      closest_ctx->closest = daemon;
+      uid_from_hash (key, &closest_ctx->closest_num);
+    }
+  return GNUNET_YES;
+}
+
+/**
+ * From the set of connections possible, choose at num connections per
+ * peer based on depth which are closest out of those allowed.  Guaranteed
+ * to add num peers to connect to, provided there are that many peers
+ * in the underlay topology to connect to.
+ *
+ * @param pg the peergroup we are dealing with
+ * @param num how many connections at least should each peer have (if possible)?
+ * @param proc processor to actually add the connections
+ */
+void
+add_closest (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num,
+             GNUNET_TESTING_ConnectionProcessor proc)
+{
+  struct FindClosestContext closest_ctx;
+  uint32_t pg_iter;
+  uint32_t i;
+
+  for (i = 0; i < num; i++)     /* Each time find a closest peer (from those available) */
+    {
+      for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
+        {
+          closest_ctx.curr_peer = &pg->peers[pg_iter];
+          closest_ctx.closest = NULL;
+          closest_ctx.closest_dist = 0;
+          closest_ctx.closest_num = 0;
+          GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                                 peers[pg_iter].allowed_peers,
+                                                 &find_closest_peers,
+                                                 &closest_ctx);
+          if (closest_ctx.closest != NULL)
+            {
+              GNUNET_assert (closest_ctx.closest_num < pg->total);
+              proc (pg, pg_iter, closest_ctx.closest_num);
+            }
+        }
+    }
+}
+
 /**
  * From the set of connections possible, choose at least num connections per
  * peer based on depth first traversal of peer connections.  If DFS leaves
@@ -2656,53 +3289,437 @@ perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (num);
     }
 
   starting_peer = 0;
   dfs_count = 0;
-  while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
+  while ((count_workingset_connections (pg) < num * pg->total)
+         && (count_allowed_connections (pg) > 0))
     {
-      if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
+      if (dfs_count % pg->total == 0)   /* Restart the DFS at some weakly connected peer */
         {
-          least_connections = -1; /* Set to very high number */
+          least_connections = -1;       /* Set to very high number */
           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
             {
-              if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
+              if (GNUNET_CONTAINER_multihashmap_size
+                  (pg->peers[pg_iter].connect_peers_working_set) <
+                  least_connections)
                 {
                   starting_peer = pg_iter;
-                  least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
+                  least_connections =
+                    GNUNET_CONTAINER_multihashmap_size (pg->
+                                                        peers
+                                                        [pg_iter].connect_peers_working_set);
                 }
             }
         }
 
-      if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
+      if (GNUNET_CONTAINER_multihashmap_size (pg->peers[starting_peer].connect_peers) == 0)     /* Ensure there is at least one peer left to connect! */
         {
           dfs_count = 0;
           continue;
         }
 
       /* Choose a random peer from the chosen peers set of connections to add */
-      dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
+      dfs_ctx.chosen =
+        GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                  GNUNET_CONTAINER_multihashmap_size
+                                  (pg->peers[starting_peer].connect_peers));
       dfs_ctx.first_uid = starting_peer;
       dfs_ctx.first = &pg->peers[starting_peer];
       dfs_ctx.pg = pg;
       dfs_ctx.current = 0;
 
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                             peers
+                                             [starting_peer].connect_peers,
+                                             &dfs_connect_iterator, &dfs_ctx);
       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
-      hash_from_uid(dfs_ctx.second_uid, &second_hash);
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
+      hash_from_uid (dfs_ctx.second_uid, &second_hash);
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->peers
+                                                           [starting_peer].connect_peers,
+                                                           &second_hash,
+                                                           pg->
+                                                           peers
+                                                           [dfs_ctx.second_uid].daemon));
       starting_peer = dfs_ctx.second_uid;
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the working set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
+    }
+}
+
+/**
+ * Internal callback for topology information for a particular peer.
+ */
+static void
+internal_topology_callback (void *cls,
+                            const struct GNUNET_PeerIdentity *peer,
+                            const struct GNUNET_TRANSPORT_ATS_Information
+                            *atsi)
+{
+  struct CoreContext *core_ctx = cls;
+  struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
+
+  if (peer == NULL)             /* Either finished, or something went wrong */
+    {
+      iter_ctx->completed++;
+      iter_ctx->connected--;
+      /* One core context allocated per iteration, must free! */
+      GNUNET_free (core_ctx);
+    }
+  else
+    {
+      iter_ctx->topology_cb (iter_ctx->cls, &core_ctx->daemon->id,
+                             peer, NULL);
+    }
+
+  if (iter_ctx->completed == iter_ctx->total)
+    {
+      iter_ctx->topology_cb (iter_ctx->cls, NULL, NULL, NULL);
+      /* Once all are done, free the iteration context */
+      GNUNET_free (iter_ctx);
+    }
+}
+
+
+/**
+ * Check running topology iteration tasks, if below max start a new one, otherwise
+ * schedule for some time in the future.
+ */
+static void
+schedule_get_topology (void *cls,
+                       const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct CoreContext *core_context = cls;
+  struct TopologyIterateContext *topology_context =
+    (struct TopologyIterateContext *) core_context->iter_context;
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    return;
+
+  if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
+#endif
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_get_topology, core_context);
+    }
+  else
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
+#endif
+      topology_context->connected++;
+
+      if (GNUNET_OK !=
+          GNUNET_CORE_iterate_peers (core_context->daemon->cfg,
+                                     &internal_topology_callback,
+                                     core_context))
+        {
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Topology iteration failed.\n");
+          internal_topology_callback (core_context, NULL, NULL);
+        }
+    }
+}
+
+/**
+ * Iterate over all (running) peers in the peer group, retrieve
+ * all connections that each currently has.
+ */
+void
+GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
+                             GNUNET_TESTING_NotifyTopology cb, void *cls)
+{
+  struct TopologyIterateContext *topology_context;
+  struct CoreContext *core_ctx;
+  unsigned int i;
+  unsigned int total_count;
+
+  /* Allocate a single topology iteration context */
+  topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
+  topology_context->topology_cb = cb;
+  topology_context->cls = cls;
+  total_count = 0;
+  for (i = 0; i < pg->total; i++)
+    {
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          /* Allocate one core context per core we need to connect to */
+          core_ctx = GNUNET_malloc (sizeof (struct CoreContext));
+          core_ctx->daemon = pg->peers[i].daemon;
+          /* Set back pointer to topology iteration context */
+          core_ctx->iter_context = topology_context;
+          GNUNET_SCHEDULER_add_now (&schedule_get_topology, core_ctx);
+          total_count++;
+        }
+    }
+  if (total_count == 0)
+    {
+      cb (cls, NULL, NULL, "Cannot iterate over topology, no running peers!");
+      GNUNET_free (topology_context);
+    }
+  else
+    topology_context->total = total_count;
+  return;
+}
+
+/**
+ * Callback function to process statistic values.
+ * This handler is here only really to insert a peer
+ * identity (or daemon) so the statistics can be uniquely
+ * tied to a single running peer.
+ *
+ * @param cls closure
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+internal_stats_callback (void *cls,
+                         const char *subsystem,
+                         const char *name, uint64_t value, int is_persistent)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  return stats_context->proc (stats_context->cls, &core_context->daemon->id,
+                              subsystem, name, value, is_persistent);
+}
+
+/**
+ * Internal continuation call for statistics iteration.
+ *
+ * @param cls closure, the CoreContext for this iteration
+ * @param success whether or not the statistics iterations
+ *        was canceled or not (we don't care)
+ */
+static void
+internal_stats_cont (void *cls, int success)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  stats_context->connected--;
+  stats_context->completed++;
+
+  if (stats_context->completed == stats_context->total)
+    {
+      stats_context->cont (stats_context->cls, GNUNET_YES);
+      GNUNET_free (stats_context);
+    }
+
+  if (core_context->stats_handle != NULL)
+    GNUNET_STATISTICS_destroy (core_context->stats_handle, GNUNET_NO);
+
+  GNUNET_free (core_context);
+}
+
+/**
+ * Check running topology iteration tasks, if below max start a new one, otherwise
+ * schedule for some time in the future.
+ */
+static void
+schedule_get_statistics (void *cls,
+                         const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    return;
+
+  if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
+#endif
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_get_statistics, core_context);
+    }
+  else
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
+#endif
+
+      stats_context->connected++;
+      core_context->stats_handle =
+        GNUNET_STATISTICS_create ("testing", core_context->daemon->cfg);
+      if (core_context->stats_handle == NULL)
+        {
+          internal_stats_cont (core_context, GNUNET_NO);
+          return;
+        }
+
+      core_context->stats_get_handle =
+        GNUNET_STATISTICS_get (core_context->stats_handle, NULL, NULL,
+                               GNUNET_TIME_relative_get_forever (),
+                               &internal_stats_cont, &internal_stats_callback,
+                               core_context);
+      if (core_context->stats_get_handle == NULL)
+        internal_stats_cont (core_context, GNUNET_NO);
+
+    }
+}
+
+struct DuplicateStats
+{
+  /**
+   * Next item in the list
+   */
+  struct DuplicateStats *next;
+
+  /**
+   * Nasty string, concatenation of relevant information.
+   */
+  char *unique_string;
+};
+
+/**
+ * Check whether the combination of port/host/unix domain socket
+ * already exists in the list of peers being checked for statistics.
+ *
+ * @param pg the peergroup in question
+ * @param specific_peer the peer we're concerned with
+ * @param stats_list the list to return to the caller
+ *
+ * @return GNUNET_YES if the statistics instance has been seen already,
+ *         GNUNET_NO if not (and we may have added it to the list)
+ */
+static int
+stats_check_existing (struct GNUNET_TESTING_PeerGroup *pg,
+                      struct PeerData *specific_peer,
+                      struct DuplicateStats **stats_list)
+{
+  struct DuplicateStats *pos;
+  char *unix_domain_socket;
+  unsigned long long port;
+  char *to_match;
+  if (GNUNET_YES !=
+      GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "testing",
+                                            "single_statistics_per_host"))
+    return GNUNET_NO;           /* Each peer has its own statistics instance, do nothing! */
+
+  pos = *stats_list;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (specific_peer->cfg, "statistics",
+                                             "unixpath", &unix_domain_socket))
+    return GNUNET_NO;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (specific_peer->cfg, "statistics",
+                                             "port", &port))
+    {
+      GNUNET_free(unix_domain_socket);
+      return GNUNET_NO;
+    }
+
+  if (specific_peer->daemon->hostname != NULL)
+    GNUNET_asprintf (&to_match, "%s%s%llu", specific_peer->daemon->hostname,
+                     unix_domain_socket, port);
+  else
+    GNUNET_asprintf (&to_match, "%s%llu", unix_domain_socket, port);
+
+  while (pos != NULL)
+    {
+      if (0 == strcmp (to_match, pos->unique_string))
+        {
+          GNUNET_free (unix_domain_socket);
+          GNUNET_free (to_match);
+          return GNUNET_YES;
+        }
+      pos = pos->next;
+    }
+  pos = GNUNET_malloc (sizeof (struct DuplicateStats));
+  pos->unique_string = to_match;
+  pos->next = *stats_list;
+  *stats_list = pos;
+  GNUNET_free (unix_domain_socket);
+  return GNUNET_NO;
+}
+
+/**
+ * Iterate over all (running) peers in the peer group, retrieve
+ * all statistics from each.
+ */
+void
+GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
+                               GNUNET_STATISTICS_Callback cont,
+                               GNUNET_TESTING_STATISTICS_Iterator proc,
+                               void *cls)
+{
+  struct StatsIterateContext *stats_context;
+  struct StatsCoreContext *core_ctx;
+  unsigned int i;
+  unsigned int total_count;
+  struct DuplicateStats *stats_list;
+  struct DuplicateStats *pos;
+  stats_list = NULL;
+
+  /* Allocate a single stats iteration context */
+  stats_context = GNUNET_malloc (sizeof (struct StatsIterateContext));
+  stats_context->cont = cont;
+  stats_context->proc = proc;
+  stats_context->cls = cls;
+  total_count = 0;
+
+  for (i = 0; i < pg->total; i++)
+    {
+      if ((pg->peers[i].daemon->running == GNUNET_YES)
+          && (GNUNET_NO ==
+              stats_check_existing (pg, &pg->peers[i], &stats_list)))
+        {
+          /* Allocate one core context per core we need to connect to */
+          core_ctx = GNUNET_malloc (sizeof (struct StatsCoreContext));
+          core_ctx->daemon = pg->peers[i].daemon;
+          /* Set back pointer to topology iteration context */
+          core_ctx->iter_context = stats_context;
+          GNUNET_SCHEDULER_add_now (&schedule_get_statistics, core_ctx);
+          total_count++;
+        }
     }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Retrieving stats from %u total instances.\n", total_count);
+  stats_context->total = total_count;
+  if (stats_list != NULL)
+    {
+      pos = stats_list;
+      while (pos != NULL)
+        {
+          GNUNET_free (pos->unique_string);
+          stats_list = pos->next;
+          GNUNET_free (pos);
+          pos = stats_list->next;
+        }
+    }
+  return;
 }
 
 /**
@@ -2718,114 +3735,135 @@ perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
  * @param topology which topology to connect the peers in
  * @param options options for connecting the topology
  * @param option_modifier modifier for options that take a parameter
+ * @param notify_callback notification to be called once all connections completed
+ * @param notify_cls closure for notification callback
+ *
  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
  */
 int
 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                  enum GNUNET_TESTING_Topology topology,
                                  enum GNUNET_TESTING_TopologyOption options,
-                                 double option_modifier)
+                                 double option_modifier,
+                                 GNUNET_TESTING_NotifyCompletion
+                                 notify_callback, void *notify_cls)
 {
   switch (topology)
-      {
-      case GNUNET_TESTING_TOPOLOGY_CLIQUE:
+    {
+    case GNUNET_TESTING_TOPOLOGY_CLIQUE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating clique CONNECT topology\n"));
 #endif
-        create_clique (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
+      create_clique (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) CONNECT topology\n"));
 #endif
-        create_small_world_ring (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
+      create_small_world_ring (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) CONNECT topology\n"));
 #endif
-        create_small_world (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_RING:
+      create_small_world (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating ring CONNECT topology\n"));
 #endif
-        create_ring (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
+      create_ring (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating 2d torus CONNECT topology\n"));
 #endif
-        create_2d_torus (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
+      create_2d_torus (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi CONNECT topology\n"));
 #endif
-        create_erdos_renyi (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_INTERNAT:
+      create_erdos_renyi (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating InterNAT CONNECT topology\n"));
 #endif
-        create_nated_internet (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
+      create_nated_internet (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Scale Free CONNECT topology\n"));
 #endif
-        create_scale_free (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_LINE:
+      create_scale_free (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_LINE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating straight line CONNECT topology\n"));
 #endif
-        create_line (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_NONE:
+      create_line (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_NONE:
 #if VERBOSE_TOPOLOGY
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating no CONNECT topology\n"));
 #endif
-        copy_allowed_topology(pg);
-        break;
-      default:
-        GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-                  _("Unknown topology specification, can't connect peers!\n"));
-        return GNUNET_SYSERR;
-      }
+      copy_allowed_topology (pg);
+      break;
+    default:
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Unknown topology specification, can't connect peers!\n"));
+      return GNUNET_SYSERR;
+    }
 
   switch (options)
     {
     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
+                  _
+                  ("Connecting random subset (%'.2f percent) of possible peers\n"),
+                  100 * option_modifier);
 #endif
-      choose_random_connections(pg, option_modifier);
+      choose_random_connections (pg, option_modifier);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
+                  _("Connecting a minimum of %u peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
 #endif
-      choose_minimum(pg, (unsigned int)option_modifier);
+      choose_minimum (pg, (unsigned int) option_modifier);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
+                  _
+                  ("Using DFS to connect a minimum of %u peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
+#endif
+      perform_dfs (pg, (int) option_modifier);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_OPTION_ADD_CLOSEST:
+#if VERBOSE_TOPOLOGY
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Finding additional %u closest peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
 #endif
-      perform_dfs(pg, (int)option_modifier);
+      add_closest (pg, (unsigned int) option_modifier,
+                   &add_actual_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
       break;
@@ -2835,7 +3873,7 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     }
 
-  return connect_topology(pg);
+  return connect_topology (pg, notify_callback, notify_cls);
 }
 
 /**
@@ -2848,20 +3886,22 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
  * @param d handle for the daemon
  * @param emsg error message (NULL on success)
  */
-static void internal_hostkey_callback (void *cls,
-                                       const struct GNUNET_PeerIdentity *id,
-                                       struct GNUNET_TESTING_Daemon *d,
-                                       const char *emsg)
+static void
+internal_hostkey_callback (void *cls,
+                           const struct GNUNET_PeerIdentity *id,
+                           struct GNUNET_TESTING_Daemon *d, const char *emsg)
 {
   struct InternalStartContext *internal_context = cls;
   internal_context->peer->pg->starting--;
   internal_context->peer->pg->started++;
   if (internal_context->hostkey_callback != NULL)
-    internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
-  else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
+    internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
+                                        emsg);
+  else if (internal_context->peer->pg->started ==
+           internal_context->peer->pg->total)
     {
-      internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
-      GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
+      internal_context->peer->pg->started = 0;  /* Internal startup may use this counter! */
+      GNUNET_TESTING_daemons_continue_startup (internal_context->peer->pg);
     }
 }
 
@@ -2872,23 +3912,26 @@ static void internal_hostkey_callback (void *cls,
  *
  * @param cls closure
  * @param id identifier for the daemon, NULL on error
+ * @param cfg config
  * @param d handle for the daemon
  * @param emsg error message (NULL on success)
  */
-static void internal_startup_callback (void *cls,
-                                       const struct GNUNET_PeerIdentity *id,
-                                       const struct GNUNET_CONFIGURATION_Handle *cfg,
-                                       struct GNUNET_TESTING_Daemon *d,
-                                       const char *emsg)
+static void
+internal_startup_callback (void *cls,
+                           const struct GNUNET_PeerIdentity *id,
+                           const struct GNUNET_CONFIGURATION_Handle *cfg,
+                           struct GNUNET_TESTING_Daemon *d, const char *emsg)
 {
   struct InternalStartContext *internal_context = cls;
   internal_context->peer->pg->starting--;
   if (internal_context->start_cb != NULL)
-    internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
+    internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
+                                emsg);
 }
 
 static void
-internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
+internal_continue_startup (void *cls,
+                           const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct InternalStartContext *internal_context = cls;
 
@@ -2904,12 +3947,98 @@ internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext
     }
   else
     {
-      GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &internal_continue_startup,
+                                    internal_context);
+    }
+}
+
+
+/**
+ * Callback for informing us about a successful
+ * or unsuccessful churn start call.
+ *
+ * @param cls a ChurnContext
+ * @param id the peer identity of the started peer
+ * @param cfg the handle to the configuration of the peer
+ * @param d handle to the daemon for the peer
+ * @param emsg NULL on success, non-NULL on failure
+ *
+ */
+void
+churn_start_callback (void *cls,
+                      const struct GNUNET_PeerIdentity *id,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      struct GNUNET_TESTING_Daemon *d, const char *emsg)
+{
+  struct ChurnRestartContext *startup_ctx = cls;
+  struct ChurnContext *churn_ctx = startup_ctx->churn_ctx;
+
+  unsigned int total_left;
+  char *error_message;
+
+  error_message = NULL;
+  if (emsg != NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Churn stop callback failed with error `%s'\n", emsg);
+      churn_ctx->num_failed_start++;
+    }
+  else
+    {
+      churn_ctx->num_to_start--;
+    }
+
+#if DEBUG_CHURN
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Started peer, %d left.\n", churn_ctx->num_to_start);
+#endif
+
+  total_left =
+    (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
+    (churn_ctx->num_to_start - churn_ctx->num_failed_start);
+
+  if (total_left == 0)
+    {
+      if ((churn_ctx->num_failed_stop > 0)
+          || (churn_ctx->num_failed_start > 0))
+        GNUNET_asprintf (&error_message,
+                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
+                         churn_ctx->num_failed_start,
+                         churn_ctx->num_failed_stop);
+      churn_ctx->cb (churn_ctx->cb_cls, error_message);
+      GNUNET_free_non_null (error_message);
+      GNUNET_free (churn_ctx);
+      GNUNET_free (startup_ctx);
+    }
+}
+
+
+static void
+schedule_churn_restart (void *cls,
+                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerRestartContext *peer_restart_ctx = cls;
+  struct ChurnRestartContext *startup_ctx =
+    peer_restart_ctx->churn_restart_ctx;
+
+  if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_churn_restart, peer_restart_ctx);
+  else
+    {
+      GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon,
+                                           startup_ctx->timeout,
+                                           &churn_start_callback,
+                                           startup_ctx);
+      GNUNET_free (peer_restart_ctx);
     }
 }
 
 static void
-internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
+internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct InternalStartContext *internal_context = cls;
 
@@ -2921,18 +4050,22 @@ internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
     {
       internal_context->peer->pg->starting++;
-      internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
-                                                                    internal_context->peer->cfg,
-                                                                    internal_context->timeout,
-                                                                    internal_context->hostname,
-                                                                    &internal_hostkey_callback,
-                                                                    internal_context,
-                                                                    &internal_startup_callback,
-                                                                    internal_context);
+      internal_context->peer->daemon =
+        GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
+                                     internal_context->timeout,
+                                     internal_context->hostname,
+                                     internal_context->username,
+                                     internal_context->sshport,
+                                     &internal_hostkey_callback,
+                                     internal_context,
+                                     &internal_startup_callback,
+                                     internal_context);
     }
   else
     {
-      GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &internal_start, internal_context);
     }
 }
 
@@ -2944,14 +4077,15 @@ internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
  *
  */
 void
-GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
+GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int i;
 
   pg->starting = 0;
   for (i = 0; i < pg->total; i++)
     {
-      GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
+      GNUNET_SCHEDULER_add_now (&internal_continue_startup,
+                                &pg->peers[i].internal_context);
       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
     }
 }
@@ -2962,7 +4096,6 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
  * adjusted to ensure that no two peers running on the same system
  * have the same port(s) in their respective configurations.
  *
- * @param sched scheduler to use
  * @param cfg configuration template to use
  * @param total number of daemons to start
  * @param timeout total time allowed for peers to start
@@ -2975,28 +4108,31 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
  * @param cb_cls closure for cb
  * @param connect_callback function to call each time two hosts are connected
  * @param connect_callback_cls closure for connect_callback
- * @param hostnames space-separated list of hostnames to use; can be NULL (to run
- *        everything on localhost).
+ * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
+ *
  * @return NULL on error, otherwise handle to control peer group
  */
 struct GNUNET_TESTING_PeerGroup *
-GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
-                              const struct GNUNET_CONFIGURATION_Handle *cfg,
+GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                               unsigned int total,
                               struct GNUNET_TIME_Relative timeout,
-                              GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
-                              void *hostkey_cls,
+                              GNUNET_TESTING_NotifyHostkeyCreated
+                              hostkey_callback, void *hostkey_cls,
                               GNUNET_TESTING_NotifyDaemonRunning cb,
                               void *cb_cls,
                               GNUNET_TESTING_NotifyConnection
                               connect_callback, void *connect_callback_cls,
-                              const char *hostnames)
+                              const struct GNUNET_TESTING_Host *hostnames)
 {
   struct GNUNET_TESTING_PeerGroup *pg;
-  const char *rpos;
+  const struct GNUNET_TESTING_Host *hostpos;
+#if 0
   char *pos;
+  const char *rpos;
   char *start;
+#endif
   const char *hostname;
+  const char *username;
   char *baseservicehome;
   char *newservicehome;
   char *tmpdir;
@@ -3004,7 +4140,9 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
   unsigned int off;
   unsigned int hostcnt;
   uint16_t minport;
+  uint16_t sshport;
   uint32_t upnum;
+  uint32_t fdnum;
 
   if (0 == total)
     {
@@ -3012,24 +4150,55 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       return NULL;
     }
   upnum = 0;
+  fdnum = 0;
   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
-  pg->sched = sched;
   pg->cfg = cfg;
   pg->notify_connection = connect_callback;
   pg->notify_connection_cls = connect_callback_cls;
   pg->total = total;
-  pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
+  pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
   if (NULL != hostnames)
     {
+      off = 0;
+      hostpos = hostnames;
+      while (hostpos != NULL)
+        {
+          hostpos = hostpos->next;
+          off++;
+        }
+      pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
+      off = 0;
+
+      hostpos = hostnames;
+      while (hostpos != NULL)
+        {
+          pg->hosts[off].minport = LOW_PORT;
+          pg->hosts[off].hostname = GNUNET_strdup (hostpos->hostname);
+          if (hostpos->username != NULL)
+            pg->hosts[off].username = GNUNET_strdup (hostpos->username);
+          pg->hosts[off].sshport = hostpos->port;
+          hostpos = hostpos->next;
+          off++;
+        }
+
+      if (off == 0)
+        {
+          pg->hosts = NULL;
+        }
+      hostcnt = off;
+      minport = 0;
+      pg->num_hosts = off;
+
+#if NO_LL
       off = 2;
       /* skip leading spaces */
-      while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
+      while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
         hostnames++;
       rpos = hostnames;
       while ('\0' != *rpos)
         {
-          if (isspace ( (unsigned char) *rpos))
+          if (isspace ((unsigned char) *rpos))
             off++;
           rpos++;
         }
@@ -3039,7 +4208,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       pos = start;
       while ('\0' != *pos)
         {
-          if (isspace ( (unsigned char) *pos))
+          if (isspace ((unsigned char) *pos))
             {
               *pos = '\0';
               if (strlen (start) > 0)
@@ -3064,6 +4233,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
         }
       hostcnt = off;
       minport = 0;              /* make gcc happy */
+#endif
     }
   else
     {
@@ -3075,18 +4245,18 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       if (hostcnt > 0)
         {
           hostname = pg->hosts[off % hostcnt].hostname;
-          pcfg = make_config (cfg, 
-                             &pg->hosts[off % hostcnt].minport,
-                             &upnum,
-                             hostname);
+          username = pg->hosts[off % hostcnt].username;
+          sshport = pg->hosts[off % hostcnt].sshport;
+          pcfg = make_config (cfg,
+                              &pg->hosts[off % hostcnt].minport,
+                              &upnum, hostname, &fdnum);
         }
       else
         {
           hostname = NULL;
-          pcfg = make_config (cfg,
-                             &minport,
-                             &upnum,
-                             hostname);
+          username = NULL;
+          sshport = 0;
+          pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
         }
 
       if (NULL == pcfg)
@@ -3102,38 +4272,42 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
                                                  &baseservicehome))
         {
-         GNUNET_asprintf (&newservicehome,
-                          "%s/%d/", baseservicehome, off);
-         GNUNET_free (baseservicehome);
+          GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
+          GNUNET_free (baseservicehome);
         }
       else
         {
           tmpdir = getenv ("TMPDIR");
           tmpdir = tmpdir ? tmpdir : "/tmp";
-         GNUNET_asprintf (&newservicehome,
-                          "%s/%s/%d/",
-                          tmpdir,
-                          "gnunet-testing-test-test", off);
+          GNUNET_asprintf (&newservicehome,
+                           "%s/%s/%d/",
+                           tmpdir, "gnunet-testing-test-test", off);
         }
       GNUNET_CONFIGURATION_set_value_string (pcfg,
                                              "PATHS",
                                              "SERVICEHOME", newservicehome);
       GNUNET_free (newservicehome);
       pg->peers[off].cfg = pcfg;
-      pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
-      pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
-      pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
+      pg->peers[off].allowed_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
+      pg->peers[off].connect_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
+      pg->peers[off].blacklisted_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
       pg->peers[off].pg = pg;
 
       pg->peers[off].internal_context.peer = &pg->peers[off];
       pg->peers[off].internal_context.timeout = timeout;
       pg->peers[off].internal_context.hostname = hostname;
+      pg->peers[off].internal_context.username = username;
+      pg->peers[off].internal_context.sshport = sshport;
       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
       pg->peers[off].internal_context.start_cb = cb;
       pg->peers[off].internal_context.start_cb_cls = cb_cls;
 
-      GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
+      GNUNET_SCHEDULER_add_now (&internal_start,
+                                &pg->peers[off].internal_context);
 
     }
   return pg;
@@ -3144,7 +4318,8 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
  * offsetting operation.
  */
 struct GNUNET_TESTING_Daemon *
-GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
+GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg,
+                           unsigned int position)
 {
   if (position < pg->total)
     return pg->peers[position].daemon;
@@ -3152,6 +4327,32 @@ GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int pos
     return NULL;
 }
 
+/*
+ * Get a daemon by peer identity, so callers can
+ * retrieve the daemon without knowing it's offset.
+ *
+ * @param pg the peer group to retrieve the daemon from
+ * @param peer_id the peer identity of the daemon to retrieve
+ *
+ * @return the daemon on success, or NULL if no such peer identity is found
+ */
+struct GNUNET_TESTING_Daemon *
+GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg,
+                                 struct GNUNET_PeerIdentity *peer_id)
+{
+  unsigned int i;
+
+  for (i = 0; i < pg->total; i++)
+    {
+      if (0 ==
+          memcmp (&pg->peers[i].daemon->id, peer_id,
+                  sizeof (struct GNUNET_PeerIdentity)))
+        return pg->peers[i].daemon;
+    }
+
+  return NULL;
+}
+
 /**
  * Prototype of a function that will be called when a
  * particular operation was completed the testing library.
@@ -3162,11 +4363,11 @@ GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int pos
  * @param d handle to the daemon that was restarted
  * @param emsg NULL on success
  */
-void restart_callback (void *cls,
-                       const struct GNUNET_PeerIdentity *id,
-                       const struct GNUNET_CONFIGURATION_Handle *cfg,
-                       struct GNUNET_TESTING_Daemon *d,
-                       const char *emsg)
+void
+restart_callback (void *cls,
+                  const struct GNUNET_PeerIdentity *id,
+                  const struct GNUNET_CONFIGURATION_Handle *cfg,
+                  struct GNUNET_TESTING_Daemon *d, const char *emsg)
 {
   struct RestartContext *restart_context = cls;
 
@@ -3181,13 +4382,16 @@ void restart_callback (void *cls,
 
   if (restart_context->peers_restarted == restart_context->peer_group->total)
     {
-      restart_context->callback(restart_context->callback_cls, NULL);
-      GNUNET_free(restart_context);
+      restart_context->callback (restart_context->callback_cls, NULL);
+      GNUNET_free (restart_context);
     }
-  else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
+  else if (restart_context->peers_restart_failed +
+           restart_context->peers_restarted ==
+           restart_context->peer_group->total)
     {
-      restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
-      GNUNET_free(restart_context);
+      restart_context->callback (restart_context->callback_cls,
+                                 "Failed to restart peers!");
+      GNUNET_free (restart_context);
     }
 
 }
@@ -3203,15 +4407,18 @@ void restart_callback (void *cls,
 void
 churn_stop_callback (void *cls, const char *emsg)
 {
-  struct ChurnContext *churn_ctx = cls;
+  struct ShutdownContext *shutdown_ctx = cls;
+  struct ChurnContext *churn_ctx = shutdown_ctx->cb_cls;
   unsigned int total_left;
   char *error_message;
 
   error_message = NULL;
+  shutdown_ctx->outstanding--;
+
   if (emsg != NULL)
     {
-      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-                "Churn stop callback failed with error `%s'\n", emsg);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Churn stop callback failed with error `%s'\n", emsg);
       churn_ctx->num_failed_stop++;
     }
   else
@@ -3220,84 +4427,86 @@ churn_stop_callback (void *cls, const char *emsg)
     }
 
 #if DEBUG_CHURN
-  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-            "Stopped peer, %d left.\n", 
-            churn_ctx->num_to_stop);
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Stopped peer, %d left.\n", churn_ctx->num_to_stop);
 #endif
-  total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
+  total_left =
+    (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
+    (churn_ctx->num_to_start - churn_ctx->num_failed_start);
 
   if (total_left == 0)
-  {
-    if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
-      {
-        GNUNET_asprintf(&error_message, 
-                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!", 
-                       churn_ctx->num_failed_start, 
-                       churn_ctx->num_failed_stop);
-      }
-    churn_ctx->cb(churn_ctx->cb_cls, error_message);
-    GNUNET_free_non_null(error_message);
-    GNUNET_free(churn_ctx);
-  }
+    {
+      if ((churn_ctx->num_failed_stop > 0)
+          || (churn_ctx->num_failed_start > 0))
+        {
+          GNUNET_asprintf (&error_message,
+                           "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
+                           churn_ctx->num_failed_start,
+                           churn_ctx->num_failed_stop);
+        }
+      churn_ctx->cb (churn_ctx->cb_cls, error_message);
+      GNUNET_free_non_null (error_message);
+      GNUNET_free (churn_ctx);
+      GNUNET_free (shutdown_ctx);
+    }
 }
 
 /**
- * Callback for informing us about a successful
- * or unsuccessful churn start call.
+ * Count the number of running peers.
  *
- * @param cls a ChurnContext
- * @param id the peer identity of the started peer
- * @param cfg the handle to the configuration of the peer
- * @param d handle to the daemon for the peer
- * @param emsg NULL on success, non-NULL on failure
+ * @param pg handle for the peer group
  *
+ * @return the number of currently running peers in the peer group
  */
-void
-churn_start_callback (void *cls,
-                      const struct GNUNET_PeerIdentity *id,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      struct GNUNET_TESTING_Daemon *d,
-                      const char *emsg)
+unsigned int
+GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
 {
-  struct ChurnContext *churn_ctx = cls;
-  unsigned int total_left;
-  char *error_message;
-
-  error_message = NULL;
-  if (emsg != NULL)
+  unsigned int i;
+  unsigned int running = 0;
+  for (i = 0; i < pg->total; i++)
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 
-                 "Churn stop callback failed with error `%s'\n",
-                 emsg);
-      churn_ctx->num_failed_start++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert (running != -1);
+          running++;
+        }
     }
+  return running;
+}
+
+/**
+ * Task to rate limit the number of outstanding peer shutdown
+ * requests.  This is necessary for making sure we don't do
+ * too many ssh connections at once, but is generally nicer
+ * to any system as well (graduated task starts, as opposed
+ * to calling gnunet-arm N times all at once).
+ */
+static void
+schedule_churn_shutdown_task (void *cls,
+                              const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerShutdownContext *peer_shutdown_ctx = cls;
+  struct ShutdownContext *shutdown_ctx;
+
+  GNUNET_assert (peer_shutdown_ctx != NULL);
+  shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
+  GNUNET_assert (shutdown_ctx != NULL);
+
+  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_churn_shutdown_task,
+                                  peer_shutdown_ctx);
   else
     {
-      churn_ctx->num_to_start--;
+      shutdown_ctx->outstanding++;
+      GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
+                                  shutdown_ctx->timeout, shutdown_ctx->cb,
+                                  shutdown_ctx, GNUNET_NO, GNUNET_YES);
+      GNUNET_free (peer_shutdown_ctx);
     }
-  
-#if DEBUG_CHURN
-  GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
-            "Started peer, %d left.\n", 
-            churn_ctx->num_to_start);
-#endif
-
-  total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
-
-  if (total_left == 0)
-  {
-    if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
-      GNUNET_asprintf(&error_message, 
-                     "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!", 
-                     churn_ctx->num_failed_start,
-                     churn_ctx->num_failed_stop);
-    churn_ctx->cb(churn_ctx->cb_cls, error_message);
-    GNUNET_free_non_null(error_message);
-    GNUNET_free(churn_ctx);
-  }
 }
 
-
 /**
  * Simulate churn by stopping some peers (and possibly
  * re-starting others if churn is called multiple times).  This
@@ -3327,6 +4536,11 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
                               void *cb_cls)
 {
   struct ChurnContext *churn_ctx;
+  struct ShutdownContext *shutdown_ctx;
+  struct PeerShutdownContext *peer_shutdown_ctx;
+  struct PeerRestartContext *peer_restart_ctx;
+  struct ChurnRestartContext *churn_startup_ctx;
+
   unsigned int running;
   unsigned int stopped;
   unsigned int total_running;
@@ -3340,57 +4554,61 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
   running = 0;
   stopped = 0;
 
-  if ((von == 0) && (voff == 0)) /* No peers at all? */
+  if ((von == 0) && (voff == 0))        /* No peers at all? */
     {
-      cb(cb_cls, NULL);
+      cb (cb_cls, NULL);
       return;
     }
 
   for (i = 0; i < pg->total; i++)
-  {
-    if (pg->peers[i].daemon->running == GNUNET_YES)
-    {
-      GNUNET_assert(running != -1);
-      running++;
-    }
-    else
     {
-      GNUNET_assert(stopped != -1);
-      stopped++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert (running != -1);
+          running++;
+        }
+      else
+        {
+          GNUNET_assert (stopped != -1);
+          stopped++;
+        }
     }
-  }
 
   if (voff > running)
-  {
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
-    cb(cb_cls, "Trying to stop more peers than are currently running!");
-    return;
-  }
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Trying to stop more peers than are currently running!\n");
+      cb (cb_cls, "Trying to stop more peers than are currently running!");
+      return;
+    }
 
   if (von > stopped)
-  {
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
-    cb(cb_cls, "Trying to start more peers than are currently stopped!");
-    return;
-  }
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Trying to start more peers than are currently stopped!\n");
+      cb (cb_cls, "Trying to start more peers than are currently stopped!");
+      return;
+    }
 
-  churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
+  churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
 
   running_arr = NULL;
   if (running > 0)
-    running_arr = GNUNET_malloc(running * sizeof(unsigned int));
+    running_arr = GNUNET_malloc (running * sizeof (unsigned int));
 
   stopped_arr = NULL;
   if (stopped > 0)
-    stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
+    stopped_arr = GNUNET_malloc (stopped * sizeof (unsigned int));
 
   running_permute = NULL;
   stopped_permute = NULL;
 
   if (running > 0)
-    running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
+    running_permute =
+      GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, running);
   if (stopped > 0)
-    stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
+    stopped_permute =
+      GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, stopped);
 
   total_running = running;
   total_stopped = stopped;
@@ -3400,48 +4618,83 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
   churn_ctx->num_to_start = von;
   churn_ctx->num_to_stop = voff;
   churn_ctx->cb = cb;
-  churn_ctx->cb_cls = cb_cls;  
+  churn_ctx->cb_cls = cb_cls;
 
   for (i = 0; i < pg->total; i++)
-  {
-    if (pg->peers[i].daemon->running == GNUNET_YES)
     {
-      GNUNET_assert((running_arr != NULL) && (total_running > running));
-      running_arr[running] = i;
-      running++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert ((running_arr != NULL) && (total_running > running));
+          running_arr[running] = i;
+          running++;
+        }
+      else
+        {
+          GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped));
+          stopped_arr[stopped] = i;
+          stopped++;
+        }
     }
-    else
+
+  GNUNET_assert (running >= voff);
+  if (voff > 0)
     {
-      GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
-      stopped_arr[stopped] = i;
-      stopped++;
+      shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+      shutdown_ctx->cb = &churn_stop_callback;
+      shutdown_ctx->cb_cls = churn_ctx;
+      shutdown_ctx->total_peers = voff;
+      shutdown_ctx->timeout = timeout;
     }
-  }
 
   for (i = 0; i < voff; i++)
-  {
+    {
 #if DEBUG_CHURN
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n",
+                  running_permute[i]);
 #endif
-    GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
-                               timeout, 
-                               &churn_stop_callback, churn_ctx, 
-                               GNUNET_NO, GNUNET_YES);
-  }
+      GNUNET_assert (running_arr != NULL);
+      peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
+      peer_shutdown_ctx->daemon =
+        pg->peers[running_arr[running_permute[i]]].daemon;
+      peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
+      GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task,
+                                peer_shutdown_ctx);
+
+      /*
+         GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
+         timeout, 
+         &churn_stop_callback, churn_ctx, 
+         GNUNET_NO, GNUNET_YES); */
+    }
 
+  GNUNET_assert (stopped >= von);
+  if (von > 0)
+    {
+      churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
+      churn_startup_ctx->churn_ctx = churn_ctx;
+      churn_startup_ctx->timeout = timeout;
+    }
   for (i = 0; i < von; i++)
     {
 #if DEBUG_CHURN
-      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n",
+                  stopped_permute[i]);
 #endif
-      GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon, 
-                                         timeout, &churn_start_callback, churn_ctx);
-  }
+      GNUNET_assert (stopped_arr != NULL);
+      peer_restart_ctx = GNUNET_malloc (sizeof (struct PeerRestartContext));
+      peer_restart_ctx->churn_restart_ctx = churn_startup_ctx;
+      peer_restart_ctx->daemon =
+        pg->peers[stopped_arr[stopped_permute[i]]].daemon;
+      GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx);
+      /*
+         GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon, 
+         timeout, &churn_start_callback, churn_ctx); */
+    }
 
-  GNUNET_free_non_null(running_arr);
-  GNUNET_free_non_null(stopped_arr);
-  GNUNET_free_non_null(running_permute);
-  GNUNET_free_non_null(stopped_permute);
+  GNUNET_free_non_null (running_arr);
+  GNUNET_free_non_null (stopped_arr);
+  GNUNET_free_non_null (running_permute);
+  GNUNET_free_non_null (stopped_permute);
 }
 
 
@@ -3453,14 +4706,16 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
  * @param callback_cls closure for the callback function
  */
 void
-GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyCompletion callback, void *callback_cls)
+GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
+                                GNUNET_TESTING_NotifyCompletion callback,
+                                void *callback_cls)
 {
   struct RestartContext *restart_context;
   unsigned int off;
 
   if (pg->total > 0)
     {
-      restart_context = GNUNET_malloc(sizeof(struct RestartContext));
+      restart_context = GNUNET_malloc (sizeof (struct RestartContext));
       restart_context->peer_group = pg;
       restart_context->peers_restarted = 0;
       restart_context->callback = callback;
@@ -3468,7 +4723,8 @@ GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TEST
 
       for (off = 0; off < pg->total; off++)
         {
-          GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
+          GNUNET_TESTING_daemon_restart (pg->peers[off].daemon,
+                                         &restart_callback, restart_context);
         }
     }
 }
@@ -3484,41 +4740,47 @@ GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TEST
  * @param cb_cls closure for cb
  */
 void
-GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg, 
-                            unsigned int offset,
-                            int desired_status,
-                            struct GNUNET_TIME_Relative timeout,
-                            GNUNET_TESTING_NotifyCompletion cb,
-                            void *cb_cls)
+GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
+                             unsigned int offset,
+                             int desired_status,
+                             struct GNUNET_TIME_Relative timeout,
+                             GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
 {
+  struct ShutdownContext *shutdown_ctx;
+  struct ChurnRestartContext *startup_ctx;
   struct ChurnContext *churn_ctx;
 
   if (GNUNET_NO == desired_status)
     {
       if (NULL != pg->peers[offset].daemon)
-       {
-         churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
-         churn_ctx->num_to_start = 0;
-         churn_ctx->num_to_stop = 1;
-         churn_ctx->cb = cb;
-         churn_ctx->cb_cls = cb_cls;  
-         GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon, 
-                                    timeout, &churn_stop_callback, churn_ctx, 
-                                    GNUNET_NO, GNUNET_YES);     
-       }
+        {
+          shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+          churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
+          churn_ctx->num_to_start = 0;
+          churn_ctx->num_to_stop = 1;
+          churn_ctx->cb = cb;
+          churn_ctx->cb_cls = cb_cls;
+          shutdown_ctx->cb_cls = churn_ctx;
+          GNUNET_TESTING_daemon_stop (pg->peers[offset].daemon,
+                                      timeout, &churn_stop_callback,
+                                      shutdown_ctx, GNUNET_NO, GNUNET_YES);
+        }
     }
   else if (GNUNET_YES == desired_status)
     {
       if (NULL == pg->peers[offset].daemon)
-       {
-         churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
-         churn_ctx->num_to_start = 1;
-         churn_ctx->num_to_stop = 0;
-         churn_ctx->cb = cb;
-         churn_ctx->cb_cls = cb_cls;  
-         GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon, 
-                                             timeout, &churn_start_callback, churn_ctx);
-       }
+        {
+          startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
+          churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
+          churn_ctx->num_to_start = 1;
+          churn_ctx->num_to_stop = 0;
+          churn_ctx->cb = cb;
+          churn_ctx->cb_cls = cb_cls;
+          startup_ctx->churn_ctx = churn_ctx;
+          GNUNET_TESTING_daemon_start_stopped (pg->peers[offset].daemon,
+                                               timeout, &churn_start_callback,
+                                               startup_ctx);
+        }
     }
   else
     GNUNET_break (0);
@@ -3531,11 +4793,12 @@ GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
  * @param cls closure (struct ShutdownContext)
  * @param emsg NULL on success
  */
-void internal_shutdown_callback (void *cls,
-                        const char *emsg)
+void
+internal_shutdown_callback (void *cls, const char *emsg)
 {
   struct ShutdownContext *shutdown_ctx = cls;
 
+  shutdown_ctx->outstanding--;
   if (emsg == NULL)
     {
       shutdown_ctx->peers_down++;
@@ -3545,13 +4808,50 @@ void internal_shutdown_callback (void *cls,
       shutdown_ctx->peers_failed++;
     }
 
-  if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
+  if ((shutdown_ctx->cb != NULL)
+      && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed ==
+          shutdown_ctx->total_peers))
     {
       if (shutdown_ctx->peers_failed > 0)
-        shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
+        shutdown_ctx->cb (shutdown_ctx->cb_cls,
+                          "Not all peers successfully shut down!");
       else
-        shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
-      GNUNET_free(shutdown_ctx);
+        shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
+      GNUNET_free (shutdown_ctx);
+    }
+}
+
+
+/**
+ * Task to rate limit the number of outstanding peer shutdown
+ * requests.  This is necessary for making sure we don't do
+ * too many ssh connections at once, but is generally nicer
+ * to any system as well (graduated task starts, as opposed
+ * to calling gnunet-arm N times all at once).
+ */
+static void
+schedule_shutdown_task (void *cls,
+                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerShutdownContext *peer_shutdown_ctx = cls;
+  struct ShutdownContext *shutdown_ctx;
+
+  GNUNET_assert (peer_shutdown_ctx != NULL);
+  shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
+  GNUNET_assert (shutdown_ctx != NULL);
+
+  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_shutdown_task, peer_shutdown_ctx);
+  else
+    {
+      shutdown_ctx->outstanding++;
+      GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
+                                  shutdown_ctx->timeout,
+                                  &internal_shutdown_callback, shutdown_ctx,
+                                  GNUNET_YES, GNUNET_NO);
+      GNUNET_free (peer_shutdown_ctx);
     }
 }
 
@@ -3564,50 +4864,48 @@ void internal_shutdown_callback (void *cls,
  * @param cb_cls closure for cb
  */
 void
-GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg, 
-                            struct GNUNET_TIME_Relative timeout,
-                            GNUNET_TESTING_NotifyCompletion cb,
-                             void *cb_cls)
+GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
+                             struct GNUNET_TIME_Relative timeout,
+                             GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
 {
   unsigned int off;
   struct ShutdownContext *shutdown_ctx;
-  GNUNET_TESTING_NotifyCompletion shutdown_cb;
-  void *shutdown_cb_cls;
+  struct PeerShutdownContext *peer_shutdown_ctx;
 
-  GNUNET_assert(pg->total > 0);
+  GNUNET_assert (pg->total > 0);
 
-  shutdown_cb = NULL;
-  shutdown_ctx = NULL;
-
-  if (cb != NULL)
-    {
-      shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
-      shutdown_ctx->cb = cb;
-      shutdown_ctx->cb_cls = cb_cls;
-      shutdown_ctx->total_peers = pg->total;
-      shutdown_cb = &internal_shutdown_callback;
-      shutdown_cb_cls = cb_cls;
-    }
+  shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+  shutdown_ctx->cb = cb;
+  shutdown_ctx->cb_cls = cb_cls;
+  shutdown_ctx->total_peers = pg->total;
+  shutdown_ctx->timeout = timeout;
+  /* shtudown_ctx->outstanding = 0; */
 
   for (off = 0; off < pg->total; off++)
     {
-      GNUNET_assert(NULL != pg->peers[off].daemon);
-      GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
+      GNUNET_assert (NULL != pg->peers[off].daemon);
+      peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
+      peer_shutdown_ctx->daemon = pg->peers[off].daemon;
+      peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
+      GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
+      //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
       if (NULL != pg->peers[off].cfg)
         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
       if (pg->peers[off].allowed_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
       if (pg->peers[off].connect_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
       if (pg->peers[off].blacklisted_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                               peers[off].blacklisted_peers);
     }
   GNUNET_free (pg->peers);
-  if (NULL != pg->hosts)
+  for (off = 0; off < pg->num_hosts; off++)
     {
-      GNUNET_free (pg->hosts[0].hostname);
-      GNUNET_free (pg->hosts);
+      GNUNET_free (pg->hosts[off].hostname);
+      GNUNET_free_non_null (pg->hosts[off].username);
     }
+  GNUNET_free_non_null (pg->hosts);
   GNUNET_free (pg);
 }