fix
[oweals/gnunet.git] / src / testing / testing_group.c
index 98610a28b0f7a6c065ff8c9811e1c9b52e51269d..a09460844476a3a4402250982459b6fdd1a2a5b6 100644 (file)
@@ -4,7 +4,7 @@
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
 
       GNUnet is free software; you can redistribute it and/or modify
       it under the terms of the GNU General Public License as published
-      by the Free Software Foundation; either version 2, or (at your
+      by the Free Software Foundation; either version 3, or (at your
       option) any later version.
 
       GNUnet is distributed in the hope that it will be useful, but
       option) any later version.
 
       GNUnet is distributed in the hope that it will be useful, but
 /**
  * @file testing/testing_group.c
  * @brief convenience API for writing testcases for GNUnet
 /**
  * @file testing/testing_group.c
  * @brief convenience API for writing testcases for GNUnet
+ * @author Nathan Evans
  * @author Christian Grothoff
  *
  * @author Christian Grothoff
  *
- * FIXME: have connection processor functions take a cls argument
- *        which specifies where to write the connection information
- *        instead of assuming it's certain peergroup places. (maybe?)
- * FIXME: create static struct which contains the TOPOLOGY enum, the
- *        associated string, and the function used to create it.
- *        Then replace the create_X calls with topology_struct[i][2]
- *        or something. (Store function pointers instead of using
- *        switch statements)
  */
 #include "platform.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
  */
 #include "platform.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
+#include "gnunet_core_service.h"
 
 #define VERBOSE_TESTING GNUNET_NO
 
 
 #define VERBOSE_TESTING GNUNET_NO
 
-#define VERBOSE_TOPOLOGY GNUNET_NO
+#define VERBOSE_TOPOLOGY GNUNET_YES
 
 #define DEBUG_CHURN GNUNET_NO
 
 
 #define DEBUG_CHURN GNUNET_NO
 
  * conflict with the port range for "local" ports (client apps; see
  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  */
  * conflict with the port range for "local" ports (client apps; see
  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  */
-#define HIGH_PORT 32000
+#define HIGH_PORT 56000
 
 
-#define MAX_OUTSTANDING_CONNECTIONS 50
+#define MAX_OUTSTANDING_CONNECTIONS 40
 
 
-#define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
+#define MAX_CONCURRENT_HOSTKEYS 10
 
 
-#define CONNECT_ATTEMPTS 8
+#define MAX_CONCURRENT_STARTING 10
+
+#define MAX_CONCURRENT_SHUTDOWN 10
+
+#define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 200)
+
+#define CONNECT_ATTEMPTS 21
 
 /**
  * Prototype of a function called whenever two peers would be connected
  * in a certain topology.
  */
 
 /**
  * Prototype of a function called whenever two peers would be connected
  * in a certain topology.
  */
-typedef int (*GNUNET_TESTING_ConnectionProcessor)
-(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second);
+typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
+                                                            GNUNET_TESTING_PeerGroup
+                                                            * pg,
+                                                            unsigned int
+                                                            first,
+                                                            unsigned int
+                                                            second);
+
 
 /**
 
 /**
- * Strings representing topologies in enum
+ * Context for handling churning a peer group
  */
  */
-static char * GNUNET_TESTING_TopologyStrings[] =
+struct ChurnContext
 {
   /**
 {
   /**
-   * A clique (everyone connected to everyone else).
+   * Callback used to notify of churning finished
    */
    */
-  "CLIQUE",
+  GNUNET_TESTING_NotifyCompletion cb;
 
   /**
 
   /**
-   * Small-world network (2d torus plus random links).
+   * Closure for callback
    */
    */
-  "SMALL_WORLD",
+  void *cb_cls;
 
   /**
 
   /**
-   * Small-world network (ring plus random links).
+   * Number of peers that still need to be started
    */
    */
-  "SMALL_WORLD_RING",
+  unsigned int num_to_start;
 
   /**
 
   /**
-   * Ring topology.
+   * Number of peers that still need to be stopped
    */
    */
-  "RING",
+  unsigned int num_to_stop;
 
   /**
 
   /**
-   * 2-d torus.
+   * Number of peers that failed to start
    */
    */
-  "2D_TORUS",
+  unsigned int num_failed_start;
 
   /**
 
   /**
-   * Random graph.
+   * Number of peers that failed to stop
    */
    */
-  "ERDOS_RENYI",
+  unsigned int num_failed_stop;
+};
 
 
+struct RestartContext
+{
   /**
   /**
-   * Certain percentage of peers are unable to communicate directly
-   * replicating NAT conditions
+   * The group of peers being restarted
    */
    */
-  "INTERNAT",
+  struct GNUNET_TESTING_PeerGroup *peer_group;
 
   /**
 
   /**
-   * Scale free topology.
+   * How many peers have been restarted thus far
    */
    */
-  "SCALE_FREE",
+  unsigned int peers_restarted;
 
   /**
 
   /**
-   * Straight line topology.
+   * How many peers got an error when restarting
    */
    */
-  "LINE",
+  unsigned int peers_restart_failed;
 
   /**
 
   /**
-   * All peers are disconnected.
+   * The function to call once all peers have been restarted
    */
    */
-  "NONE"
-};
+  GNUNET_TESTING_NotifyCompletion callback;
 
 
-/**
- * Options for connecting a topology as strings.
- */
-static char * GNUNET_TESTING_TopologyOptionStrings[] =
-{
   /**
   /**
-   * Try to connect all peers specified in the topology.
+   * Closure for callback function
    */
    */
-  "CONNECT_ALL",
+  void *callback_cls;
 
 
-  /**
-   * Choose a random subset of connections to create.
-   */
-  "CONNECT_RANDOM_SUBSET",
+};
 
 
-  /**
-   * Create at least X connections for each peer.
-   */
-  "CONNECT_MINIMUM",
 
 
+struct ShutdownContext
+{
   /**
   /**
-   * Using a depth first search, create one connection
-   * per peer.  If any are missed (graph disconnected)
-   * start over at those peers until all have at least one
-   * connection.
+   * Total peers to wait for
    */
    */
-  "CONNECT_DFS",
+  unsigned int total_peers;
 
   /**
 
   /**
-   * No options specified.
+   * Number of peers successfully shut down
    */
    */
-  "CONNECT_NONE"
-};
+  unsigned int peers_down;
 
 
-/**
- * Context for handling churning a peer group
- */
-struct ChurnContext
-{
   /**
   /**
-   * Callback used to notify of churning finished
+   * Number of peers failed to shut down
    */
    */
-  GNUNET_TESTING_NotifyCompletion cb;
+  unsigned int peers_failed;
 
   /**
 
   /**
-   * Closure for callback
+   * Number of peers we have started shutting
+   * down.  If too many, wait on them.
    */
    */
-  void *cb_cls;
+  unsigned int outstanding;
 
   /**
 
   /**
-   * Number of peers that still need to be started
+   * Timeout for shutdown.
    */
    */
-  unsigned int num_to_start;
+  struct GNUNET_TIME_Relative timeout;
 
   /**
 
   /**
-   * Number of peers that still need to be stopped
+   * Callback to call when all peers either
+   * shutdown or failed to shutdown
    */
    */
-  unsigned int num_to_stop;
-  /**
-   * Number of peers that failed to start
-   */
-  unsigned int num_failed_start;
+  GNUNET_TESTING_NotifyCompletion cb;
 
   /**
 
   /**
-   * Number of peers that failed to stop
+   * Closure for cb
    */
    */
-  unsigned int num_failed_stop;
+  void *cb_cls;
 };
 
 };
 
-struct RestartContext
+/**
+ * Individual shutdown context for a particular peer.
+ */
+struct PeerShutdownContext
 {
   /**
 {
   /**
-   * The group of peers being restarted
+   * Pointer to the high level shutdown context.
    */
    */
-  struct GNUNET_TESTING_PeerGroup *peer_group;
+  struct ShutdownContext *shutdown_ctx;
 
   /**
 
   /**
-   * How many peers have been restarted thus far
+   * The daemon handle for the peer to shut down.
    */
    */
-  unsigned int peers_restarted;
+  struct GNUNET_TESTING_Daemon *daemon;
+};
 
 
+/**
+ * Individual shutdown context for a particular peer.
+ */
+struct PeerRestartContext
+{
   /**
   /**
-   * How many peers got an error when restarting
+   * Pointer to the high level restart context.
    */
    */
-  unsigned int peers_restart_failed;
+  struct ChurnRestartContext *churn_restart_ctx;
 
   /**
 
   /**
-   * The function to call once all peers have been restarted
+   * The daemon handle for the peer to shut down.
    */
    */
-  GNUNET_TESTING_NotifyCompletion callback;
-
-  /**
-   * Closure for callback function
-   */
-  void *callback_cls;
-
+  struct GNUNET_TESTING_Daemon *daemon;
 };
 
 };
 
+
 struct CreateTopologyContext
 {
 
 struct CreateTopologyContext
 {
 
@@ -256,6 +245,74 @@ struct PeerConnection
 };
 #endif
 
 };
 #endif
 
+struct InternalStartContext
+{
+  /**
+   * Pointer to peerdata
+   */
+  struct PeerData *peer;
+
+  /**
+   * Timeout for peer startup
+   */
+  struct GNUNET_TIME_Relative timeout;
+
+  /**
+   * Client callback for hostkey notification
+   */
+  GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
+
+  /**
+   * Closure for hostkey_callback
+   */
+  void *hostkey_cls;
+
+  /**
+   * Client callback for peer start notification
+   */
+  GNUNET_TESTING_NotifyDaemonRunning start_cb;
+
+  /**
+   * Closure for cb
+   */
+  void *start_cb_cls;
+
+  /**
+   * Hostname, where to start the peer
+   */
+  const char *hostname;
+
+  /**
+   * Username to use when connecting to the
+   * host via ssh.
+   */
+  const char *username;
+
+  /**
+   * Port to use for ssh.
+   */
+  uint16_t sshport;
+
+};
+
+struct ChurnRestartContext
+{
+  /**
+   * Number of restarts currently in flight.
+   */
+  unsigned int outstanding;
+
+  /**
+   * Handle to the underlying churn context.
+   */
+  struct ChurnContext *churn_ctx;
+
+  /**
+   * How long to allow the operation to take.
+   */
+  struct GNUNET_TIME_Relative timeout;
+};
+
 /**
  * Data we keep per peer.
  */
 /**
  * Data we keep per peer.
  */
@@ -279,10 +336,6 @@ struct PeerData
    */
   struct GNUNET_TESTING_PeerGroup *pg;
 
    */
   struct GNUNET_TESTING_PeerGroup *pg;
 
-  /**
-   * Linked list of peer connections (pointers)
-   */
-  //struct PeerConnection *connected_peers;
   /**
    * Hash map of allowed peer connections (F2F created topology)
    */
   /**
    * Hash map of allowed peer connections (F2F created topology)
    */
@@ -308,11 +361,17 @@ struct PeerData
    * creating any topology so the count is valid once finished.
    */
   int num_connections;
    * creating any topology so the count is valid once finished.
    */
   int num_connections;
+
+  /**
+   * Context to keep track of peers being started, to
+   * stagger hostkey generation and peer startup.
+   */
+  struct InternalStartContext internal_context;
 };
 
 
 /**
 };
 
 
 /**
- * Data we keep per host.
+ * Linked list of per-host data.
  */
 struct HostData
 {
  */
 struct HostData
 {
@@ -321,6 +380,16 @@ struct HostData
    */
   char *hostname;
 
    */
   char *hostname;
 
+  /**
+   * SSH username to use when connecting to this host.
+   */
+  char *username;
+
+  /**
+   * SSH port to use when connecting to this host.
+   */
+  uint16_t sshport;
+
   /**
    * Lowest port that we have not yet used
    * for GNUnet.
   /**
    * Lowest port that we have not yet used
    * for GNUnet.
@@ -328,17 +397,93 @@ struct HostData
   uint16_t minport;
 };
 
   uint16_t minport;
 };
 
+struct TopologyIterateContext
+{
+  /**
+   * Callback for notifying of two connected peers.
+   */
+  GNUNET_TESTING_NotifyTopology topology_cb;
+
+  /**
+   * Closure for topology_cb
+   */
+  void *cls;
+
+  /**
+   * Number of peers currently connected to.
+   */
+  unsigned int connected;
+
+  /**
+   * Number of peers we have finished iterating.
+   */
+  unsigned int completed;
+
+  /**
+   * Number of peers total.
+   */
+  unsigned int total;
+};
+
+struct StatsIterateContext
+{
+  /**
+   * Continuation to call once all stats information has been retrieved.
+   */
+  GNUNET_STATISTICS_Callback cont;
+
+  /**
+   * Proc function to call on each value received.
+   */
+  GNUNET_TESTING_STATISTICS_Iterator proc;
+
+  /**
+   * Closure for topology_cb
+   */
+  void *cls;
+
+  /**
+   * Number of peers currently connected to.
+   */
+  unsigned int connected;
+
+  /**
+   * Number of peers we have finished iterating.
+   */
+  unsigned int completed;
+
+  /**
+   * Number of peers total.
+   */
+  unsigned int total;
+};
+
+struct CoreContext
+{
+  void *iter_context;
+  struct GNUNET_TESTING_Daemon *daemon;
+};
+
+struct StatsCoreContext
+{
+  void *iter_context;
+  struct GNUNET_TESTING_Daemon *daemon;
+  /**
+   * Handle to the statistics service.
+   */
+  struct GNUNET_STATISTICS_Handle *stats_handle;
+
+  /**
+   * Handle for getting statistics.
+   */
+  struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
+};
 
 /**
  * Handle to a group of GNUnet peers.
  */
 struct GNUNET_TESTING_PeerGroup
 {
 
 /**
  * Handle to a group of GNUnet peers.
  */
 struct GNUNET_TESTING_PeerGroup
 {
-  /**
-   * Our scheduler.
-   */
-  struct GNUNET_SCHEDULER_Handle *sched;
-
   /**
    * Configuration template.
    */
   /**
    * Configuration template.
    */
@@ -347,12 +492,12 @@ struct GNUNET_TESTING_PeerGroup
   /**
    * Function to call on each started daemon.
    */
   /**
    * Function to call on each started daemon.
    */
-  GNUNET_TESTING_NotifyDaemonRunning cb;
+  //GNUNET_TESTING_NotifyDaemonRunning cb;
 
   /**
    * Closure for cb.
    */
 
   /**
    * Closure for cb.
    */
-  void *cb_cls;
+  //void *cb_cls;
 
   /*
    * Function to call on each topology connection created
 
   /*
    * Function to call on each topology connection created
@@ -365,11 +510,15 @@ struct GNUNET_TESTING_PeerGroup
   void *notify_connection_cls;
 
   /**
   void *notify_connection_cls;
 
   /**
-   * NULL-terminated array of information about
-   * hosts.
+   * Array of information about hosts.
    */
   struct HostData *hosts;
 
    */
   struct HostData *hosts;
 
+  /**
+   * Number of hosts (size of HostData)
+   */
+  unsigned int num_hosts;
+
   /**
    * Array of "total" peers.
    */
   /**
    * Array of "total" peers.
    */
@@ -384,6 +533,72 @@ struct GNUNET_TESTING_PeerGroup
    * At what time should we fail the peer startup process?
    */
   struct GNUNET_TIME_Absolute max_timeout;
    * At what time should we fail the peer startup process?
    */
   struct GNUNET_TIME_Absolute max_timeout;
+
+  /**
+   * How many peers are being started right now?
+   */
+  unsigned int starting;
+
+  /**
+   * How many peers have already been started?
+   */
+  unsigned int started;
+};
+
+struct UpdateContext
+{
+  struct GNUNET_CONFIGURATION_Handle *ret;
+  const struct GNUNET_CONFIGURATION_Handle *orig;
+  const char *hostname;
+  unsigned int nport;
+  unsigned int upnum;
+  unsigned int fdnum;
+};
+
+struct ConnectTopologyContext
+{
+  /**
+   * How many connections are left to create.
+   */
+  unsigned int remaining_connections;
+
+  /**
+   * Handle to group of peers.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
+  /**
+   * Temp value set for each iteration.
+   */
+  struct PeerData *first;
+
+  /**
+   * Notification that all peers are connected.
+   */
+  GNUNET_TESTING_NotifyCompletion notify_connections_done;
+
+  /**
+   * Closure for notify.
+   */
+  void *notify_cls;
+};
+
+struct ConnectContext
+{
+  /**
+   * Peer to connect second to.
+   */
+  struct GNUNET_TESTING_Daemon *first;
+
+  /**
+   * Peer to connect first to.
+   */
+  struct GNUNET_TESTING_Daemon *second;
+
+  /**
+   * Higher level topology connection context.
+   */
+  struct ConnectTopologyContext *ct_ctx;
 };
 
 /**
 };
 
 /**
@@ -393,11 +608,10 @@ struct GNUNET_TESTING_PeerGroup
  * @param hash set to uid (extended with zeros)
  */
 static void
  * @param hash set to uid (extended with zeros)
  */
 static void
-hash_from_uid (uint32_t uid,
-               GNUNET_HashCode *hash)
+hash_from_uid (uint32_t uid, GNUNET_HashCode * hash)
 {
 {
-  memset (hash, 0, sizeof(GNUNET_HashCode));
-  *((uint32_t*)hash) = uid;
+  memset (hash, 0, sizeof (GNUNET_HashCode));
+  *((uint32_t *) hash) = uid;
 }
 
 /**
 }
 
 /**
@@ -407,29 +621,11 @@ hash_from_uid (uint32_t uid,
  * @param hash set to uid (extended with zeros)
  */
 static void
  * @param hash set to uid (extended with zeros)
  */
 static void
-uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
+uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 {
 {
-  memcpy (uid, hash, sizeof(uint32_t));
+  memcpy (uid, hash, sizeof (uint32_t));
 }
 
 }
 
-struct UpdateContext
-{
-  struct GNUNET_CONFIGURATION_Handle *ret;
-  const char *hostname;
-  unsigned int nport;
-  unsigned int upnum;
-};
-
-
-struct ConnectContext
-{
-  struct GNUNET_TESTING_Daemon *first;
-
-  struct GNUNET_TESTING_Daemon *second;
-
-  struct GNUNET_TESTING_PeerGroup *pg;
-};
-
 /**
  * Number of connects we are waiting on, allows us to rate limit
  * connect attempts.
 /**
  * Number of connects we are waiting on, allows us to rate limit
  * connect attempts.
@@ -446,62 +642,154 @@ static int outstanding_connects;
  *         known topology, GNUNET_NO if not
  */
 int
  *         known topology, GNUNET_NO if not
  */
 int
-GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
+GNUNET_TESTING_topology_get (enum GNUNET_TESTING_Topology *topology,
+                             const char *topology_string)
 {
 {
-  int found = 0;
-  int curr = 0;
+  /**
+   * Strings representing topologies in enum
+   */
+  static const char *topology_strings[] = {
+      /**
+       * A clique (everyone connected to everyone else).
+       */
+    "CLIQUE",
+
+      /**
+       * Small-world network (2d torus plus random links).
+       */
+    "SMALL_WORLD",
+
+      /**
+       * Small-world network (ring plus random links).
+       */
+    "SMALL_WORLD_RING",
+
+      /**
+       * Ring topology.
+       */
+    "RING",
+
+      /**
+       * 2-d torus.
+       */
+    "2D_TORUS",
+
+      /**
+       * Random graph.
+       */
+    "ERDOS_RENYI",
+
+      /**
+       * Certain percentage of peers are unable to communicate directly
+       * replicating NAT conditions
+       */
+    "INTERNAT",
+
+      /**
+       * Scale free topology.
+       */
+    "SCALE_FREE",
+
+      /**
+       * Straight line topology.
+       */
+    "LINE",
+
+      /**
+       * All peers are disconnected.
+       */
+    "NONE",
+
+    NULL
+  };
 
 
+  int curr = 0;
   if (topology_string == NULL)
     return GNUNET_NO;
   if (topology_string == NULL)
     return GNUNET_NO;
-
-  do
-  {
-    if (strcmp(GNUNET_TESTING_TopologyStrings[curr], topology_string) == 0)
+  while (topology_strings[curr] != NULL)
     {
     {
-      found = GNUNET_YES;
-      break;
+      if (strcasecmp (topology_strings[curr], topology_string) == 0)
+        {
+          *topology = curr;
+          return GNUNET_YES;
+        }
+      curr++;
     }
     }
-    curr++;
-  } while (strcmp(GNUNET_TESTING_TopologyStrings[curr], "NONE") != 0);
-  *topology = curr;
-  if (found)
-    return GNUNET_YES;
-  else
-    return GNUNET_NO;
+  *topology = GNUNET_TESTING_TOPOLOGY_NONE;
+  return GNUNET_NO;
 }
 
 }
 
+
 /**
  * Get connect topology option from string input.
  *
 /**
  * Get connect topology option from string input.
  *
- * @param topology where to write the retrieved topology
+ * @param topology_option where to write the retrieved topology
  * @param topology_string The string to attempt to
  *        get a configuration value from
  * @return GNUNET_YES if string matched a known
  *         topology option, GNUNET_NO if not
  */
 int
  * @param topology_string The string to attempt to
  *        get a configuration value from
  * @return GNUNET_YES if string matched a known
  *         topology option, GNUNET_NO if not
  */
 int
-GNUNET_TESTING_topology_option_get(enum GNUNET_TESTING_TopologyOption *topology, char * topology_string)
+GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption
+                                    *topology_option,
+                                    const char *topology_string)
 {
 {
-  int found = 0;
+  /**
+   * Options for connecting a topology as strings.
+   */
+  static const char *topology_option_strings[] = {
+      /**
+       * Try to connect all peers specified in the topology.
+       */
+    "CONNECT_ALL",
+
+      /**
+       * Choose a random subset of connections to create.
+       */
+    "CONNECT_RANDOM_SUBSET",
+
+      /**
+       * Create at least X connections for each peer.
+       */
+    "CONNECT_MINIMUM",
+
+      /**
+       * Using a depth first search, create one connection
+       * per peer.  If any are missed (graph disconnected)
+       * start over at those peers until all have at least one
+       * connection.
+       */
+    "CONNECT_DFS",
+
+      /**
+       * Find the N closest peers to each allowed peer in the
+       * topology and make sure a connection to those peers
+       * exists in the connect topology.
+       */
+    "CONNECT_CLOSEST",
+
+      /**
+       * No options specified.
+       */
+    "CONNECT_NONE",
+
+    NULL
+  };
   int curr = 0;
 
   if (topology_string == NULL)
     return GNUNET_NO;
   int curr = 0;
 
   if (topology_string == NULL)
     return GNUNET_NO;
-
-  do
-  {
-    if (strcmp(GNUNET_TESTING_TopologyOptionStrings[curr], topology_string) == 0)
+  while (NULL != topology_option_strings[curr])
     {
     {
-      found = GNUNET_YES;
-      break;
+      if (strcasecmp (topology_option_strings[curr], topology_string) == 0)
+        {
+          *topology_option = curr;
+          return GNUNET_YES;
+        }
+      curr++;
     }
     }
-    curr++;
-  } while (strcmp(GNUNET_TESTING_TopologyOptionStrings[curr], "CONNECT_NONE") != 0);
-  *topology = curr;
-  if (found)
-    return GNUNET_YES;
-  else
-    return GNUNET_NO;
+  *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
+  return GNUNET_NO;
 }
 
 /**
 }
 
 /**
@@ -522,24 +810,53 @@ update_config (void *cls,
   unsigned int ival;
   char cval[12];
   char uval[128];
   unsigned int ival;
   char cval[12];
   char uval[128];
+  char *single_variable;
+  char *per_host_variable;
+  unsigned long long num_per_host;
 
   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
     {
 
   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
     {
-      if (ival != 0)
-       {
-         GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
-         value = cval;
-       }
+      GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
+      if ((ival != 0)
+          && (GNUNET_YES !=
+              GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
+                                                    single_variable)))
+        {
+          GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
+          value = cval;
+        }
+
+      GNUNET_free (single_variable);
     }
 
   if (0 == strcmp (option, "UNIXPATH"))
     {
     }
 
   if (0 == strcmp (option, "UNIXPATH"))
     {
-      GNUNET_snprintf (uval, 
-                      sizeof (uval),
-                      "/tmp/test-service-%s-%u", 
-                      section,
-                      ctx->upnum++);
-      value = uval;
+      GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
+      GNUNET_asprintf (&per_host_variable, "num_%s_per_host", section);
+      if (GNUNET_YES !=
+          GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
+                                                single_variable))
+        {
+          GNUNET_snprintf (uval,
+                           sizeof (uval),
+                           "/tmp/test-service-%s-%u", section, ctx->upnum++);
+          value = uval;
+        }
+      else if ((GNUNET_YES ==
+               GNUNET_CONFIGURATION_get_value_number (ctx->orig, "testing",
+                                                      per_host_variable,
+                                                      &num_per_host)) && (num_per_host > 0))
+
+        {
+          GNUNET_snprintf (uval,
+                           sizeof (uval),
+                           "/tmp/test-service-%s-%u",
+                           section, ctx->fdnum % num_per_host);
+          value = uval;
+        }
+      GNUNET_free (single_variable);
+      GNUNET_free (per_host_variable);
+
     }
 
   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
     }
 
   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
@@ -562,14 +879,15 @@ update_config (void *cls,
  *             port numbers that were used
  * @param upnum number to make unix domain socket names unique
  * @param hostname hostname of the controlling host, to allow control connections from
  *             port numbers that were used
  * @param upnum number to make unix domain socket names unique
  * @param hostname hostname of the controlling host, to allow control connections from
+ * @param fdnum number used to offset the unix domain socket for grouped processes
+ *              (such as statistics or peerinfo, which can be shared among others)
  *
  * @return new configuration, NULL on error
  */
 static struct GNUNET_CONFIGURATION_Handle *
  *
  * @return new configuration, NULL on error
  */
 static struct GNUNET_CONFIGURATION_Handle *
-make_config (const struct GNUNET_CONFIGURATION_Handle *cfg, 
-            uint16_t * port,
-            uint32_t * upnum,
-            const char *hostname)
+make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
+             uint16_t * port,
+             uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 {
   struct UpdateContext uc;
   uint16_t orig;
 {
   struct UpdateContext uc;
   uint16_t orig;
@@ -579,8 +897,10 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
   orig = *port;
   uc.nport = *port;
   uc.upnum = *upnum;
   orig = *port;
   uc.nport = *port;
   uc.upnum = *upnum;
+  uc.fdnum = *fdnum;
   uc.ret = GNUNET_CONFIGURATION_create ();
   uc.hostname = hostname;
   uc.ret = GNUNET_CONFIGURATION_create ();
   uc.hostname = hostname;
+  uc.orig = cfg;
 
   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
   if (uc.nport >= HIGH_PORT)
 
   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
   if (uc.nport >= HIGH_PORT)
@@ -590,12 +910,25 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
       return NULL;
     }
 
       return NULL;
     }
 
-  if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
+  if (GNUNET_CONFIGURATION_get_value_string
+      (cfg, "testing", "control_host", &control_host) == GNUNET_OK)
     {
     {
-      GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
-      GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
-      GNUNET_free_non_null(control_host);
-      GNUNET_free(allowed_hosts);
+      if (hostname != NULL)
+        GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1; %s;", control_host,
+                         hostname);
+      else
+        GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", control_host);
+
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
+                                             "ACCEPT_FROM", allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
+                                             "ACCEPT_FROM", allowed_hosts);
+      GNUNET_free_non_null (control_host);
+      GNUNET_free (allowed_hosts);
     }
 
 
     }
 
 
@@ -603,13 +936,27 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
    * otherwise gnunet-arm is unable to connect to it in some instances */
   if (hostname != NULL)
     {
    * otherwise gnunet-arm is unable to connect to it in some instances */
   if (hostname != NULL)
     {
-      GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
-      GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
-      GNUNET_free(allowed_hosts);
+      GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
+                                             "BINDTO", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
+                                             "BINDTO", hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
+                                             allowed_hosts);
+      GNUNET_free (allowed_hosts);
+    }
+  else
+    {
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
+                                             "BINDTO", "127.0.0.1");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
+                                             "BINDTO", "127.0.0.1");
     }
 
   *port = (uint16_t) uc.nport;
   *upnum = uc.upnum;
     }
 
   *port = (uint16_t) uc.nport;
   *upnum = uc.upnum;
+  uc.fdnum++;
+  *fdnum = uc.fdnum;
   return uc.ret;
 }
 
   return uc.ret;
 }
 
@@ -621,14 +968,13 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
  * @param first index of the first peer
  * @param second index of the second peer
  *
  * @param first index of the first peer
  * @param second index of the second peer
  *
- * @return the number of connections added (can be 0, 1 or 2)
- *         technically should only be 0 or 2, but the small price
- *         of iterating over the lists (hashmaps in the future)
- *         for being sure doesn't bother me!
+ * @return the number of connections added
+ *         technically should only be 0 or 2
  *
  */
  *
  */
-static int
-add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+add_actual_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                        unsigned int first, unsigned int second)
 {
   int added;
   int add_first;
 {
   int added;
   int add_first;
@@ -637,17 +983,21 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].connect_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].connect_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
     {
       add_second = GNUNET_YES;
     }
@@ -655,14 +1005,28 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   added = 0;
   if (add_first)
     {
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].connect_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].connect_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[second].num_connections++;
       added++;
     }
       pg->peers[second].num_connections++;
       added++;
     }
@@ -684,8 +1048,9 @@ add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  *         for being sure doesn't bother me!
  *
  */
  *         for being sure doesn't bother me!
  *
  */
-static int
-add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+add_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                         unsigned int first, unsigned int second)
 {
   int added;
 #if OLD
 {
   int added;
 #if OLD
@@ -700,17 +1065,21 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].allowed_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].allowed_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
     {
       add_second = GNUNET_YES;
     }
@@ -736,9 +1105,16 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   added = 0;
   if (add_first)
     {
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].allowed_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 #if OLD
 #if OLD
-      new_first = GNUNET_malloc(sizeof(struct PeerConnection));
+      new_first = GNUNET_malloc (sizeof (struct PeerConnection));
       new_first->daemon = pg->peers[second].daemon;
       new_first->next = pg->peers[first].connected_peers;
       pg->peers[first].connected_peers = new_first;
       new_first->daemon = pg->peers[second].daemon;
       new_first->next = pg->peers[first].connected_peers;
       pg->peers[first].connected_peers = new_first;
@@ -749,9 +1125,16 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
 
   if (add_second)
     {
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].allowed_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 #if OLD
 #if OLD
-      new_second = GNUNET_malloc(sizeof(struct PeerConnection));
+      new_second = GNUNET_malloc (sizeof (struct PeerConnection));
       new_second->daemon = pg->peers[first].daemon;
       new_second->next = pg->peers[second].connected_peers;
       pg->peers[second].connected_peers = new_second;
       new_second->daemon = pg->peers[first].daemon;
       new_second->next = pg->peers[second].connected_peers;
       pg->peers[second].connected_peers = new_second;
@@ -774,8 +1157,9 @@ add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  * @return the number of connections added (can be 0, 1 or 2)
  *
  */
  * @return the number of connections added (can be 0, 1 or 2)
  *
  */
-static int
-blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+blacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                       unsigned int first, unsigned int second)
 {
   int added;
   int add_first;
 {
   int added;
   int add_first;
@@ -783,17 +1167,23 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
   add_first = GNUNET_NO;
 
   add_first = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->
+                                              peers[first].blacklisted_peers,
+                                              &hash_second))
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
     {
       add_first = GNUNET_YES;
     }
 
   add_second = GNUNET_NO;
-  if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
+  if (GNUNET_NO ==
+      GNUNET_CONTAINER_multihashmap_contains (pg->
+                                              peers[second].blacklisted_peers,
+                                              &hash_first))
     {
       add_second = GNUNET_YES;
     }
     {
       add_second = GNUNET_YES;
     }
@@ -801,14 +1191,28 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
   added = 0;
   if (add_first)
     {
   added = 0;
   if (add_first)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [first].blacklisted_peers,
+                                                        &hash_second,
+                                                        pg->
+                                                        peers[second].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (pg->
+                                                        peers
+                                                        [second].blacklisted_peers,
+                                                        &hash_first,
+                                                        pg->
+                                                        peers[first].daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
       pg->peers[second].num_connections++;
       added++;
     }
       pg->peers[second].num_connections++;
       added++;
     }
@@ -826,8 +1230,9 @@ blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, u
  * @return the number of connections removed (can be 0, 1 or 2)
  *
  */
  * @return the number of connections removed (can be 0, 1 or 2)
  *
  */
-static int
-unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
+static unsigned int
+unblacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                         unsigned int first, unsigned int second)
 {
   int removed;
   int remove_first;
 {
   int removed;
   int remove_first;
@@ -835,22 +1240,42 @@ unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
   GNUNET_HashCode hash_first;
   GNUNET_HashCode hash_second;
 
-  hash_from_uid(first, &hash_first);
-  hash_from_uid(second, &hash_second);
+  hash_from_uid (first, &hash_first);
+  hash_from_uid (second, &hash_second);
 
 
-  remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
-  remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
+  remove_first =
+    GNUNET_CONTAINER_multihashmap_contains (pg->
+                                            peers[first].blacklisted_peers,
+                                            &hash_second);
+  remove_second =
+    GNUNET_CONTAINER_multihashmap_contains (pg->
+                                            peers[second].blacklisted_peers,
+                                            &hash_first);
 
   removed = 0;
   if (remove_first)
     {
 
   removed = 0;
   if (remove_first)
     {
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->
+                                                           peers
+                                                           [first].blacklisted_peers,
+                                                           &hash_second,
+                                                           pg->
+                                                           peers
+                                                           [second].daemon));
       removed++;
     }
 
   if (remove_second)
     {
       removed++;
     }
 
   if (remove_second)
     {
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->
+                                                           peers
+                                                           [second].blacklisted_peers,
+                                                           &hash_first,
+                                                           pg->
+                                                           peers
+                                                           [first].daemon));
       removed++;
     }
 
       removed++;
     }
 
@@ -872,8 +1297,9 @@ unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
  *
  * @return the number of connections created
  */
  *
  * @return the number of connections created
  */
-static int
-create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_scale_free (struct GNUNET_TESTING_PeerGroup *pg,
+                   GNUNET_TESTING_ConnectionProcessor proc)
 {
 
   unsigned int total_connections;
 {
 
   unsigned int total_connections;
@@ -883,19 +1309,23 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
   double random;
   double probability;
 
   double random;
   double probability;
 
-  GNUNET_assert(pg->total > 1);
+  GNUNET_assert (pg->total > 1);
 
   /* Add a connection between the first two nodes */
 
   /* Add a connection between the first two nodes */
-  total_connections = proc(pg, 0, 1);
+  total_connections = proc (pg, 0, 1);
 
   for (outer_count = 1; outer_count < pg->total; outer_count++)
     {
       previous_total_connections = total_connections;
       for (i = 0; i < outer_count; i++)
         {
 
   for (outer_count = 1; outer_count < pg->total; outer_count++)
     {
       previous_total_connections = total_connections;
       for (i = 0; i < outer_count; i++)
         {
-          probability = pg->peers[i].num_connections / (double)previous_total_connections;
-          random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
+          probability =
+            pg->peers[i].num_connections /
+            (double) previous_total_connections;
+          random =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX)) / ((double) UINT64_MAX);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Considering connecting peer %d to peer %d\n",
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Considering connecting peer %d to peer %d\n",
@@ -905,10 +1335,9 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
             {
 #if VERBOSE_TESTING
               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
             {
 #if VERBOSE_TESTING
               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                          "Connecting peer %d to peer %d\n",
-                          outer_count, i);
+                          "Connecting peer %d to peer %d\n", outer_count, i);
 #endif
 #endif
-              total_connections += proc(pg, outer_count, i);
+              total_connections += proc (pg, outer_count, i);
             }
         }
     }
             }
         }
     }
@@ -927,8 +1356,9 @@ create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connectio
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-int
-create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_small_world_ring (struct GNUNET_TESTING_PeerGroup *pg,
+                         GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i, j;
   int nodeToConnect;
 {
   unsigned int i, j;
   int nodeToConnect;
@@ -943,32 +1373,30 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
   unsigned int useAnd;
   int connect_attempts;
 
   unsigned int useAnd;
   int connect_attempts;
 
-  logNModifier = 0.5; /* FIXME: default value? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "LOGNMODIFIER",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &logNModifier) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "LOGNMODIFIER",
-                   "TESTING");
+  logNModifier = 0.5;           /* FIXME: default value? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "LOGNMODIFIER",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &logNModifier) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "LOGNMODIFIER", "TESTING");
       GNUNET_free (p_string);
     }
       GNUNET_free (p_string);
     }
-  percentage = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PERCENTAGE",
-                   "TESTING");
+  percentage = 0.5;             /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
   natLog = log (pg->total);
       GNUNET_free (p_string);
     }
   natLog = log (pg->total);
@@ -999,25 +1427,26 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
 
       for (j = 0; j < connsPerPeer / 2; j++)
         {
 
       for (j = 0; j < connsPerPeer / 2; j++)
         {
-          random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                     UINT64_MAX) / ( (double) UINT64_MAX));
+          random =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX) / ((double) UINT64_MAX));
           if (random < percentage)
             {
               /* Connect to uniformly selected random peer */
               randomPeer =
                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
           if (random < percentage)
             {
               /* Connect to uniformly selected random peer */
               randomPeer =
                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                   pg->total);
+                                          pg->total);
               while ((((randomPeer < max) && (randomPeer > min))
                       && (useAnd == 0)) || (((randomPeer > min)
                                              || (randomPeer < max))
                                             && (useAnd == 1)))
                 {
                   randomPeer =
               while ((((randomPeer < max) && (randomPeer > min))
                       && (useAnd == 0)) || (((randomPeer > min)
                                              || (randomPeer < max))
                                             && (useAnd == 1)))
                 {
                   randomPeer =
-                      GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
-                                                         pg->total);
+                    GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                              pg->total);
                 }
                 }
-              smallWorldConnections +=
-                proc (pg, i, randomPeer);
+              smallWorldConnections += proc (pg, i, randomPeer);
             }
           else
             {
             }
           else
             {
@@ -1026,8 +1455,7 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
                 {
                   nodeToConnect = nodeToConnect - pg->total;
                 }
                 {
                   nodeToConnect = nodeToConnect - pg->total;
                 }
-              connect_attempts +=
-                proc (pg, i, nodeToConnect);
+              connect_attempts += proc (pg, i, nodeToConnect);
             }
         }
 
             }
         }
 
@@ -1049,8 +1477,9 @@ create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conn
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg,
+                       GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int outer_count, inner_count;
   unsigned int cutoff;
 {
   unsigned int outer_count, inner_count;
   unsigned int cutoff;
@@ -1058,18 +1487,17 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
   double nat_percentage;
   char *p_string;
 
   double nat_percentage;
   char *p_string;
 
-  nat_percentage = 0.6; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "NATPERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &nat_percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "NATPERCENTAGE",
-                   "TESTING");
+  nat_percentage = 0.6;         /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &nat_percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
 
       GNUNET_free (p_string);
     }
 
@@ -1091,7 +1519,7 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
                           "Connecting peer %d to peer %d\n",
                           outer_count, inner_count);
 #endif
                           "Connecting peer %d to peer %d\n",
                           outer_count, inner_count);
 #endif
-              connect_attempts += proc(pg, outer_count, inner_count);
+              connect_attempts += proc (pg, outer_count, inner_count);
             }
         }
     }
             }
         }
     }
@@ -1111,8 +1539,9 @@ create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Conne
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_small_world (struct GNUNET_TESTING_PeerGroup *pg,
+                    GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i, j, k;
   unsigned int square;
 {
   unsigned int i, j, k;
   unsigned int square;
@@ -1128,38 +1557,45 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
   unsigned int distance;
   double probability, random, percentage;
   unsigned int smallWorldConnections;
   unsigned int distance;
   double probability, random, percentage;
   unsigned int smallWorldConnections;
+  unsigned int small_world_it;
   char *p_string;
   int connect_attempts;
   square = floor (sqrt (pg->total));
   rows = square;
   cols = square;
 
   char *p_string;
   int connect_attempts;
   square = floor (sqrt (pg->total));
   rows = square;
   cols = square;
 
-  percentage = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PERCENTAGE",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &percentage) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PERCENTAGE",
-                   "TESTING");
+  percentage = 0.5;             /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PERCENTAGE",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &percentage) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PERCENTAGE", "TESTING");
       GNUNET_free (p_string);
     }
       GNUNET_free (p_string);
     }
-  probability = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PROBABILITY",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &probability) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PROBABILITY",
-                   "TESTING");
+  if (percentage < 0.0)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
+                  "PERCENTAGE", "TESTING", percentage);
+      percentage = 0.5;
+    }
+  probability = 0.5;            /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PROBABILITY",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &probability) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PROBABILITY", "TESTING");
       GNUNET_free (p_string);
     }
   if (square * square != pg->total)
       GNUNET_free (p_string);
     }
   if (square * square != pg->total)
@@ -1175,9 +1611,10 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
         }
     }
 #if VERBOSE_TESTING
         }
     }
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
-                  rows, cols);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _
+              ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
+              rows, cols);
 #endif
 
   connect_attempts = 0;
 #endif
 
   connect_attempts = 0;
@@ -1210,11 +1647,16 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
 #if VERBOSE_TESTING > 2
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("natural log of %d is %d, will run %d iterations\n"),
 #if VERBOSE_TESTING > 2
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("natural log of %d is %d, will run %d iterations\n"),
-             pg->total, natLog, (int) (natLog * percentage));
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
+              pg->total, natLog, (int) (natLog * percentage));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Total connections added thus far: %u!\n"), connect_attempts);
 #endif
   smallWorldConnections = 0;
 #endif
   smallWorldConnections = 0;
-  for (i = 0; i < (int) (natLog * percentage); i++)
+  small_world_it = (unsigned int) (natLog * percentage);
+  if (small_world_it < 1)
+    small_world_it = 1;
+  GNUNET_assert (small_world_it > 0 && small_world_it < (unsigned int) -1);
+  for (i = 0; i < small_world_it; i++)
     {
       for (j = 0; j < pg->total; j++)
         {
     {
       for (j = 0; j < pg->total; j++)
         {
@@ -1227,14 +1669,18 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
               node2Row = k / cols;
               node2Col = k - (node2Row * cols);
               /* Simple Cartesian distance */
               node2Row = k / cols;
               node2Col = k - (node2Row * cols);
               /* Simple Cartesian distance */
-              distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
+              distance =
+                abs (node1Row - node2Row) + abs (node1Col - node2Col);
               if (distance > 1)
                 {
                   /* Calculate probability as 1 over the square of the distance */
                   probability = 1.0 / (distance * distance);
                   /* Choose a random value between 0 and 1 */
               if (distance > 1)
                 {
                   /* Calculate probability as 1 over the square of the distance */
                   probability = 1.0 / (distance * distance);
                   /* Choose a random value between 0 and 1 */
-                 random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                             UINT64_MAX)) / ( (double) UINT64_MAX);
+                  random =
+                    ((double)
+                     GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                               UINT64_MAX)) /
+                    ((double) UINT64_MAX);
                   /* If random < probability, then connect the two nodes */
                   if (random < probability)
                     smallWorldConnections += proc (pg, j, k);
                   /* If random < probability, then connect the two nodes */
                   if (random < probability)
                     smallWorldConnections += proc (pg, j, k);
@@ -1245,9 +1691,9 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
     }
   connect_attempts += smallWorldConnections;
 #if VERBOSE_TESTING > 2
     }
   connect_attempts += smallWorldConnections;
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Total connections added for small world: %d!\n"),
-                      smallWorldConnections);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Total connections added for small world: %d!\n"),
+              smallWorldConnections);
 #endif
   return connect_attempts;
 }
 #endif
   return connect_attempts;
 }
@@ -1263,8 +1709,9 @@ create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg,
+                    GNUNET_TESTING_ConnectionProcessor proc)
 {
   double temp_rand;
   unsigned int outer_count;
 {
   double temp_rand;
   unsigned int outer_count;
@@ -1273,18 +1720,17 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
   double probability;
   char *p_string;
 
   double probability;
   char *p_string;
 
-  probability = 0.5; /* FIXME: default percentage? */
-  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
-                                                        "TESTING",
-                                                        "PROBABILITY",
-                                                        &p_string))
-    {
-      if (sscanf(p_string, "%lf", &probability) != 1)
-       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                   _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
-                   p_string,
-                   "PROBABILITY",
-                   "TESTING");
+  probability = 0.5;            /* FIXME: default percentage? */
+  if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
+                                                          "TESTING",
+                                                          "PROBABILITY",
+                                                          &p_string))
+    {
+      if (sscanf (p_string, "%lf", &probability) != 1)
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                    _
+                    ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
+                    p_string, "PROBABILITY", "TESTING");
       GNUNET_free (p_string);
     }
   connect_attempts = 0;
       GNUNET_free (p_string);
     }
   connect_attempts = 0;
@@ -1293,8 +1739,10 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
       for (inner_count = outer_count + 1; inner_count < pg->total;
            inner_count++)
         {
       for (inner_count = outer_count + 1; inner_count < pg->total;
            inner_count++)
         {
-          temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                        UINT64_MAX)) / ( (double) UINT64_MAX);
+          temp_rand =
+            ((double)
+             GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                       UINT64_MAX)) / ((double) UINT64_MAX);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("rand is %f probability is %f\n"), temp_rand,
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("rand is %f probability is %f\n"), temp_rand,
@@ -1312,7 +1760,9 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
 
 /**
  * Create a topology given a peer group (set of running peers)
 
 /**
  * Create a topology given a peer group (set of running peers)
- * and a connection processor.
+ * and a connection processor.  This particular function creates
+ * the connections for a 2d-torus, plus additional "closest"
+ * connections per peer.
  *
  * @param pg the peergroup to create the topology on
  * @param proc the connection processor to call to actually set
  *
  * @param pg the peergroup to create the topology on
  * @param proc the connection processor to call to actually set
@@ -1321,8 +1771,9 @@ create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_Connecti
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg,
+                 GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int i;
   unsigned int square;
 {
   unsigned int i;
   unsigned int square;
@@ -1351,9 +1802,10 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
         }
     }
 #if VERBOSE_TESTING
         }
     }
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
-                  rows, cols);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _
+              ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
+              rows, cols);
 #endif
   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
    * to the node to its right and above.  Once this is over, we'll have our torus!
 #endif
   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
    * to the node to its right and above.  Once this is over, we'll have our torus!
@@ -1370,11 +1822,10 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
       else
         nodeToConnect = i - cols + 1;
 #if VERBOSE_TESTING
       else
         nodeToConnect = i - cols + 1;
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      i, nodeToConnect);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", i, nodeToConnect);
 #endif
 #endif
-      connect_attempts += proc(pg, i, nodeToConnect);
+      connect_attempts += proc (pg, i, nodeToConnect);
 
       /* Second connect to the node immediately above */
       if (i < cols)
 
       /* Second connect to the node immediately above */
       if (i < cols)
@@ -1386,10 +1837,9 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
         {
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
         {
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      i, nodeToConnect);
+                      "Connecting peer %d to peer %d\n", i, nodeToConnect);
 #endif
 #endif
-          connect_attempts += proc(pg, i, nodeToConnect);
+          connect_attempts += proc (pg, i, nodeToConnect);
         }
 
     }
         }
 
     }
@@ -1409,8 +1859,9 @@ create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionP
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_clique (struct GNUNET_TESTING_PeerGroup *pg,
+               GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int outer_count;
   unsigned int inner_count;
 {
   unsigned int outer_count;
   unsigned int inner_count;
@@ -1428,7 +1879,7 @@ create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionPro
                       "Connecting peer %d to peer %d\n",
                       outer_count, inner_count);
 #endif
                       "Connecting peer %d to peer %d\n",
                       outer_count, inner_count);
 #endif
-          connect_attempts += proc(pg, outer_count, inner_count);
+          connect_attempts += proc (pg, outer_count, inner_count);
         }
     }
 
         }
     }
 
@@ -1446,8 +1897,9 @@ create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionPro
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_line (struct GNUNET_TESTING_PeerGroup *pg,
+             GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int count;
   int connect_attempts;
 {
   unsigned int count;
   int connect_attempts;
@@ -1458,11 +1910,10 @@ create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      count, count + 1);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", count, count + 1);
 #endif
 #endif
-      connect_attempts += proc(pg, count, count + 1);
+      connect_attempts += proc (pg, count, count + 1);
     }
 
   return connect_attempts;
     }
 
   return connect_attempts;
@@ -1479,8 +1930,9 @@ create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
  * @return the number of connections that were set up
  *
  */
  * @return the number of connections that were set up
  *
  */
-static int
-create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
+static unsigned int
+create_ring (struct GNUNET_TESTING_PeerGroup *pg,
+             GNUNET_TESTING_ConnectionProcessor proc)
 {
   unsigned int count;
   int connect_attempts;
 {
   unsigned int count;
   int connect_attempts;
@@ -1491,15 +1943,14 @@ create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      "Connecting peer %d to peer %d\n",
-                      count, count + 1);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  "Connecting peer %d to peer %d\n", count, count + 1);
 #endif
 #endif
-      connect_attempts += proc(pg, count, count + 1);
+      connect_attempts += proc (pg, count, count + 1);
     }
 
   /* Connect the last peer to the first peer */
     }
 
   /* Connect the last peer to the first peer */
-  connect_attempts += proc(pg, pg->total - 1, 0);
+  connect_attempts += proc (pg, pg->total - 1, 0);
 
   return connect_attempts;
 }
 
   return connect_attempts;
 }
@@ -1523,9 +1974,7 @@ create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProce
  *       "fixing" now.
  */
 static int
  *       "fixing" now.
  */
 static int
-friend_file_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+friend_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   FILE *temp_friend_handle = cls;
   struct GNUNET_TESTING_Daemon *peer = value;
 {
   FILE *temp_friend_handle = cls;
   struct GNUNET_TESTING_Daemon *peer = value;
@@ -1533,8 +1982,8 @@ friend_file_iterator (void *cls,
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
-  GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
-  fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
+  GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
+  fprintf (temp_friend_handle, "%s\n", (char *) &peer_enc);
 
   return GNUNET_YES;
 }
 
   return GNUNET_YES;
 }
@@ -1562,9 +2011,7 @@ struct BlacklistContext
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-blacklist_file_iterator (void *cls,
-                         const GNUNET_HashCode * key,
-                         void *value)
+blacklist_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct BlacklistContext *blacklist_ctx = cls;
   //FILE *temp_blacklist_handle = cls;
 {
   struct BlacklistContext *blacklist_ctx = cls;
   //FILE *temp_blacklist_handle = cls;
@@ -1573,8 +2020,9 @@ blacklist_file_iterator (void *cls,
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
 
   temppeer = &peer->id;
-  GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
-  fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
+  GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
+  fprintf (blacklist_ctx->temp_file_handle, "%s:%s\n",
+           blacklist_ctx->transport, (char *) &peer_enc);
 
   return GNUNET_YES;
 }
 
   return GNUNET_YES;
 }
@@ -1592,64 +2040,78 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
   FILE *temp_friend_handle;
   unsigned int pg_iter;
   char *temp_service_path;
   FILE *temp_friend_handle;
   unsigned int pg_iter;
   char *temp_service_path;
-  pid_t *pidarr;
+  struct GNUNET_OS_Process **procarr;
   char *arg;
   char *arg;
-  char * mytemp;
+  char *mytemp;
   enum GNUNET_OS_ProcessStatusType type;
   unsigned long return_code;
   int count;
   int ret;
   int max_wait = 10;
 
   enum GNUNET_OS_ProcessStatusType type;
   unsigned long return_code;
   int count;
   int ret;
   int max_wait = 10;
 
-  pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
+  procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      mytemp = GNUNET_DISK_mktemp("friends");
-      GNUNET_assert(mytemp != NULL);
+      mytemp = GNUNET_DISK_mktemp ("friends");
+      GNUNET_assert (mytemp != NULL);
       temp_friend_handle = fopen (mytemp, "wt");
       temp_friend_handle = fopen (mytemp, "wt");
-      GNUNET_assert(temp_friend_handle != NULL);
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
-      fclose(temp_friend_handle);
+      GNUNET_assert (temp_friend_handle != NULL);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].allowed_peers,
+                                             &friend_file_iterator,
+                                             temp_friend_handle);
+      fclose (temp_friend_handle);
 
       if (GNUNET_OK !=
 
       if (GNUNET_OK !=
-         GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
-       {
+          GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
+                                                 daemon->cfg, "PATHS",
+                                                 "SERVICEHOME",
+                                                 &temp_service_path))
+        {
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                     _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
-                     "SERVICEHOME",
-                     "PATHS");
+                      _
+                      ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
+                      "SERVICEHOME", "PATHS");
           if (UNLINK (mytemp) != 0)
           if (UNLINK (mytemp) != 0)
-            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
-         GNUNET_free (mytemp);
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+                                      mytemp);
+          GNUNET_free (mytemp);
           break;
         }
 
           break;
         }
 
-      if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
+      if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
         {
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
         {
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
-                                         "mv", mytemp, arg, NULL);
+          procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
+                                                      "mv", mytemp, arg,
+                                                      NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
         }
-      else /* Remote, scp the file to the correct place */
+      else                      /* Remote, scp the file to the correct place */
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
-            GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
+            GNUNET_asprintf (&arg, "%s@%s:%s/friends",
+                             pg->peers[pg_iter].daemon->username,
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
           else
           else
-            GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
-                                         "scp", mytemp, arg, NULL);
+            GNUNET_asprintf (&arg, "%s:%s/friends",
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
+          procarr[pg_iter] =
+            GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
+                                     NULL);
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Copying file with command scp %s %s\n"), mytemp, arg);
+                      _("Copying file with command scp %s %s\n"), mytemp,
+                      arg);
 #endif
 #endif
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
@@ -1666,22 +2128,25 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
-          if (pidarr[pg_iter] != 0) /* Check for already completed! */
+          if (procarr[pg_iter] != NULL) /* Check for already completed! */
             {
             {
-              if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
+              if (GNUNET_OS_process_status
+                  (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
                 {
                   ret = GNUNET_SYSERR;
                 }
                 {
                   ret = GNUNET_SYSERR;
                 }
-              else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
+              else if ((type != GNUNET_OS_PROCESS_EXITED)
+                       || (return_code != 0))
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
-                  pidarr[pg_iter] = 0;
+                  GNUNET_OS_process_close (procarr[pg_iter]);
+                  procarr[pg_iter] = NULL;
 #if VERBOSE_TESTING
 #if VERBOSE_TESTING
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("File %d copied\n"), pg_iter);
+                  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                              _("File %d copied\n"), pg_iter);
 #endif
                 }
             }
 #endif
                 }
             }
@@ -1689,16 +2154,16 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
       count++;
       if (ret == GNUNET_SYSERR)
         {
       count++;
       if (ret == GNUNET_SYSERR)
         {
-         /* FIXME: why sleep here? -CG */
-          sleep(1);
+          /* FIXME: why sleep here? -CG */
+          sleep (1);
         }
     }
 
 #if VERBOSE_TESTING
         }
     }
 
 #if VERBOSE_TESTING
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                _("Finished copying all friend files!\n"));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Finished copying all friend files!\n"));
 #endif
 #endif
-  GNUNET_free(pidarr);
+  GNUNET_free (procarr);
   return ret;
 }
 
   return ret;
 }
 
@@ -1712,13 +2177,14 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
  * @param transports space delimited list of transports to blacklist
  */
 static int
  * @param transports space delimited list of transports to blacklist
  */
 static int
-create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
+create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
+                                 const char *transports)
 {
   FILE *temp_file_handle;
   static struct BlacklistContext blacklist_ctx;
   unsigned int pg_iter;
   char *temp_service_path;
 {
   FILE *temp_file_handle;
   static struct BlacklistContext blacklist_ctx;
   unsigned int pg_iter;
   char *temp_service_path;
-  pid_t *pidarr;
+  struct GNUNET_OS_Process **procarr;
   char *arg;
   char *mytemp;
   enum GNUNET_OS_ProcessStatusType type;
   char *arg;
   char *mytemp;
   enum GNUNET_OS_ProcessStatusType type;
@@ -1731,77 +2197,93 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
   char *pos;
   char *temp_transports;
 
   char *pos;
   char *temp_transports;
 
-  pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
+  procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      mytemp = GNUNET_DISK_mktemp("blacklist");
-      GNUNET_assert(mytemp != NULL);
+      mytemp = GNUNET_DISK_mktemp ("blacklist");
+      GNUNET_assert (mytemp != NULL);
       temp_file_handle = fopen (mytemp, "wt");
       temp_file_handle = fopen (mytemp, "wt");
-      GNUNET_assert(temp_file_handle != NULL);
-      temp_transports = GNUNET_strdup(transports);
+      GNUNET_assert (temp_file_handle != NULL);
+      temp_transports = GNUNET_strdup (transports);
       blacklist_ctx.temp_file_handle = temp_file_handle;
       blacklist_ctx.temp_file_handle = temp_file_handle;
-      transport_len = strlen(temp_transports) + 1;
+      transport_len = strlen (temp_transports) + 1;
       pos = NULL;
 
       for (i = 0; i < transport_len; i++)
       pos = NULL;
 
       for (i = 0; i < transport_len; i++)
-      {
-        if ((temp_transports[i] == ' ') && (pos == NULL))
-          continue; /* At start of string (whitespace) */
-        else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
-        {
-          temp_transports[i] = '\0';
-          blacklist_ctx.transport = pos;
-          GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
-          pos = NULL;
-        } /* At beginning of actual string */
-        else if (pos == NULL)
         {
         {
-          pos = &temp_transports[i];
+          if ((temp_transports[i] == ' ') && (pos == NULL))
+            continue;           /* At start of string (whitespace) */
+          else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
+            {
+              temp_transports[i] = '\0';
+              blacklist_ctx.transport = pos;
+              GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                                     peers
+                                                     [pg_iter].blacklisted_peers,
+                                                     &blacklist_file_iterator,
+                                                     &blacklist_ctx);
+              pos = NULL;
+            }                   /* At beginning of actual string */
+          else if (pos == NULL)
+            {
+              pos = &temp_transports[i];
+            }
         }
         }
-      }
 
       GNUNET_free (temp_transports);
 
       GNUNET_free (temp_transports);
-      fclose(temp_file_handle);
+      fclose (temp_file_handle);
 
       if (GNUNET_OK !=
 
       if (GNUNET_OK !=
-          GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
+          GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
+                                                 daemon->cfg, "PATHS",
+                                                 "SERVICEHOME",
+                                                 &temp_service_path))
         {
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
         {
           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                      _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
-                      "SERVICEHOME",
-                      "PATHS");
+                      _
+                      ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
+                      "SERVICEHOME", "PATHS");
           if (UNLINK (mytemp) != 0)
           if (UNLINK (mytemp) != 0)
-            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
+            GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
+                                      mytemp);
           GNUNET_free (mytemp);
           break;
         }
 
           GNUNET_free (mytemp);
           break;
         }
 
-      if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
+      if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
         {
           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
         {
           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
-                                         "mv", mytemp, arg, NULL);
+          procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
+                                                      "mv", mytemp, arg,
+                                                      NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command cp %s %s\n"), mytemp, arg);
 #endif
 
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
         }
-      else /* Remote, scp the file to the correct place */
+      else                      /* Remote, scp the file to the correct place */
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
         {
           if (NULL != pg->peers[pg_iter].daemon->username)
-            GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
+            GNUNET_asprintf (&arg, "%s@%s:%s/blacklist",
+                             pg->peers[pg_iter].daemon->username,
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
           else
           else
-            GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
-          pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
-                                         "scp", mytemp, arg, NULL);
+            GNUNET_asprintf (&arg, "%s:%s/blacklist",
+                             pg->peers[pg_iter].daemon->hostname,
+                             temp_service_path);
+          procarr[pg_iter] =
+            GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
+                                     NULL);
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Copying file with command scp %s %s\n"), mytemp, arg);
+                      _("Copying file with command scp %s %s\n"), mytemp,
+                      arg);
 #endif
 #endif
-          GNUNET_free(arg);
+          GNUNET_free (arg);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
         }
       GNUNET_free (temp_service_path);
       GNUNET_free (mytemp);
@@ -1818,22 +2300,25 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Checking copy status of file %d\n"), pg_iter);
 #endif
-          if (pidarr[pg_iter] != 0) /* Check for already completed! */
+          if (procarr[pg_iter] != NULL) /* Check for already completed! */
             {
             {
-              if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
+              if (GNUNET_OS_process_status
+                  (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
                 {
                   ret = GNUNET_SYSERR;
                 }
                 {
                   ret = GNUNET_SYSERR;
                 }
-              else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
+              else if ((type != GNUNET_OS_PROCESS_EXITED)
+                       || (return_code != 0))
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
                 {
                   ret = GNUNET_SYSERR;
                 }
               else
                 {
-                  pidarr[pg_iter] = 0;
+                  GNUNET_OS_process_close (procarr[pg_iter]);
+                  procarr[pg_iter] = NULL;
 #if VERBOSE_TESTING
 #if VERBOSE_TESTING
-            GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("File %d copied\n"), pg_iter);
+                  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                              _("File %d copied\n"), pg_iter);
 #endif
                 }
             }
 #endif
                 }
             }
@@ -1841,16 +2326,16 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
       count++;
       if (ret == GNUNET_SYSERR)
         {
       count++;
       if (ret == GNUNET_SYSERR)
         {
-         /* FIXME: why sleep here? -CG */
-          sleep(1);
+          /* FIXME: why sleep here? -CG */
+          sleep (1);
         }
     }
 
 #if VERBOSE_TESTING
         }
     }
 
 #if VERBOSE_TESTING
-    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                _("Finished copying all blacklist files!\n"));
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              _("Finished copying all blacklist files!\n"));
 #endif
 #endif
-  GNUNET_free(pidarr);
+  GNUNET_free (procarr);
   return ret;
 }
 
   return ret;
 }
 
@@ -1859,24 +2344,44 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *tran
  * Internal notification of a connection, kept so that we can ensure some connections
  * happen instead of flooding all testing daemons with requests to connect.
  */
  * Internal notification of a connection, kept so that we can ensure some connections
  * happen instead of flooding all testing daemons with requests to connect.
  */
-static void internal_connect_notify (void *cls,
-                                     const struct GNUNET_PeerIdentity *first,
-                                     const struct GNUNET_PeerIdentity *second,
-                                     uint32_t distance,
-                                     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
-                                     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
-                                     struct GNUNET_TESTING_Daemon *first_daemon,
-                                     struct GNUNET_TESTING_Daemon *second_daemon,
-                                     const char *emsg)
-{
-  struct GNUNET_TESTING_PeerGroup *pg = cls;
+static void
+internal_connect_notify (void *cls,
+                         const struct GNUNET_PeerIdentity *first,
+                         const struct GNUNET_PeerIdentity *second,
+                         uint32_t distance,
+                         const struct GNUNET_CONFIGURATION_Handle *first_cfg,
+                         const struct GNUNET_CONFIGURATION_Handle *second_cfg,
+                         struct GNUNET_TESTING_Daemon *first_daemon,
+                         struct GNUNET_TESTING_Daemon *second_daemon,
+                         const char *emsg)
+{
+  struct ConnectTopologyContext *ct_ctx = cls;
+  struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
   outstanding_connects--;
   outstanding_connects--;
+  ct_ctx->remaining_connections--;
+  if (ct_ctx->remaining_connections == 0)
+    {
+      if (ct_ctx->notify_connections_done != NULL)
+        ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+      GNUNET_free (ct_ctx);
+    }
 
 
-  pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
+  if (pg->notify_connection != NULL)
+    pg->notify_connection (pg->notify_connection_cls, first, second, distance,
+                           first_cfg, second_cfg, first_daemon, second_daemon,
+                           emsg);
 }
 
 
 }
 
 
-static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+/**
+ * Either delay a connection (because there are too many outstanding)
+ * or schedule it for right now.
+ *
+ * @param cls a connection context
+ * @param tc the task runtime context
+ */
+static void
+schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct ConnectContext *connect_context = cls;
 
 {
   struct ConnectContext *connect_context = cls;
 
@@ -1886,16 +2391,20 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
     {
 #if VERBOSE_TESTING > 2
   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
     {
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Delaying connect, we have too many outstanding connections!\n"));
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
 #endif
 #endif
-      GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 3), &schedule_connect, connect_context);
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_connect, connect_context);
     }
   else
     {
 #if VERBOSE_TESTING > 2
     }
   else
     {
 #if VERBOSE_TESTING > 2
-          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                      _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
 #endif
       outstanding_connects++;
       GNUNET_TESTING_daemons_connect (connect_context->first,
 #endif
       outstanding_connects++;
       GNUNET_TESTING_daemons_connect (connect_context->first,
@@ -1903,8 +2412,8 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
                                       CONNECT_TIMEOUT,
                                       CONNECT_ATTEMPTS,
                                       &internal_connect_notify,
                                       CONNECT_TIMEOUT,
                                       CONNECT_ATTEMPTS,
                                       &internal_connect_notify,
-                                      connect_context->pg);
-      GNUNET_free(connect_context);
+                                      connect_context->ct_ctx);
+      GNUNET_free (connect_context);
     }
 }
 
     }
 }
 
@@ -1920,19 +2429,18 @@ static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContex
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
 {
-  struct PeerData *first = cls;
+  struct ConnectTopologyContext *ct_ctx = cls;
+  struct PeerData *first = ct_ctx->first;
   struct GNUNET_TESTING_Daemon *second = value;
   struct ConnectContext *connect_context;
 
   struct GNUNET_TESTING_Daemon *second = value;
   struct ConnectContext *connect_context;
 
-  connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
-  connect_context->pg = first->pg;
+  connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
   connect_context->first = first->daemon;
   connect_context->second = second;
   connect_context->first = first->daemon;
   connect_context->second = second;
-  GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
+  connect_context->ct_ctx = ct_ctx;
+  GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
 
   return GNUNET_YES;
 }
 
   return GNUNET_YES;
 }
@@ -1949,13 +2457,14 @@ connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-copy_topology_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+copy_topology_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct PeerData *first = cls;
 
 {
   struct PeerData *first = cls;
 
-  GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+  GNUNET_assert (GNUNET_OK ==
+                 GNUNET_CONTAINER_multihashmap_put (first->connect_peers, key,
+                                                    value,
+                                                    GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 
   return GNUNET_YES;
 }
 
   return GNUNET_YES;
 }
@@ -1976,7 +2485,11 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
   total = 0;
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
   total = 0;
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
+      ret =
+        GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                               peers[pg_iter].allowed_peers,
+                                               &copy_topology_iterator,
+                                               &pg->peers[pg_iter]);
       if (GNUNET_SYSERR == ret)
         return GNUNET_SYSERR;
 
       if (GNUNET_SYSERR == ret)
         return GNUNET_SYSERR;
 
@@ -1992,37 +2505,64 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * of each peer in the peer group
  *
  * @param pg the peer group we are dealing with
  * of each peer in the peer group
  *
  * @param pg the peer group we are dealing with
+ * @param notify_callback callback to notify when finished
+ * @param notify_cls closure for notify callback
+ *
  * @return the number of connections that will be attempted
  */
 static int
  * @return the number of connections that will be attempted
  */
 static int
-connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
+connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
+                  GNUNET_TESTING_NotifyCompletion notify_callback,
+                  void *notify_cls)
 {
   unsigned int pg_iter;
   int ret;
 {
   unsigned int pg_iter;
   int ret;
-  int total;
+  unsigned int total;
+  struct ConnectTopologyContext *ct_ctx;
 #if OLD
   struct PeerConnection *connection_iter;
   struct ConnectContext *connect_context;
 #endif
 
   total = 0;
 #if OLD
   struct PeerConnection *connection_iter;
   struct ConnectContext *connect_context;
 #endif
 
   total = 0;
+  ct_ctx = GNUNET_malloc (sizeof (struct ConnectTopologyContext));
+  ct_ctx->notify_connections_done = notify_callback;
+  ct_ctx->notify_cls = notify_cls;
+  ct_ctx->pg = pg;
+
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
-      if (GNUNET_SYSERR == ret)
-        return GNUNET_SYSERR;
+      total +=
+        GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
+    }
+
+  if (total == 0)
+    {
+      GNUNET_free (ct_ctx);
+      return total;
+    }
+  ct_ctx->remaining_connections = total;
+  total = 0;
 
 
+  for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
+    {
+      ct_ctx->first = &pg->peers[pg_iter];
+      ret =
+        GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                               peers[pg_iter].connect_peers,
+                                               &connect_iterator, ct_ctx);
+      GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
       total = total + ret;
 
 #if OLD
       total = total + ret;
 
 #if OLD
-      connection_iter = ;
+      connection_iter = FIXME;
       while (connection_iter != NULL)
         {
       while (connection_iter != NULL)
         {
-          connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
+          connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
           connect_context->pg = pg;
           connect_context->pg = pg;
-          connect_context->first = ;
+          connect_context->first = FIXME;
           connect_context->second = connection_iter->daemon;
           connect_context->second = connection_iter->daemon;
-          GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
+          GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
           connection_iter = connection_iter->next;
         }
 #endif
           connection_iter = connection_iter->next;
         }
 #endif
@@ -2038,12 +2578,13 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * by the topology.  This will only have an effect once peers
  * are started if the FRIENDS_ONLY option is set in the base
  * config.  Also takes an optional restrict topology which
  * by the topology.  This will only have an effect once peers
  * are started if the FRIENDS_ONLY option is set in the base
  * config.  Also takes an optional restrict topology which
- * disallows direct TCP connections UNLESS they are specified in
- * the restricted topology.
+ * disallows connections based on particular transports
+ * UNLESS they are specified in the restricted topology.
  *
  * @param pg the peer group struct representing the running peers
  * @param topology which topology to connect the peers in
  *
  * @param pg the peer group struct representing the running peers
  * @param topology which topology to connect the peers in
- * @param restrict_topology allow only direct TCP connections in this topology
+ * @param restrict_topology disallow restrict_transports transport
+ *                          connections to peers NOT in this topology
  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
  * @param restrict_transports space delimited list of transports to blacklist
  *                            to create restricted topology
  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
  * @param restrict_transports space delimited list of transports to blacklist
  *                            to create restricted topology
@@ -2051,67 +2592,62 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * @return the maximum number of connections were all allowed peers
  *         connected to each other
  */
  * @return the maximum number of connections were all allowed peers
  *         connected to each other
  */
-int
+unsigned int
 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                 enum GNUNET_TESTING_Topology topology,
 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                 enum GNUNET_TESTING_Topology topology,
-                                enum GNUNET_TESTING_Topology restrict_topology,
-                                char *restrict_transports)
+                                enum GNUNET_TESTING_Topology
+                                restrict_topology,
+                                const char *restrict_transports)
 {
   int ret;
 {
   int ret;
-  int num_connections;
+  unsigned int num_connections;
   int unblacklisted_connections;
 
   int unblacklisted_connections;
 
-  GNUNET_assert (pg->notify_connection != NULL);
-  ret = GNUNET_OK;
-
   switch (topology)
     {
     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
   switch (topology)
     {
     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
-#if VERBOSE_TOPOLOGY
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating clique topology\n"));
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating clique topology\n"));
 #endif
       num_connections = create_clique (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #endif
       num_connections = create_clique (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) topology\n"));
 #endif
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) topology\n"));
 #endif
-      num_connections = create_small_world_ring (pg, &add_allowed_connections);
+      num_connections =
+        create_small_world_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) topology\n"));
 #endif
       num_connections = create_small_world (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) topology\n"));
 #endif
       num_connections = create_small_world (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
-#if VERBOSE_TOPOLOGY
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating ring topology\n"));
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating ring topology\n"));
 #endif
       num_connections = create_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
 #endif
       num_connections = create_ring (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
-#if VERBOSE_TOPOLOGY
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating 2d torus topology\n"));
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating 2d torus topology\n"));
 #endif
       num_connections = create_2d_torus (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #endif
       num_connections = create_2d_torus (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
-#if VERBOSE_TOPOLOGY
+#if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi topology\n"));
 #endif
       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi topology\n"));
 #endif
       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
-#if VERBOSE_TOPOLOGY
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Creating InterNAT topology\n"));
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating InterNAT topology\n"));
 #endif
       num_connections = create_nated_internet (pg, &add_allowed_connections);
       break;
 #endif
       num_connections = create_nated_internet (pg, &add_allowed_connections);
       break;
@@ -2130,38 +2666,43 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       num_connections = create_line (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
       num_connections = create_line (pg, &add_allowed_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Creating no allowed topology (all peers can connect at core level)\n"));
+#endif
       num_connections = 0;
       break;
     default:
       num_connections = 0;
       break;
     }
       num_connections = 0;
       break;
     default:
       num_connections = 0;
       break;
     }
-  if (num_connections < 1)
-    return GNUNET_SYSERR;
-
-  if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
-    {
-      ret = create_and_copy_friend_files(pg);
-    }
 
 
-  if (ret != GNUNET_OK)
+  if (GNUNET_YES ==
+      GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
     {
     {
+      ret = create_and_copy_friend_files (pg);
+      if (ret != GNUNET_OK)
+        {
 #if VERBOSE_TESTING
 #if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Failed during friend file copying!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Failed during friend file copying!\n"));
 #endif
 #endif
-      return GNUNET_SYSERR;
-    }
-  else
-    {
+          return GNUNET_SYSERR;
+        }
+      else
+        {
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Friend files created/copied successfully!\n"));
 #endif
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Friend files created/copied successfully!\n"));
 #endif
+        }
     }
 
   /* Use the create clique method to initially set all connections as blacklisted. */
     }
 
   /* Use the create clique method to initially set all connections as blacklisted. */
-  create_clique (pg, &blacklist_connections);
+  if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
+    create_clique (pg, &blacklist_connections);
+
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
   switch (restrict_topology)
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
   switch (restrict_topology)
@@ -2171,21 +2712,25 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but clique topology\n"));
 #endif
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but clique topology\n"));
 #endif
-      unblacklisted_connections = create_clique (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_clique (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but small world (ring) topology\n"));
 #endif
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but small world (ring) topology\n"));
 #endif
-      unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_small_world_ring (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
       break;
     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Blacklisting all but small world (2d-torus) topology\n"));
+                  _
+                  ("Blacklisting all but small world (2d-torus) topology\n"));
 #endif
 #endif
-      unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_small_world (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TESTING
       break;
     case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TESTING
@@ -2199,28 +2744,32 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but 2d torus topology\n"));
 #endif
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but 2d torus topology\n"));
 #endif
-      unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_2d_torus (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Erdos-Renyi topology\n"));
 #endif
       break;
     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Erdos-Renyi topology\n"));
 #endif
-      unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_erdos_renyi (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but InterNAT topology\n"));
 #endif
       break;
     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but InterNAT topology\n"));
 #endif
-      unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_nated_internet (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Scale Free topology\n"));
 #endif
       break;
     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Blacklisting all but Scale Free topology\n"));
 #endif
-      unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
+      unblacklisted_connections =
+        create_scale_free (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_LINE:
 #if VERBOSE_TESTING
       break;
     case GNUNET_TESTING_TOPOLOGY_LINE:
 #if VERBOSE_TESTING
@@ -2230,30 +2779,34 @@ GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
       unblacklisted_connections = create_line (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
       unblacklisted_connections = create_line (pg, &unblacklist_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_NONE:
-      /* Fall through */
+#if VERBOSE_TESTING
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Creating no blacklist topology (all peers can connect at transport level)\n"));
+#endif
     default:
       break;
     }
 
   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
     default:
       break;
     }
 
   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
-  {
-    ret = create_and_copy_blacklist_files(pg, restrict_transports);
-    if (ret != GNUNET_OK)
-      {
+    {
+      ret = create_and_copy_blacklist_files (pg, restrict_transports);
+      if (ret != GNUNET_OK)
+        {
 #if VERBOSE_TESTING
 #if VERBOSE_TESTING
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    _("Failed during blacklist file copying!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Failed during blacklist file copying!\n"));
 #endif
 #endif
-        return GNUNET_SYSERR;
-      }
-    else
-      {
+          return 0;
+        }
+      else
+        {
 #if VERBOSE_TESTING
 #if VERBOSE_TESTING
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                    _("Blacklist files created/copied successfully!\n"));
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                      _("Blacklist files created/copied successfully!\n"));
 #endif
 #endif
-      }
-  }
+        }
+    }
   return num_connections;
 }
 
   return num_connections;
 }
 
@@ -2357,25 +2910,35 @@ struct DFSContext
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-random_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+random_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct RandomContext *random_ctx = cls;
   double random_number;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
 {
   struct RandomContext *random_ctx = cls;
   double random_number;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
-  random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
-                                                    UINT64_MAX)) / ( (double) UINT64_MAX);
+  random_number =
+    ((double)
+     GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
+                               UINT64_MAX)) / ((double) UINT64_MAX);
   if (random_number < random_ctx->percentage)
   if (random_number < random_ctx->percentage)
-  {
-    GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-  }
+    {
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (random_ctx->
+                                                        first->connect_peers_working_set,
+                                                        key, value,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+    }
   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
-  uid_from_hash(key, &second_pos);
-  hash_from_uid(random_ctx->first_uid, &first_hash);
-  GNUNET_assert(random_ctx->pg->total > second_pos);
-  GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
+  uid_from_hash (key, &second_pos);
+  hash_from_uid (random_ctx->first_uid, &first_hash);
+  GNUNET_assert (random_ctx->pg->total > second_pos);
+  GNUNET_assert (GNUNET_YES ==
+                 GNUNET_CONTAINER_multihashmap_remove (random_ctx->
+                                                       pg->peers
+                                                       [second_pos].connect_peers,
+                                                       &first_hash,
+                                                       random_ctx->
+                                                       first->daemon));
 
   return GNUNET_YES;
 }
 
   return GNUNET_YES;
 }
@@ -2390,35 +2953,48 @@ random_connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-minimum_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+minimum_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct MinimumContext *min_ctx = cls;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
   unsigned int i;
 
 {
   struct MinimumContext *min_ctx = cls;
   uint32_t second_pos;
   GNUNET_HashCode first_hash;
   unsigned int i;
 
-  if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
-  {
-    for (i = 0; i < min_ctx->num_to_add; i++)
-    {
-      if (min_ctx->pg_array[i] == min_ctx->current)
-      {
-        GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        uid_from_hash(key, &second_pos);
-        hash_from_uid(min_ctx->first_uid, &first_hash);
-        GNUNET_assert(min_ctx->pg->total > second_pos);
-        GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-        /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
-        GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
-      }
-    }
-    min_ctx->current++;
-    return GNUNET_YES;
-  }
+  if (GNUNET_CONTAINER_multihashmap_size
+      (min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
+    {
+      for (i = 0; i < min_ctx->num_to_add; i++)
+        {
+          if (min_ctx->pg_array[i] == min_ctx->current)
+            {
+              GNUNET_assert (GNUNET_OK ==
+                             GNUNET_CONTAINER_multihashmap_put
+                             (min_ctx->first->connect_peers_working_set, key,
+                              value,
+                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+              uid_from_hash (key, &second_pos);
+              hash_from_uid (min_ctx->first_uid, &first_hash);
+              GNUNET_assert (min_ctx->pg->total > second_pos);
+              GNUNET_assert (GNUNET_OK ==
+                             GNUNET_CONTAINER_multihashmap_put (min_ctx->
+                                                                pg->peers
+                                                                [second_pos].connect_peers_working_set,
+                                                                &first_hash,
+                                                                min_ctx->first->
+                                                                daemon,
+                                                                GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+              /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
+              GNUNET_assert (GNUNET_YES ==
+                             GNUNET_CONTAINER_multihashmap_remove
+                             (min_ctx->pg->peers[second_pos].connect_peers,
+                              &first_hash, min_ctx->first->daemon));
+            }
+        }
+      min_ctx->current++;
+      return GNUNET_YES;
+    }
   else
   else
-    return GNUNET_NO; /* We can stop iterating, we have enough peers! */
+    return GNUNET_NO;           /* We can stop iterating, we have enough peers! */
 
 }
 
 
 }
 
@@ -2433,22 +3009,37 @@ minimum_connect_iterator (void *cls,
  * @return GNUNET_YES to continue iteration
  */
 static int
  * @return GNUNET_YES to continue iteration
  */
 static int
-dfs_connect_iterator (void *cls,
-                  const GNUNET_HashCode * key,
-                  void *value)
+dfs_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
 {
   struct DFSContext *dfs_ctx = cls;
   GNUNET_HashCode first_hash;
 
   if (dfs_ctx->current == dfs_ctx->chosen)
     {
 {
   struct DFSContext *dfs_ctx = cls;
   GNUNET_HashCode first_hash;
 
   if (dfs_ctx->current == dfs_ctx->chosen)
     {
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-      uid_from_hash(key, &dfs_ctx->second_uid);
-      hash_from_uid(dfs_ctx->first_uid, &first_hash);
-      GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
+                                                        first->connect_peers_working_set,
+                                                        key, value,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      uid_from_hash (key, &dfs_ctx->second_uid);
+      hash_from_uid (dfs_ctx->first_uid, &first_hash);
+      GNUNET_assert (GNUNET_OK ==
+                     GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
+                                                        pg->peers
+                                                        [dfs_ctx->second_uid].connect_peers_working_set,
+                                                        &first_hash,
+                                                        dfs_ctx->
+                                                        first->daemon,
+                                                        GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (dfs_ctx->
+                                                           pg->peers
+                                                           [dfs_ctx->second_uid].connect_peers,
+                                                           &first_hash,
+                                                           dfs_ctx->
+                                                           first->daemon));
       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
-      return GNUNET_NO; /* We have found our peer, don't iterate more */
+      return GNUNET_NO;         /* We have found our peer, don't iterate more */
     }
 
   dfs_ctx->current++;
     }
 
   dfs_ctx->current++;
@@ -2464,7 +3055,8 @@ dfs_connect_iterator (void *cls,
  * @param percentage what percent of total connections to make
  */
 void
  * @param percentage what percent of total connections to make
  */
 void
-choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
+choose_random_connections (struct GNUNET_TESTING_PeerGroup *pg,
+                           double percentage)
 {
   struct RandomContext random_ctx;
   uint32_t pg_iter;
 {
   struct RandomContext random_ctx;
   uint32_t pg_iter;
@@ -2475,12 +3067,17 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage
       random_ctx.first = &pg->peers[pg_iter];
       random_ctx.percentage = percentage;
       random_ctx.pg = pg;
       random_ctx.first = &pg->peers[pg_iter];
       random_ctx.percentage = percentage;
       random_ctx.pg = pg;
-      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (pg->total);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
+                                             &random_connect_iterator,
+                                             &random_ctx);
       /* Now remove the old connections */
       /* Now remove the old connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the random set */
       /* And replace with the random set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
     }
 }
 
     }
 }
 
@@ -2492,43 +3089,48 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage
  * @param num how many connections at least should each peer have (if possible)?
  */
 static void
  * @param num how many connections at least should each peer have (if possible)?
  */
 static void
-choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
+choose_minimum (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
 {
   struct MinimumContext minimum_ctx;
   uint32_t pg_iter;
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
 {
   struct MinimumContext minimum_ctx;
   uint32_t pg_iter;
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
-   {
-     pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
-   }
+    {
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (num);
+    }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       minimum_ctx.first_uid = pg_iter;
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       minimum_ctx.first_uid = pg_iter;
-      minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, 
-                                                         GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
+      minimum_ctx.pg_array =
+        GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK,
+                                      GNUNET_CONTAINER_multihashmap_size
+                                      (pg->peers[pg_iter].connect_peers));
       minimum_ctx.first = &pg->peers[pg_iter];
       minimum_ctx.pg = pg;
       minimum_ctx.num_to_add = num;
       minimum_ctx.current = 0;
       minimum_ctx.first = &pg->peers[pg_iter];
       minimum_ctx.pg = pg;
       minimum_ctx.num_to_add = num;
       minimum_ctx.current = 0;
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
-                                           &minimum_connect_iterator, 
-                                           &minimum_ctx);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
+                                             &minimum_connect_iterator,
+                                             &minimum_ctx);
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the working set */
       /* And replace with the working set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
     }
 
 }
 
 
 static unsigned int
     }
 
 }
 
 
 static unsigned int
-count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
+count_workingset_connections (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int count;
   unsigned int pg_iter;
 {
   unsigned int count;
   unsigned int pg_iter;
@@ -2537,14 +3139,18 @@ count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
+      count +=
+        GNUNET_CONTAINER_multihashmap_size (pg->
+                                            peers
+                                            [pg_iter].connect_peers_working_set);
     }
 
   return count;
 }
 
 
     }
 
   return count;
 }
 
 
-static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
+static unsigned int
+count_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int count;
   unsigned int pg_iter;
 {
   unsigned int count;
   unsigned int pg_iter;
@@ -2553,12 +3159,116 @@ static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *p
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
+      count +=
+        GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
     }
 
   return count;
 }
 
     }
 
   return count;
 }
 
+
+struct FindClosestContext
+{
+  /**
+   * The currently known closest peer.
+   */
+  struct GNUNET_TESTING_Daemon *closest;
+
+  /**
+   * The info for the peer we are adding connections for.
+   */
+  struct PeerData *curr_peer;
+
+  /**
+   * The distance (bits) between the current
+   * peer and the currently known closest.
+   */
+  unsigned int closest_dist;
+
+  /**
+   * The offset of the closest known peer in
+   * the peer group.
+   */
+  unsigned int closest_num;
+};
+
+/**
+ * Iterator over hash map entries of the allowed
+ * peer connections.  Find the closest, not already
+ * connected peer and return it.
+ *
+ * @param cls closure (struct FindClosestContext)
+ * @param key current key code (hash of offset in pg)
+ * @param value value in the hash map - a GNUNET_TESTING_Daemon
+ * @return GNUNET_YES if we should continue to
+ *         iterate,
+ *         GNUNET_NO if not.
+ */
+static int
+find_closest_peers (void *cls, const GNUNET_HashCode * key, void *value)
+{
+  struct FindClosestContext *closest_ctx = cls;
+  struct GNUNET_TESTING_Daemon *daemon = value;
+
+  if (((closest_ctx->closest == NULL) ||
+       (GNUNET_CRYPTO_hash_matching_bits
+        (&daemon->id.hashPubKey,
+         &closest_ctx->curr_peer->daemon->id.hashPubKey) >
+        closest_ctx->closest_dist))
+      && (GNUNET_YES !=
+          GNUNET_CONTAINER_multihashmap_contains (closest_ctx->
+                                                  curr_peer->connect_peers,
+                                                  key)))
+    {
+      closest_ctx->closest_dist =
+        GNUNET_CRYPTO_hash_matching_bits (&daemon->id.hashPubKey,
+                                          &closest_ctx->curr_peer->daemon->
+                                          id.hashPubKey);
+      closest_ctx->closest = daemon;
+      uid_from_hash (key, &closest_ctx->closest_num);
+    }
+  return GNUNET_YES;
+}
+
+/**
+ * From the set of connections possible, choose at num connections per
+ * peer based on depth which are closest out of those allowed.  Guaranteed
+ * to add num peers to connect to, provided there are that many peers
+ * in the underlay topology to connect to.
+ *
+ * @param pg the peergroup we are dealing with
+ * @param num how many connections at least should each peer have (if possible)?
+ * @param proc processor to actually add the connections
+ */
+void
+add_closest (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num,
+             GNUNET_TESTING_ConnectionProcessor proc)
+{
+  struct FindClosestContext closest_ctx;
+  uint32_t pg_iter;
+  uint32_t i;
+
+  for (i = 0; i < num; i++)     /* Each time find a closest peer (from those available) */
+    {
+      for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
+        {
+          closest_ctx.curr_peer = &pg->peers[pg_iter];
+          closest_ctx.closest = NULL;
+          closest_ctx.closest_dist = 0;
+          closest_ctx.closest_num = 0;
+          GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                                 peers[pg_iter].allowed_peers,
+                                                 &find_closest_peers,
+                                                 &closest_ctx);
+          if (closest_ctx.closest != NULL)
+            {
+              GNUNET_assert (closest_ctx.closest_num < pg->total);
+              proc (pg, pg_iter, closest_ctx.closest_num);
+            }
+        }
+    }
+}
+
 /**
  * From the set of connections possible, choose at least num connections per
  * peer based on depth first traversal of peer connections.  If DFS leaves
 /**
  * From the set of connections possible, choose at least num connections per
  * peer based on depth first traversal of peer connections.  If DFS leaves
@@ -2579,53 +3289,437 @@ perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
+      pg->peers[pg_iter].connect_peers_working_set =
+        GNUNET_CONTAINER_multihashmap_create (num);
     }
 
   starting_peer = 0;
   dfs_count = 0;
     }
 
   starting_peer = 0;
   dfs_count = 0;
-  while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
+  while ((count_workingset_connections (pg) < num * pg->total)
+         && (count_allowed_connections (pg) > 0))
     {
     {
-      if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
+      if (dfs_count % pg->total == 0)   /* Restart the DFS at some weakly connected peer */
         {
         {
-          least_connections = -1; /* Set to very high number */
+          least_connections = -1;       /* Set to very high number */
           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
             {
           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
             {
-              if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
+              if (GNUNET_CONTAINER_multihashmap_size
+                  (pg->peers[pg_iter].connect_peers_working_set) <
+                  least_connections)
                 {
                   starting_peer = pg_iter;
                 {
                   starting_peer = pg_iter;
-                  least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
+                  least_connections =
+                    GNUNET_CONTAINER_multihashmap_size (pg->
+                                                        peers
+                                                        [pg_iter].connect_peers_working_set);
                 }
             }
         }
 
                 }
             }
         }
 
-      if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
+      if (GNUNET_CONTAINER_multihashmap_size (pg->peers[starting_peer].connect_peers) == 0)     /* Ensure there is at least one peer left to connect! */
         {
           dfs_count = 0;
           continue;
         }
 
       /* Choose a random peer from the chosen peers set of connections to add */
         {
           dfs_count = 0;
           continue;
         }
 
       /* Choose a random peer from the chosen peers set of connections to add */
-      dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
+      dfs_ctx.chosen =
+        GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
+                                  GNUNET_CONTAINER_multihashmap_size
+                                  (pg->peers[starting_peer].connect_peers));
       dfs_ctx.first_uid = starting_peer;
       dfs_ctx.first = &pg->peers[starting_peer];
       dfs_ctx.pg = pg;
       dfs_ctx.current = 0;
 
       dfs_ctx.first_uid = starting_peer;
       dfs_ctx.first = &pg->peers[starting_peer];
       dfs_ctx.pg = pg;
       dfs_ctx.current = 0;
 
-      GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
+      GNUNET_CONTAINER_multihashmap_iterate (pg->
+                                             peers
+                                             [starting_peer].connect_peers,
+                                             &dfs_connect_iterator, &dfs_ctx);
       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
-      hash_from_uid(dfs_ctx.second_uid, &second_hash);
-      GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
+      hash_from_uid (dfs_ctx.second_uid, &second_hash);
+      GNUNET_assert (GNUNET_YES ==
+                     GNUNET_CONTAINER_multihashmap_remove (pg->peers
+                                                           [starting_peer].connect_peers,
+                                                           &second_hash,
+                                                           pg->
+                                                           peers
+                                                           [dfs_ctx.second_uid].daemon));
       starting_peer = dfs_ctx.second_uid;
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
       starting_peer = dfs_ctx.second_uid;
     }
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
       /* Remove the "old" connections */
-      GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
+      GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                             peers[pg_iter].connect_peers);
       /* And replace with the working set */
       /* And replace with the working set */
-      pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers =
+        pg->peers[pg_iter].connect_peers_working_set;
+    }
+}
+
+/**
+ * Internal callback for topology information for a particular peer.
+ */
+static void
+internal_topology_callback (void *cls,
+                            const struct GNUNET_PeerIdentity *peer,
+                            const struct GNUNET_TRANSPORT_ATS_Information
+                            *atsi)
+{
+  struct CoreContext *core_ctx = cls;
+  struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
+
+  if (peer == NULL)             /* Either finished, or something went wrong */
+    {
+      iter_ctx->completed++;
+      iter_ctx->connected--;
+      /* One core context allocated per iteration, must free! */
+      GNUNET_free (core_ctx);
+    }
+  else
+    {
+      iter_ctx->topology_cb (iter_ctx->cls, &core_ctx->daemon->id,
+                             peer, NULL);
+    }
+
+  if (iter_ctx->completed == iter_ctx->total)
+    {
+      iter_ctx->topology_cb (iter_ctx->cls, NULL, NULL, NULL);
+      /* Once all are done, free the iteration context */
+      GNUNET_free (iter_ctx);
+    }
+}
+
+
+/**
+ * Check running topology iteration tasks, if below max start a new one, otherwise
+ * schedule for some time in the future.
+ */
+static void
+schedule_get_topology (void *cls,
+                       const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct CoreContext *core_context = cls;
+  struct TopologyIterateContext *topology_context =
+    (struct TopologyIterateContext *) core_context->iter_context;
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    return;
+
+  if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
+#endif
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_get_topology, core_context);
+    }
+  else
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
+#endif
+      topology_context->connected++;
+
+      if (GNUNET_OK !=
+          GNUNET_CORE_iterate_peers (core_context->daemon->cfg,
+                                     &internal_topology_callback,
+                                     core_context))
+        {
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Topology iteration failed.\n");
+          internal_topology_callback (core_context, NULL, NULL);
+        }
+    }
+}
+
+/**
+ * Iterate over all (running) peers in the peer group, retrieve
+ * all connections that each currently has.
+ */
+void
+GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
+                             GNUNET_TESTING_NotifyTopology cb, void *cls)
+{
+  struct TopologyIterateContext *topology_context;
+  struct CoreContext *core_ctx;
+  unsigned int i;
+  unsigned int total_count;
+
+  /* Allocate a single topology iteration context */
+  topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
+  topology_context->topology_cb = cb;
+  topology_context->cls = cls;
+  total_count = 0;
+  for (i = 0; i < pg->total; i++)
+    {
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          /* Allocate one core context per core we need to connect to */
+          core_ctx = GNUNET_malloc (sizeof (struct CoreContext));
+          core_ctx->daemon = pg->peers[i].daemon;
+          /* Set back pointer to topology iteration context */
+          core_ctx->iter_context = topology_context;
+          GNUNET_SCHEDULER_add_now (&schedule_get_topology, core_ctx);
+          total_count++;
+        }
+    }
+  if (total_count == 0)
+    {
+      cb (cls, NULL, NULL, "Cannot iterate over topology, no running peers!");
+      GNUNET_free (topology_context);
+    }
+  else
+    topology_context->total = total_count;
+  return;
+}
+
+/**
+ * Callback function to process statistic values.
+ * This handler is here only really to insert a peer
+ * identity (or daemon) so the statistics can be uniquely
+ * tied to a single running peer.
+ *
+ * @param cls closure
+ * @param subsystem name of subsystem that created the statistic
+ * @param name the name of the datum
+ * @param value the current value
+ * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
+ * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
+ */
+static int
+internal_stats_callback (void *cls,
+                         const char *subsystem,
+                         const char *name, uint64_t value, int is_persistent)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  return stats_context->proc (stats_context->cls, &core_context->daemon->id,
+                              subsystem, name, value, is_persistent);
+}
+
+/**
+ * Internal continuation call for statistics iteration.
+ *
+ * @param cls closure, the CoreContext for this iteration
+ * @param success whether or not the statistics iterations
+ *        was canceled or not (we don't care)
+ */
+static void
+internal_stats_cont (void *cls, int success)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  stats_context->connected--;
+  stats_context->completed++;
+
+  if (stats_context->completed == stats_context->total)
+    {
+      stats_context->cont (stats_context->cls, GNUNET_YES);
+      GNUNET_free (stats_context);
+    }
+
+  if (core_context->stats_handle != NULL)
+    GNUNET_STATISTICS_destroy (core_context->stats_handle, GNUNET_NO);
+
+  GNUNET_free (core_context);
+}
+
+/**
+ * Check running topology iteration tasks, if below max start a new one, otherwise
+ * schedule for some time in the future.
+ */
+static void
+schedule_get_statistics (void *cls,
+                         const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct StatsCoreContext *core_context = cls;
+  struct StatsIterateContext *stats_context =
+    (struct StatsIterateContext *) core_context->iter_context;
+
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    return;
+
+  if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _
+                  ("Delaying connect, we have too many outstanding connections!\n"));
+#endif
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &schedule_get_statistics, core_context);
+    }
+  else
+    {
+#if VERBOSE_TESTING > 2
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                  _("Creating connection, outstanding_connections is %d\n"),
+                  outstanding_connects);
+#endif
+
+      stats_context->connected++;
+      core_context->stats_handle =
+        GNUNET_STATISTICS_create ("testing", core_context->daemon->cfg);
+      if (core_context->stats_handle == NULL)
+        {
+          internal_stats_cont (core_context, GNUNET_NO);
+          return;
+        }
+
+      core_context->stats_get_handle =
+        GNUNET_STATISTICS_get (core_context->stats_handle, NULL, NULL,
+                               GNUNET_TIME_relative_get_forever (),
+                               &internal_stats_cont, &internal_stats_callback,
+                               core_context);
+      if (core_context->stats_get_handle == NULL)
+        internal_stats_cont (core_context, GNUNET_NO);
+
+    }
+}
+
+struct DuplicateStats
+{
+  /**
+   * Next item in the list
+   */
+  struct DuplicateStats *next;
+
+  /**
+   * Nasty string, concatenation of relevant information.
+   */
+  char *unique_string;
+};
+
+/**
+ * Check whether the combination of port/host/unix domain socket
+ * already exists in the list of peers being checked for statistics.
+ *
+ * @param pg the peergroup in question
+ * @param specific_peer the peer we're concerned with
+ * @param stats_list the list to return to the caller
+ *
+ * @return GNUNET_YES if the statistics instance has been seen already,
+ *         GNUNET_NO if not (and we may have added it to the list)
+ */
+static int
+stats_check_existing (struct GNUNET_TESTING_PeerGroup *pg,
+                      struct PeerData *specific_peer,
+                      struct DuplicateStats **stats_list)
+{
+  struct DuplicateStats *pos;
+  char *unix_domain_socket;
+  unsigned long long port;
+  char *to_match;
+  if (GNUNET_YES !=
+      GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "testing",
+                                            "single_statistics_per_host"))
+    return GNUNET_NO;           /* Each peer has its own statistics instance, do nothing! */
+
+  pos = *stats_list;
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_string (specific_peer->cfg, "statistics",
+                                             "unixpath", &unix_domain_socket))
+    return GNUNET_NO;
+
+  if (GNUNET_OK !=
+      GNUNET_CONFIGURATION_get_value_number (specific_peer->cfg, "statistics",
+                                             "port", &port))
+    {
+      GNUNET_free(unix_domain_socket);
+      return GNUNET_NO;
+    }
+
+  if (specific_peer->daemon->hostname != NULL)
+    GNUNET_asprintf (&to_match, "%s%s%llu", specific_peer->daemon->hostname,
+                     unix_domain_socket, port);
+  else
+    GNUNET_asprintf (&to_match, "%s%llu", unix_domain_socket, port);
+
+  while (pos != NULL)
+    {
+      if (0 == strcmp (to_match, pos->unique_string))
+        {
+          GNUNET_free (unix_domain_socket);
+          GNUNET_free (to_match);
+          return GNUNET_YES;
+        }
+      pos = pos->next;
+    }
+  pos = GNUNET_malloc (sizeof (struct DuplicateStats));
+  pos->unique_string = to_match;
+  pos->next = *stats_list;
+  *stats_list = pos;
+  GNUNET_free (unix_domain_socket);
+  return GNUNET_NO;
+}
+
+/**
+ * Iterate over all (running) peers in the peer group, retrieve
+ * all statistics from each.
+ */
+void
+GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
+                               GNUNET_STATISTICS_Callback cont,
+                               GNUNET_TESTING_STATISTICS_Iterator proc,
+                               void *cls)
+{
+  struct StatsIterateContext *stats_context;
+  struct StatsCoreContext *core_ctx;
+  unsigned int i;
+  unsigned int total_count;
+  struct DuplicateStats *stats_list;
+  struct DuplicateStats *pos;
+  stats_list = NULL;
+
+  /* Allocate a single stats iteration context */
+  stats_context = GNUNET_malloc (sizeof (struct StatsIterateContext));
+  stats_context->cont = cont;
+  stats_context->proc = proc;
+  stats_context->cls = cls;
+  total_count = 0;
+
+  for (i = 0; i < pg->total; i++)
+    {
+      if ((pg->peers[i].daemon->running == GNUNET_YES)
+          && (GNUNET_NO ==
+              stats_check_existing (pg, &pg->peers[i], &stats_list)))
+        {
+          /* Allocate one core context per core we need to connect to */
+          core_ctx = GNUNET_malloc (sizeof (struct StatsCoreContext));
+          core_ctx->daemon = pg->peers[i].daemon;
+          /* Set back pointer to topology iteration context */
+          core_ctx->iter_context = stats_context;
+          GNUNET_SCHEDULER_add_now (&schedule_get_statistics, core_ctx);
+          total_count++;
+        }
+    }
+
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+              "Retrieving stats from %u total instances.\n", total_count);
+  stats_context->total = total_count;
+  if (stats_list != NULL)
+    {
+      pos = stats_list;
+      while (pos != NULL)
+        {
+          GNUNET_free (pos->unique_string);
+          stats_list = pos->next;
+          GNUNET_free (pos);
+          pos = stats_list->next;
+        }
     }
     }
+  return;
 }
 
 /**
 }
 
 /**
@@ -2641,114 +3735,135 @@ perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
  * @param topology which topology to connect the peers in
  * @param options options for connecting the topology
  * @param option_modifier modifier for options that take a parameter
  * @param topology which topology to connect the peers in
  * @param options options for connecting the topology
  * @param option_modifier modifier for options that take a parameter
+ * @param notify_callback notification to be called once all connections completed
+ * @param notify_cls closure for notification callback
+ *
  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
  */
 int
 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                  enum GNUNET_TESTING_Topology topology,
                                  enum GNUNET_TESTING_TopologyOption options,
  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
  */
 int
 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                  enum GNUNET_TESTING_Topology topology,
                                  enum GNUNET_TESTING_TopologyOption options,
-                                 double option_modifier)
+                                 double option_modifier,
+                                 GNUNET_TESTING_NotifyCompletion
+                                 notify_callback, void *notify_cls)
 {
   switch (topology)
 {
   switch (topology)
-      {
-      case GNUNET_TESTING_TOPOLOGY_CLIQUE:
+    {
+    case GNUNET_TESTING_TOPOLOGY_CLIQUE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating clique CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating clique CONNECT topology\n"));
 #endif
-        create_clique (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
+      create_clique (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (ring) CONNECT topology\n"));
 #endif
-        create_small_world_ring (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
+      create_small_world_ring (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating small world (2d-torus) CONNECT topology\n"));
 #endif
-        create_small_world (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_RING:
+      create_small_world (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_RING:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating ring CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating ring CONNECT topology\n"));
 #endif
-        create_ring (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
+      create_ring (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating 2d torus CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating 2d torus CONNECT topology\n"));
 #endif
-        create_2d_torus (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
+      create_2d_torus (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Erdos-Renyi CONNECT topology\n"));
 #endif
-        create_erdos_renyi (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_INTERNAT:
+      create_erdos_renyi (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_INTERNAT:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating InterNAT CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating InterNAT CONNECT topology\n"));
 #endif
-        create_nated_internet (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
+      create_nated_internet (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Scale Free CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating Scale Free CONNECT topology\n"));
 #endif
-        create_scale_free (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_LINE:
+      create_scale_free (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_LINE:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating straight line CONNECT topology\n"));
 #endif
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating straight line CONNECT topology\n"));
 #endif
-        create_line (pg, &add_actual_connections);
-        break;
-      case GNUNET_TESTING_TOPOLOGY_NONE:
+      create_line (pg, &add_actual_connections);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_NONE:
 #if VERBOSE_TOPOLOGY
 #if VERBOSE_TOPOLOGY
-        GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _("Creating no CONNECT topology\n"));
 #endif
                   _("Creating no CONNECT topology\n"));
 #endif
-        copy_allowed_topology(pg);
-        break;
-      default:
-        GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-                  _("Unknown topology specification, can't connect peers!\n"));
-        return GNUNET_SYSERR;
-      }
+      copy_allowed_topology (pg);
+      break;
+    default:
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Unknown topology specification, can't connect peers!\n"));
+      return GNUNET_SYSERR;
+    }
 
   switch (options)
     {
     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 
   switch (options)
     {
     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
+                  _
+                  ("Connecting random subset (%'.2f percent) of possible peers\n"),
+                  100 * option_modifier);
 #endif
 #endif
-      choose_random_connections(pg, option_modifier);
+      choose_random_connections (pg, option_modifier);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
+                  _("Connecting a minimum of %u peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
 #endif
 #endif
-      choose_minimum(pg, (unsigned int)option_modifier);
+      choose_minimum (pg, (unsigned int) option_modifier);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
 #if VERBOSE_TOPOLOGY
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-                  _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
+                  _
+                  ("Using DFS to connect a minimum of %u peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
+#endif
+      perform_dfs (pg, (int) option_modifier);
+      break;
+    case GNUNET_TESTING_TOPOLOGY_OPTION_ADD_CLOSEST:
+#if VERBOSE_TOPOLOGY
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  _
+                  ("Finding additional %u closest peers each (if possible)\n"),
+                  (unsigned int) option_modifier);
 #endif
 #endif
-      perform_dfs(pg, (int)option_modifier);
+      add_closest (pg, (unsigned int) option_modifier,
+                   &add_actual_connections);
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
       break;
       break;
     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
       break;
@@ -2758,7 +3873,200 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     }
 
       break;
     }
 
-  return connect_topology(pg);
+  return connect_topology (pg, notify_callback, notify_cls);
+}
+
+/**
+ * Callback that is called whenever a hostkey is generated
+ * for a peer.  Call the real callback and decrement the
+ * starting counter for the peergroup.
+ *
+ * @param cls closure
+ * @param id identifier for the daemon, NULL on error
+ * @param d handle for the daemon
+ * @param emsg error message (NULL on success)
+ */
+static void
+internal_hostkey_callback (void *cls,
+                           const struct GNUNET_PeerIdentity *id,
+                           struct GNUNET_TESTING_Daemon *d, const char *emsg)
+{
+  struct InternalStartContext *internal_context = cls;
+  internal_context->peer->pg->starting--;
+  internal_context->peer->pg->started++;
+  if (internal_context->hostkey_callback != NULL)
+    internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
+                                        emsg);
+  else if (internal_context->peer->pg->started ==
+           internal_context->peer->pg->total)
+    {
+      internal_context->peer->pg->started = 0;  /* Internal startup may use this counter! */
+      GNUNET_TESTING_daemons_continue_startup (internal_context->peer->pg);
+    }
+}
+
+/**
+ * Callback that is called whenever a peer has finished starting.
+ * Call the real callback and decrement the starting counter
+ * for the peergroup.
+ *
+ * @param cls closure
+ * @param id identifier for the daemon, NULL on error
+ * @param cfg config
+ * @param d handle for the daemon
+ * @param emsg error message (NULL on success)
+ */
+static void
+internal_startup_callback (void *cls,
+                           const struct GNUNET_PeerIdentity *id,
+                           const struct GNUNET_CONFIGURATION_Handle *cfg,
+                           struct GNUNET_TESTING_Daemon *d, const char *emsg)
+{
+  struct InternalStartContext *internal_context = cls;
+  internal_context->peer->pg->starting--;
+  if (internal_context->start_cb != NULL)
+    internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
+                                emsg);
+}
+
+static void
+internal_continue_startup (void *cls,
+                           const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct InternalStartContext *internal_context = cls;
+
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    {
+      return;
+    }
+
+  if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
+    {
+      internal_context->peer->pg->starting++;
+      GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
+    }
+  else
+    {
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &internal_continue_startup,
+                                    internal_context);
+    }
+}
+
+
+/**
+ * Callback for informing us about a successful
+ * or unsuccessful churn start call.
+ *
+ * @param cls a ChurnContext
+ * @param id the peer identity of the started peer
+ * @param cfg the handle to the configuration of the peer
+ * @param d handle to the daemon for the peer
+ * @param emsg NULL on success, non-NULL on failure
+ *
+ */
+void
+churn_start_callback (void *cls,
+                      const struct GNUNET_PeerIdentity *id,
+                      const struct GNUNET_CONFIGURATION_Handle *cfg,
+                      struct GNUNET_TESTING_Daemon *d, const char *emsg)
+{
+  struct ChurnRestartContext *startup_ctx = cls;
+  struct ChurnContext *churn_ctx = startup_ctx->churn_ctx;
+
+  unsigned int total_left;
+  char *error_message;
+
+  error_message = NULL;
+  if (emsg != NULL)
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Churn stop callback failed with error `%s'\n", emsg);
+      churn_ctx->num_failed_start++;
+    }
+  else
+    {
+      churn_ctx->num_to_start--;
+    }
+
+#if DEBUG_CHURN
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Started peer, %d left.\n", churn_ctx->num_to_start);
+#endif
+
+  total_left =
+    (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
+    (churn_ctx->num_to_start - churn_ctx->num_failed_start);
+
+  if (total_left == 0)
+    {
+      if ((churn_ctx->num_failed_stop > 0)
+          || (churn_ctx->num_failed_start > 0))
+        GNUNET_asprintf (&error_message,
+                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
+                         churn_ctx->num_failed_start,
+                         churn_ctx->num_failed_stop);
+      churn_ctx->cb (churn_ctx->cb_cls, error_message);
+      GNUNET_free_non_null (error_message);
+      GNUNET_free (churn_ctx);
+      GNUNET_free (startup_ctx);
+    }
+}
+
+
+static void
+schedule_churn_restart (void *cls,
+                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerRestartContext *peer_restart_ctx = cls;
+  struct ChurnRestartContext *startup_ctx =
+    peer_restart_ctx->churn_restart_ctx;
+
+  if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_churn_restart, peer_restart_ctx);
+  else
+    {
+      GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon,
+                                           startup_ctx->timeout,
+                                           &churn_start_callback,
+                                           startup_ctx);
+      GNUNET_free (peer_restart_ctx);
+    }
+}
+
+static void
+internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct InternalStartContext *internal_context = cls;
+
+  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    {
+      return;
+    }
+
+  if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
+    {
+      internal_context->peer->pg->starting++;
+      internal_context->peer->daemon =
+        GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
+                                     internal_context->timeout,
+                                     internal_context->hostname,
+                                     internal_context->username,
+                                     internal_context->sshport,
+                                     &internal_hostkey_callback,
+                                     internal_context,
+                                     &internal_startup_callback,
+                                     internal_context);
+    }
+  else
+    {
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                    &internal_start, internal_context);
+    }
 }
 
 /**
 }
 
 /**
@@ -2769,23 +4077,25 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
  *
  */
 void
  *
  */
 void
-GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
+GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int i;
 
 {
   unsigned int i;
 
+  pg->starting = 0;
   for (i = 0; i < pg->total; i++)
     {
   for (i = 0; i < pg->total; i++)
     {
-      GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
+      GNUNET_SCHEDULER_add_now (&internal_continue_startup,
+                                &pg->peers[i].internal_context);
+      //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
     }
 }
 
 /**
     }
 }
 
 /**
- * Start count gnunetd processes with the same set of transports and
+ * Start count gnunet instances with the same set of transports and
  * applications.  The port numbers (any option called "PORT") will be
  * adjusted to ensure that no two peers running on the same system
  * have the same port(s) in their respective configurations.
  *
  * applications.  The port numbers (any option called "PORT") will be
  * adjusted to ensure that no two peers running on the same system
  * have the same port(s) in their respective configurations.
  *
- * @param sched scheduler to use
  * @param cfg configuration template to use
  * @param total number of daemons to start
  * @param timeout total time allowed for peers to start
  * @param cfg configuration template to use
  * @param total number of daemons to start
  * @param timeout total time allowed for peers to start
@@ -2798,28 +4108,31 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
  * @param cb_cls closure for cb
  * @param connect_callback function to call each time two hosts are connected
  * @param connect_callback_cls closure for connect_callback
  * @param cb_cls closure for cb
  * @param connect_callback function to call each time two hosts are connected
  * @param connect_callback_cls closure for connect_callback
- * @param hostnames space-separated list of hostnames to use; can be NULL (to run
- *        everything on localhost).
+ * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
+ *
  * @return NULL on error, otherwise handle to control peer group
  */
 struct GNUNET_TESTING_PeerGroup *
  * @return NULL on error, otherwise handle to control peer group
  */
 struct GNUNET_TESTING_PeerGroup *
-GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
-                              const struct GNUNET_CONFIGURATION_Handle *cfg,
+GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                               unsigned int total,
                               struct GNUNET_TIME_Relative timeout,
                               unsigned int total,
                               struct GNUNET_TIME_Relative timeout,
-                              GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
-                              void *hostkey_cls,
+                              GNUNET_TESTING_NotifyHostkeyCreated
+                              hostkey_callback, void *hostkey_cls,
                               GNUNET_TESTING_NotifyDaemonRunning cb,
                               void *cb_cls,
                               GNUNET_TESTING_NotifyConnection
                               connect_callback, void *connect_callback_cls,
                               GNUNET_TESTING_NotifyDaemonRunning cb,
                               void *cb_cls,
                               GNUNET_TESTING_NotifyConnection
                               connect_callback, void *connect_callback_cls,
-                              const char *hostnames)
+                              const struct GNUNET_TESTING_Host *hostnames)
 {
   struct GNUNET_TESTING_PeerGroup *pg;
 {
   struct GNUNET_TESTING_PeerGroup *pg;
-  const char *rpos;
+  const struct GNUNET_TESTING_Host *hostpos;
+#if 0
   char *pos;
   char *pos;
+  const char *rpos;
   char *start;
   char *start;
+#endif
   const char *hostname;
   const char *hostname;
+  const char *username;
   char *baseservicehome;
   char *newservicehome;
   char *tmpdir;
   char *baseservicehome;
   char *newservicehome;
   char *tmpdir;
@@ -2827,7 +4140,9 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
   unsigned int off;
   unsigned int hostcnt;
   uint16_t minport;
   unsigned int off;
   unsigned int hostcnt;
   uint16_t minport;
+  uint16_t sshport;
   uint32_t upnum;
   uint32_t upnum;
+  uint32_t fdnum;
 
   if (0 == total)
     {
 
   if (0 == total)
     {
@@ -2835,26 +4150,55 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       return NULL;
     }
   upnum = 0;
       return NULL;
     }
   upnum = 0;
+  fdnum = 0;
   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
-  pg->sched = sched;
   pg->cfg = cfg;
   pg->cfg = cfg;
-  pg->cb = cb;
-  pg->cb_cls = cb_cls;
   pg->notify_connection = connect_callback;
   pg->notify_connection_cls = connect_callback_cls;
   pg->total = total;
   pg->notify_connection = connect_callback;
   pg->notify_connection_cls = connect_callback_cls;
   pg->total = total;
-  pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
+  pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
   if (NULL != hostnames)
     {
   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
   if (NULL != hostnames)
     {
+      off = 0;
+      hostpos = hostnames;
+      while (hostpos != NULL)
+        {
+          hostpos = hostpos->next;
+          off++;
+        }
+      pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
+      off = 0;
+
+      hostpos = hostnames;
+      while (hostpos != NULL)
+        {
+          pg->hosts[off].minport = LOW_PORT;
+          pg->hosts[off].hostname = GNUNET_strdup (hostpos->hostname);
+          if (hostpos->username != NULL)
+            pg->hosts[off].username = GNUNET_strdup (hostpos->username);
+          pg->hosts[off].sshport = hostpos->port;
+          hostpos = hostpos->next;
+          off++;
+        }
+
+      if (off == 0)
+        {
+          pg->hosts = NULL;
+        }
+      hostcnt = off;
+      minport = 0;
+      pg->num_hosts = off;
+
+#if NO_LL
       off = 2;
       /* skip leading spaces */
       off = 2;
       /* skip leading spaces */
-      while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
+      while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
         hostnames++;
       rpos = hostnames;
       while ('\0' != *rpos)
         {
         hostnames++;
       rpos = hostnames;
       while ('\0' != *rpos)
         {
-          if (isspace ( (unsigned char) *rpos))
+          if (isspace ((unsigned char) *rpos))
             off++;
           rpos++;
         }
             off++;
           rpos++;
         }
@@ -2864,7 +4208,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       pos = start;
       while ('\0' != *pos)
         {
       pos = start;
       while ('\0' != *pos)
         {
-          if (isspace ( (unsigned char) *pos))
+          if (isspace ((unsigned char) *pos))
             {
               *pos = '\0';
               if (strlen (start) > 0)
             {
               *pos = '\0';
               if (strlen (start) > 0)
@@ -2889,6 +4233,7 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
         }
       hostcnt = off;
       minport = 0;              /* make gcc happy */
         }
       hostcnt = off;
       minport = 0;              /* make gcc happy */
+#endif
     }
   else
     {
     }
   else
     {
@@ -2900,18 +4245,18 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
       if (hostcnt > 0)
         {
           hostname = pg->hosts[off % hostcnt].hostname;
       if (hostcnt > 0)
         {
           hostname = pg->hosts[off % hostcnt].hostname;
-          pcfg = make_config (cfg, 
-                             &pg->hosts[off % hostcnt].minport,
-                             &upnum,
-                             hostname);
+          username = pg->hosts[off % hostcnt].username;
+          sshport = pg->hosts[off % hostcnt].sshport;
+          pcfg = make_config (cfg,
+                              &pg->hosts[off % hostcnt].minport,
+                              &upnum, hostname, &fdnum);
         }
       else
         {
           hostname = NULL;
         }
       else
         {
           hostname = NULL;
-          pcfg = make_config (cfg,
-                             &minport,
-                             &upnum,
-                             hostname);
+          username = NULL;
+          sshport = 0;
+          pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
         }
 
       if (NULL == pcfg)
         }
 
       if (NULL == pcfg)
@@ -2927,38 +4272,42 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
                                                  &baseservicehome))
         {
           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
                                                  &baseservicehome))
         {
-         GNUNET_asprintf (&newservicehome,
-                          "%s/%d/", baseservicehome, off);
-         GNUNET_free (baseservicehome);
+          GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
+          GNUNET_free (baseservicehome);
         }
       else
         {
           tmpdir = getenv ("TMPDIR");
           tmpdir = tmpdir ? tmpdir : "/tmp";
         }
       else
         {
           tmpdir = getenv ("TMPDIR");
           tmpdir = tmpdir ? tmpdir : "/tmp";
-         GNUNET_asprintf (&newservicehome,
-                          "%s/%s/%d/",
-                          tmpdir,
-                          "gnunet-testing-test-test", off);
+          GNUNET_asprintf (&newservicehome,
+                           "%s/%s/%d/",
+                           tmpdir, "gnunet-testing-test-test", off);
         }
       GNUNET_CONFIGURATION_set_value_string (pcfg,
                                              "PATHS",
                                              "SERVICEHOME", newservicehome);
       GNUNET_free (newservicehome);
       pg->peers[off].cfg = pcfg;
         }
       GNUNET_CONFIGURATION_set_value_string (pcfg,
                                              "PATHS",
                                              "SERVICEHOME", newservicehome);
       GNUNET_free (newservicehome);
       pg->peers[off].cfg = pcfg;
-      pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
-      pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
-      pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
+      pg->peers[off].allowed_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
+      pg->peers[off].connect_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
+      pg->peers[off].blacklisted_peers =
+        GNUNET_CONTAINER_multihashmap_create (total);
       pg->peers[off].pg = pg;
       pg->peers[off].pg = pg;
-      pg->peers[off].daemon = GNUNET_TESTING_daemon_start (sched,
-                                                           pcfg,
-                                                           timeout,
-                                                           hostname,
-                                                           hostkey_callback,
-                                                           hostkey_cls,
-                                                           cb, cb_cls);
-      if (NULL == pg->peers[off].daemon)
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    _("Could not start peer number %u!\n"), off);
+
+      pg->peers[off].internal_context.peer = &pg->peers[off];
+      pg->peers[off].internal_context.timeout = timeout;
+      pg->peers[off].internal_context.hostname = hostname;
+      pg->peers[off].internal_context.username = username;
+      pg->peers[off].internal_context.sshport = sshport;
+      pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
+      pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
+      pg->peers[off].internal_context.start_cb = cb;
+      pg->peers[off].internal_context.start_cb_cls = cb_cls;
+
+      GNUNET_SCHEDULER_add_now (&internal_start,
+                                &pg->peers[off].internal_context);
 
     }
   return pg;
 
     }
   return pg;
@@ -2969,7 +4318,8 @@ GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
  * offsetting operation.
  */
 struct GNUNET_TESTING_Daemon *
  * offsetting operation.
  */
 struct GNUNET_TESTING_Daemon *
-GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
+GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg,
+                           unsigned int position)
 {
   if (position < pg->total)
     return pg->peers[position].daemon;
 {
   if (position < pg->total)
     return pg->peers[position].daemon;
@@ -2977,6 +4327,32 @@ GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int pos
     return NULL;
 }
 
     return NULL;
 }
 
+/*
+ * Get a daemon by peer identity, so callers can
+ * retrieve the daemon without knowing it's offset.
+ *
+ * @param pg the peer group to retrieve the daemon from
+ * @param peer_id the peer identity of the daemon to retrieve
+ *
+ * @return the daemon on success, or NULL if no such peer identity is found
+ */
+struct GNUNET_TESTING_Daemon *
+GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg,
+                                 struct GNUNET_PeerIdentity *peer_id)
+{
+  unsigned int i;
+
+  for (i = 0; i < pg->total; i++)
+    {
+      if (0 ==
+          memcmp (&pg->peers[i].daemon->id, peer_id,
+                  sizeof (struct GNUNET_PeerIdentity)))
+        return pg->peers[i].daemon;
+    }
+
+  return NULL;
+}
+
 /**
  * Prototype of a function that will be called when a
  * particular operation was completed the testing library.
 /**
  * Prototype of a function that will be called when a
  * particular operation was completed the testing library.
@@ -2987,11 +4363,11 @@ GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int pos
  * @param d handle to the daemon that was restarted
  * @param emsg NULL on success
  */
  * @param d handle to the daemon that was restarted
  * @param emsg NULL on success
  */
-void restart_callback (void *cls,
-                       const struct GNUNET_PeerIdentity *id,
-                       const struct GNUNET_CONFIGURATION_Handle *cfg,
-                       struct GNUNET_TESTING_Daemon *d,
-                       const char *emsg)
+void
+restart_callback (void *cls,
+                  const struct GNUNET_PeerIdentity *id,
+                  const struct GNUNET_CONFIGURATION_Handle *cfg,
+                  struct GNUNET_TESTING_Daemon *d, const char *emsg)
 {
   struct RestartContext *restart_context = cls;
 
 {
   struct RestartContext *restart_context = cls;
 
@@ -3006,13 +4382,16 @@ void restart_callback (void *cls,
 
   if (restart_context->peers_restarted == restart_context->peer_group->total)
     {
 
   if (restart_context->peers_restarted == restart_context->peer_group->total)
     {
-      restart_context->callback(restart_context->callback_cls, NULL);
-      GNUNET_free(restart_context);
+      restart_context->callback (restart_context->callback_cls, NULL);
+      GNUNET_free (restart_context);
     }
     }
-  else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
+  else if (restart_context->peers_restart_failed +
+           restart_context->peers_restarted ==
+           restart_context->peer_group->total)
     {
     {
-      restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
-      GNUNET_free(restart_context);
+      restart_context->callback (restart_context->callback_cls,
+                                 "Failed to restart peers!");
+      GNUNET_free (restart_context);
     }
 
 }
     }
 
 }
@@ -3028,15 +4407,18 @@ void restart_callback (void *cls,
 void
 churn_stop_callback (void *cls, const char *emsg)
 {
 void
 churn_stop_callback (void *cls, const char *emsg)
 {
-  struct ChurnContext *churn_ctx = cls;
+  struct ShutdownContext *shutdown_ctx = cls;
+  struct ChurnContext *churn_ctx = shutdown_ctx->cb_cls;
   unsigned int total_left;
   char *error_message;
 
   error_message = NULL;
   unsigned int total_left;
   char *error_message;
 
   error_message = NULL;
+  shutdown_ctx->outstanding--;
+
   if (emsg != NULL)
     {
   if (emsg != NULL)
     {
-      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-                "Churn stop callback failed with error `%s'\n", emsg);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Churn stop callback failed with error `%s'\n", emsg);
       churn_ctx->num_failed_stop++;
     }
   else
       churn_ctx->num_failed_stop++;
     }
   else
@@ -3045,84 +4427,86 @@ churn_stop_callback (void *cls, const char *emsg)
     }
 
 #if DEBUG_CHURN
     }
 
 #if DEBUG_CHURN
-  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
-            "Stopped peer, %d left.\n", 
-            churn_ctx->num_to_stop);
+  GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              "Stopped peer, %d left.\n", churn_ctx->num_to_stop);
 #endif
 #endif
-  total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
+  total_left =
+    (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
+    (churn_ctx->num_to_start - churn_ctx->num_failed_start);
 
   if (total_left == 0)
 
   if (total_left == 0)
-  {
-    if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
-      {
-        GNUNET_asprintf(&error_message, 
-                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!", 
-                       churn_ctx->num_failed_start, 
-                       churn_ctx->num_failed_stop);
-      }
-    churn_ctx->cb(churn_ctx->cb_cls, error_message);
-    GNUNET_free_non_null(error_message);
-    GNUNET_free(churn_ctx);
-  }
+    {
+      if ((churn_ctx->num_failed_stop > 0)
+          || (churn_ctx->num_failed_start > 0))
+        {
+          GNUNET_asprintf (&error_message,
+                           "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
+                           churn_ctx->num_failed_start,
+                           churn_ctx->num_failed_stop);
+        }
+      churn_ctx->cb (churn_ctx->cb_cls, error_message);
+      GNUNET_free_non_null (error_message);
+      GNUNET_free (churn_ctx);
+      GNUNET_free (shutdown_ctx);
+    }
 }
 
 /**
 }
 
 /**
- * Callback for informing us about a successful
- * or unsuccessful churn start call.
+ * Count the number of running peers.
  *
  *
- * @param cls a ChurnContext
- * @param id the peer identity of the started peer
- * @param cfg the handle to the configuration of the peer
- * @param d handle to the daemon for the peer
- * @param emsg NULL on success, non-NULL on failure
+ * @param pg handle for the peer group
  *
  *
+ * @return the number of currently running peers in the peer group
  */
  */
-void
-churn_start_callback (void *cls,
-                      const struct GNUNET_PeerIdentity *id,
-                      const struct GNUNET_CONFIGURATION_Handle *cfg,
-                      struct GNUNET_TESTING_Daemon *d,
-                      const char *emsg)
+unsigned int
+GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
 {
 {
-  struct ChurnContext *churn_ctx = cls;
-  unsigned int total_left;
-  char *error_message;
-
-  error_message = NULL;
-  if (emsg != NULL)
+  unsigned int i;
+  unsigned int running = 0;
+  for (i = 0; i < pg->total; i++)
     {
     {
-      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 
-                 "Churn stop callback failed with error `%s'\n",
-                 emsg);
-      churn_ctx->num_failed_start++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert (running != -1);
+          running++;
+        }
     }
     }
+  return running;
+}
+
+/**
+ * Task to rate limit the number of outstanding peer shutdown
+ * requests.  This is necessary for making sure we don't do
+ * too many ssh connections at once, but is generally nicer
+ * to any system as well (graduated task starts, as opposed
+ * to calling gnunet-arm N times all at once).
+ */
+static void
+schedule_churn_shutdown_task (void *cls,
+                              const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerShutdownContext *peer_shutdown_ctx = cls;
+  struct ShutdownContext *shutdown_ctx;
+
+  GNUNET_assert (peer_shutdown_ctx != NULL);
+  shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
+  GNUNET_assert (shutdown_ctx != NULL);
+
+  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_churn_shutdown_task,
+                                  peer_shutdown_ctx);
   else
     {
   else
     {
-      churn_ctx->num_to_start--;
+      shutdown_ctx->outstanding++;
+      GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
+                                  shutdown_ctx->timeout, shutdown_ctx->cb,
+                                  shutdown_ctx, GNUNET_NO, GNUNET_YES);
+      GNUNET_free (peer_shutdown_ctx);
     }
     }
-  
-#if DEBUG_CHURN
-  GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
-            "Started peer, %d left.\n", 
-            churn_ctx->num_to_start);
-#endif
-
-  total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
-
-  if (total_left == 0)
-  {
-    if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
-      GNUNET_asprintf(&error_message, 
-                     "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!", 
-                     churn_ctx->num_failed_start,
-                     churn_ctx->num_failed_stop);
-    churn_ctx->cb(churn_ctx->cb_cls, error_message);
-    GNUNET_free_non_null(error_message);
-    GNUNET_free(churn_ctx);
-  }
 }
 
 }
 
-
 /**
  * Simulate churn by stopping some peers (and possibly
  * re-starting others if churn is called multiple times).  This
 /**
  * Simulate churn by stopping some peers (and possibly
  * re-starting others if churn is called multiple times).  This
@@ -3152,8 +4536,15 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
                               void *cb_cls)
 {
   struct ChurnContext *churn_ctx;
                               void *cb_cls)
 {
   struct ChurnContext *churn_ctx;
+  struct ShutdownContext *shutdown_ctx;
+  struct PeerShutdownContext *peer_shutdown_ctx;
+  struct PeerRestartContext *peer_restart_ctx;
+  struct ChurnRestartContext *churn_startup_ctx;
+
   unsigned int running;
   unsigned int stopped;
   unsigned int running;
   unsigned int stopped;
+  unsigned int total_running;
+  unsigned int total_stopped;
   unsigned int i;
   unsigned int *running_arr;
   unsigned int *stopped_arr;
   unsigned int i;
   unsigned int *running_arr;
   unsigned int *stopped_arr;
@@ -3163,98 +4554,147 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
   running = 0;
   stopped = 0;
 
   running = 0;
   stopped = 0;
 
-  if ((von == 0) && (voff == 0)) /* No peers at all? */
+  if ((von == 0) && (voff == 0))        /* No peers at all? */
     {
     {
-      cb(cb_cls, NULL);
+      cb (cb_cls, NULL);
       return;
     }
 
   for (i = 0; i < pg->total; i++)
       return;
     }
 
   for (i = 0; i < pg->total; i++)
-  {
-    if (pg->peers[i].daemon->running == GNUNET_YES)
-    {
-      GNUNET_assert(running != -1);
-      running++;
-    }
-    else
     {
     {
-      GNUNET_assert(stopped != -1);
-      stopped++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert (running != -1);
+          running++;
+        }
+      else
+        {
+          GNUNET_assert (stopped != -1);
+          stopped++;
+        }
     }
     }
-  }
 
   if (voff > running)
 
   if (voff > running)
-  {
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
-    cb(cb_cls, "Trying to stop more peers than are currently running!");
-    return;
-  }
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Trying to stop more peers than are currently running!\n");
+      cb (cb_cls, "Trying to stop more peers than are currently running!");
+      return;
+    }
 
   if (von > stopped)
 
   if (von > stopped)
-  {
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
-    cb(cb_cls, "Trying to start more peers than are currently stopped!");
-    return;
-  }
+    {
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                  "Trying to start more peers than are currently stopped!\n");
+      cb (cb_cls, "Trying to start more peers than are currently stopped!");
+      return;
+    }
+
+  churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
+
+  running_arr = NULL;
+  if (running > 0)
+    running_arr = GNUNET_malloc (running * sizeof (unsigned int));
 
 
-  churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
-  running_arr = GNUNET_malloc(running * sizeof(unsigned int));
-  stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
+  stopped_arr = NULL;
+  if (stopped > 0)
+    stopped_arr = GNUNET_malloc (stopped * sizeof (unsigned int));
 
   running_permute = NULL;
   stopped_permute = NULL;
 
   if (running > 0)
 
   running_permute = NULL;
   stopped_permute = NULL;
 
   if (running > 0)
-    running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
+    running_permute =
+      GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, running);
   if (stopped > 0)
   if (stopped > 0)
-    stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
+    stopped_permute =
+      GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, stopped);
 
 
+  total_running = running;
+  total_stopped = stopped;
   running = 0;
   stopped = 0;
 
   churn_ctx->num_to_start = von;
   churn_ctx->num_to_stop = voff;
   churn_ctx->cb = cb;
   running = 0;
   stopped = 0;
 
   churn_ctx->num_to_start = von;
   churn_ctx->num_to_stop = voff;
   churn_ctx->cb = cb;
-  churn_ctx->cb_cls = cb_cls;  
+  churn_ctx->cb_cls = cb_cls;
 
   for (i = 0; i < pg->total; i++)
 
   for (i = 0; i < pg->total; i++)
-  {
-    if (pg->peers[i].daemon->running == GNUNET_YES)
     {
     {
-      running_arr[running] = i;
-      running++;
+      if (pg->peers[i].daemon->running == GNUNET_YES)
+        {
+          GNUNET_assert ((running_arr != NULL) && (total_running > running));
+          running_arr[running] = i;
+          running++;
+        }
+      else
+        {
+          GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped));
+          stopped_arr[stopped] = i;
+          stopped++;
+        }
     }
     }
-    else
+
+  GNUNET_assert (running >= voff);
+  if (voff > 0)
     {
     {
-      stopped_arr[stopped] = i;
-      stopped++;
+      shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+      shutdown_ctx->cb = &churn_stop_callback;
+      shutdown_ctx->cb_cls = churn_ctx;
+      shutdown_ctx->total_peers = voff;
+      shutdown_ctx->timeout = timeout;
     }
     }
-  }
 
   for (i = 0; i < voff; i++)
 
   for (i = 0; i < voff; i++)
-  {
+    {
 #if DEBUG_CHURN
 #if DEBUG_CHURN
-    GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n",
+                  running_permute[i]);
 #endif
 #endif
-    GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
-                               timeout, 
-                               &churn_stop_callback, churn_ctx, 
-                               GNUNET_NO, GNUNET_YES);
-  }
+      GNUNET_assert (running_arr != NULL);
+      peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
+      peer_shutdown_ctx->daemon =
+        pg->peers[running_arr[running_permute[i]]].daemon;
+      peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
+      GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task,
+                                peer_shutdown_ctx);
+
+      /*
+         GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
+         timeout, 
+         &churn_stop_callback, churn_ctx, 
+         GNUNET_NO, GNUNET_YES); */
+    }
 
 
+  GNUNET_assert (stopped >= von);
+  if (von > 0)
+    {
+      churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
+      churn_startup_ctx->churn_ctx = churn_ctx;
+      churn_startup_ctx->timeout = timeout;
+    }
   for (i = 0; i < von; i++)
     {
 #if DEBUG_CHURN
   for (i = 0; i < von; i++)
     {
 #if DEBUG_CHURN
-      GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
+      GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n",
+                  stopped_permute[i]);
 #endif
 #endif
-      GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon, 
-                                         timeout, &churn_start_callback, churn_ctx);
-  }
+      GNUNET_assert (stopped_arr != NULL);
+      peer_restart_ctx = GNUNET_malloc (sizeof (struct PeerRestartContext));
+      peer_restart_ctx->churn_restart_ctx = churn_startup_ctx;
+      peer_restart_ctx->daemon =
+        pg->peers[stopped_arr[stopped_permute[i]]].daemon;
+      GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx);
+      /*
+         GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon, 
+         timeout, &churn_start_callback, churn_ctx); */
+    }
 
 
-  GNUNET_free(running_arr);
-  GNUNET_free(stopped_arr);
-  GNUNET_free_non_null(running_permute);
-  GNUNET_free_non_null(stopped_permute);
+  GNUNET_free_non_null (running_arr);
+  GNUNET_free_non_null (stopped_arr);
+  GNUNET_free_non_null (running_permute);
+  GNUNET_free_non_null (stopped_permute);
 }
 
 
 }
 
 
@@ -3266,14 +4706,16 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
  * @param callback_cls closure for the callback function
  */
 void
  * @param callback_cls closure for the callback function
  */
 void
-GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyCompletion callback, void *callback_cls)
+GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
+                                GNUNET_TESTING_NotifyCompletion callback,
+                                void *callback_cls)
 {
   struct RestartContext *restart_context;
   unsigned int off;
 
   if (pg->total > 0)
     {
 {
   struct RestartContext *restart_context;
   unsigned int off;
 
   if (pg->total > 0)
     {
-      restart_context = GNUNET_malloc(sizeof(struct RestartContext));
+      restart_context = GNUNET_malloc (sizeof (struct RestartContext));
       restart_context->peer_group = pg;
       restart_context->peers_restarted = 0;
       restart_context->callback = callback;
       restart_context->peer_group = pg;
       restart_context->peers_restarted = 0;
       restart_context->callback = callback;
@@ -3281,7 +4723,8 @@ GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TEST
 
       for (off = 0; off < pg->total; off++)
         {
 
       for (off = 0; off < pg->total; off++)
         {
-          GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
+          GNUNET_TESTING_daemon_restart (pg->peers[off].daemon,
+                                         &restart_callback, restart_context);
         }
     }
 }
         }
     }
 }
@@ -3297,82 +4740,172 @@ GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TEST
  * @param cb_cls closure for cb
  */
 void
  * @param cb_cls closure for cb
  */
 void
-GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg, 
-                            unsigned int offset,
-                            int desired_status,
-                            struct GNUNET_TIME_Relative timeout,
-                            GNUNET_TESTING_NotifyCompletion cb,
-                            void *cb_cls)
+GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
+                             unsigned int offset,
+                             int desired_status,
+                             struct GNUNET_TIME_Relative timeout,
+                             GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
 {
 {
+  struct ShutdownContext *shutdown_ctx;
+  struct ChurnRestartContext *startup_ctx;
   struct ChurnContext *churn_ctx;
 
   if (GNUNET_NO == desired_status)
     {
       if (NULL != pg->peers[offset].daemon)
   struct ChurnContext *churn_ctx;
 
   if (GNUNET_NO == desired_status)
     {
       if (NULL != pg->peers[offset].daemon)
-       {
-         churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
-         churn_ctx->num_to_start = 0;
-         churn_ctx->num_to_stop = 1;
-         churn_ctx->cb = cb;
-         churn_ctx->cb_cls = cb_cls;  
-         GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon, 
-                                    timeout, &churn_stop_callback, churn_ctx, 
-                                    GNUNET_NO, GNUNET_YES);     
-       }
+        {
+          shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+          churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
+          churn_ctx->num_to_start = 0;
+          churn_ctx->num_to_stop = 1;
+          churn_ctx->cb = cb;
+          churn_ctx->cb_cls = cb_cls;
+          shutdown_ctx->cb_cls = churn_ctx;
+          GNUNET_TESTING_daemon_stop (pg->peers[offset].daemon,
+                                      timeout, &churn_stop_callback,
+                                      shutdown_ctx, GNUNET_NO, GNUNET_YES);
+        }
     }
   else if (GNUNET_YES == desired_status)
     {
       if (NULL == pg->peers[offset].daemon)
     }
   else if (GNUNET_YES == desired_status)
     {
       if (NULL == pg->peers[offset].daemon)
-       {
-         churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
-         churn_ctx->num_to_start = 1;
-         churn_ctx->num_to_stop = 0;
-         churn_ctx->cb = cb;
-         churn_ctx->cb_cls = cb_cls;  
-         GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon, 
-                                             timeout, &churn_start_callback, churn_ctx);
-       }
+        {
+          startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
+          churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
+          churn_ctx->num_to_start = 1;
+          churn_ctx->num_to_stop = 0;
+          churn_ctx->cb = cb;
+          churn_ctx->cb_cls = cb_cls;
+          startup_ctx->churn_ctx = churn_ctx;
+          GNUNET_TESTING_daemon_start_stopped (pg->peers[offset].daemon,
+                                               timeout, &churn_start_callback,
+                                               startup_ctx);
+        }
     }
   else
     GNUNET_break (0);
 }
 
 
     }
   else
     GNUNET_break (0);
 }
 
 
+/**
+ * Callback for shutting down peers in a peer group.
+ *
+ * @param cls closure (struct ShutdownContext)
+ * @param emsg NULL on success
+ */
+void
+internal_shutdown_callback (void *cls, const char *emsg)
+{
+  struct ShutdownContext *shutdown_ctx = cls;
+
+  shutdown_ctx->outstanding--;
+  if (emsg == NULL)
+    {
+      shutdown_ctx->peers_down++;
+    }
+  else
+    {
+      shutdown_ctx->peers_failed++;
+    }
+
+  if ((shutdown_ctx->cb != NULL)
+      && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed ==
+          shutdown_ctx->total_peers))
+    {
+      if (shutdown_ctx->peers_failed > 0)
+        shutdown_ctx->cb (shutdown_ctx->cb_cls,
+                          "Not all peers successfully shut down!");
+      else
+        shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
+      GNUNET_free (shutdown_ctx);
+    }
+}
+
+
+/**
+ * Task to rate limit the number of outstanding peer shutdown
+ * requests.  This is necessary for making sure we don't do
+ * too many ssh connections at once, but is generally nicer
+ * to any system as well (graduated task starts, as opposed
+ * to calling gnunet-arm N times all at once).
+ */
+static void
+schedule_shutdown_task (void *cls,
+                        const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerShutdownContext *peer_shutdown_ctx = cls;
+  struct ShutdownContext *shutdown_ctx;
+
+  GNUNET_assert (peer_shutdown_ctx != NULL);
+  shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
+  GNUNET_assert (shutdown_ctx != NULL);
+
+  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+    GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                  (GNUNET_TIME_UNIT_MILLISECONDS, 100),
+                                  &schedule_shutdown_task, peer_shutdown_ctx);
+  else
+    {
+      shutdown_ctx->outstanding++;
+      GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
+                                  shutdown_ctx->timeout,
+                                  &internal_shutdown_callback, shutdown_ctx,
+                                  GNUNET_YES, GNUNET_NO);
+      GNUNET_free (peer_shutdown_ctx);
+    }
+}
+
 /**
  * Shutdown all peers started in the given group.
  *
  * @param pg handle to the peer group
  * @param timeout how long to wait for shutdown
 /**
  * Shutdown all peers started in the given group.
  *
  * @param pg handle to the peer group
  * @param timeout how long to wait for shutdown
+ * @param cb callback to notify upon success or failure
+ * @param cb_cls closure for cb
  */
 void
  */
 void
-GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg, 
-                            struct GNUNET_TIME_Relative timeout)
+GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
+                             struct GNUNET_TIME_Relative timeout,
+                             GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
 {
   unsigned int off;
 {
   unsigned int off;
+  struct ShutdownContext *shutdown_ctx;
+  struct PeerShutdownContext *peer_shutdown_ctx;
+
+  GNUNET_assert (pg->total > 0);
+
+  shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
+  shutdown_ctx->cb = cb;
+  shutdown_ctx->cb_cls = cb_cls;
+  shutdown_ctx->total_peers = pg->total;
+  shutdown_ctx->timeout = timeout;
+  /* shtudown_ctx->outstanding = 0; */
 
   for (off = 0; off < pg->total; off++)
     {
 
   for (off = 0; off < pg->total; off++)
     {
-      /* FIXME: should we wait for our continuations to be called
-         here? This would require us to take a continuation as
-         well... */
-
-      if (NULL != pg->peers[off].daemon)
-        GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, NULL, NULL, GNUNET_YES, GNUNET_NO);
+      GNUNET_assert (NULL != pg->peers[off].daemon);
+      peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
+      peer_shutdown_ctx->daemon = pg->peers[off].daemon;
+      peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
+      GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
+      //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
       if (NULL != pg->peers[off].cfg)
         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
       if (pg->peers[off].allowed_peers != NULL)
       if (NULL != pg->peers[off].cfg)
         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
       if (pg->peers[off].allowed_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
       if (pg->peers[off].connect_peers != NULL)
       if (pg->peers[off].connect_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
       if (pg->peers[off].blacklisted_peers != NULL)
       if (pg->peers[off].blacklisted_peers != NULL)
-        GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
+        GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                               peers[off].blacklisted_peers);
     }
   GNUNET_free (pg->peers);
     }
   GNUNET_free (pg->peers);
-  if (NULL != pg->hosts)
+  for (off = 0; off < pg->num_hosts; off++)
     {
     {
-      GNUNET_free (pg->hosts[0].hostname);
-      GNUNET_free (pg->hosts);
+      GNUNET_free (pg->hosts[off].hostname);
+      GNUNET_free_non_null (pg->hosts[off].username);
     }
     }
+  GNUNET_free_non_null (pg->hosts);
   GNUNET_free (pg);
 }
 
   GNUNET_free (pg);
 }