free in proper place
[oweals/gnunet.git] / src / testing / testing_group.c
index 6505ebc4c108034269fe8ffcdd48632ad2abdde6..f9b3cc3fc51eeb8c6dfac8da260999525c387d3e 100644 (file)
@@ -44,7 +44,7 @@
  * enough to not conflict with client-ports (typically starting around
  * 32k).
  */
-#define LOW_PORT 10000
+#define LOW_PORT 12000
 
 /**
  * Highest port used for GNUnet testing.  Should be low enough to not
  */
 #define HIGH_PORT 56000
 
-#define MAX_OUTSTANDING_CONNECTIONS 200
-
 /* Maximum time to delay connect attempt */
 #define MAX_CONNECT_DELAY 300
 
-#define MAX_CONCURRENT_HOSTKEYS 500
-
-#define MAX_CONCURRENT_STARTING 200
-
-#define MAX_CONCURRENT_SHUTDOWN 200
-
-#define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
-
-#define CONNECT_ATTEMPTS 12
-
 /**
  * Which list of peers do we need to modify?
  */
@@ -105,6 +93,11 @@ typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
  */
 struct ChurnContext
 {
+  /**
+   * The peergroup we are dealing with.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
   /**
    * Callback used to notify of churning finished
    */
@@ -168,6 +161,7 @@ struct RestartContext
 
 struct ShutdownContext
 {
+  struct GNUNET_TESTING_PeerGroup *pg;
   /**
    * Total peers to wait for
    */
@@ -348,6 +342,11 @@ struct InternalStartContext
 
 struct ChurnRestartContext
 {
+  /**
+   * PeerGroup that we are working with.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
   /**
    * Number of restarts currently in flight.
    */
@@ -364,6 +363,23 @@ struct ChurnRestartContext
   struct GNUNET_TIME_Relative timeout;
 };
 
+struct OutstandingSSH
+{
+  struct OutstandingSSH *next;
+
+  struct OutstandingSSH *prev;
+
+  /**
+   * Number of current ssh connections.
+   */
+  uint32_t outstanding;
+
+  /**
+   * The hostname of this peer.
+   */
+  const char *hostname;
+};
+
 /**
  * Data we keep per peer.
  */
@@ -493,6 +509,11 @@ struct HostData
 
 struct TopologyIterateContext
 {
+  /**
+   * The peergroup we are working with.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
   /**
    * Callback for notifying of two connected peers.
    */
@@ -521,6 +542,11 @@ struct TopologyIterateContext
 
 struct StatsIterateContext
 {
+  /**
+   * The peergroup that we are dealing with.
+   */
+  struct GNUNET_TESTING_PeerGroup *pg;
+
   /**
    * Continuation to call once all stats information has been retrieved.
    */
@@ -638,10 +664,44 @@ struct GNUNET_TESTING_PeerGroup
    */
   unsigned int started;
 
+  /**
+   * Number of possible connections to peers
+   * at a time.
+   */
+  unsigned int max_outstanding_connections;
+
+  /**
+   * Number of ssh connections to peers (max).
+   */
+  unsigned int max_concurrent_ssh;
+
+  /**
+   * Number of connects we are waiting on, allows us to rate limit
+   * connect attempts.
+   */
+  unsigned int outstanding_connects;
+
+  /**
+   * How many connects have already been scheduled?
+   */
+  unsigned int total_connects_scheduled;
+
   /**
    * Hostkeys loaded from a file.
    */
   char *hostkey_data;
+
+  /**
+   * Head of DLL to keep track of the number of outstanding
+   * ssh connections per peer.
+   */
+  struct OutstandingSSH *ssh_head;
+
+  /**
+   * Tail of DLL to keep track of the number of outstanding
+   * ssh connections per peer.
+   */
+  struct OutstandingSSH *ssh_tail;
 };
 
 struct UpdateContext
@@ -661,11 +721,25 @@ struct ConnectTopologyContext
    */
   unsigned int remaining_connections;
 
+  /**
+   * How many more connections do we need to schedule?
+   */
+  unsigned int remaining_connects_to_schedule;
+
   /**
    * Handle to group of peers.
    */
   struct GNUNET_TESTING_PeerGroup *pg;
 
+  /**
+   * How long to try this connection before timing out.
+   */
+  struct GNUNET_TIME_Relative connect_timeout;
+
+  /**
+   * How many times to retry connecting the two peers.
+   */
+  unsigned int connect_attempts;
 
   /**
    * Temp value set for each iteration.
@@ -836,19 +910,6 @@ uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 }
 #endif
 
-/**
- * Number of connects we are waiting on, allows us to rate limit
- * connect attempts.
- */
-static int outstanding_connects;
-
-/**
- * Number of connects we have scheduled at the same
- * time, the more we already have scheduled the longer
- * we should wait before calling schedule_connect again.
- */
-static int outstanding_scheduled_connects;
-
 /**
  * Get a topology from a string input.
  *
@@ -1096,6 +1157,7 @@ update_config (void *cls,
  * out of "*port" numbers, return NULL.
  *
  * @param cfg template configuration
+ * @param off the current peer offset
  * @param port port numbers to use, update to reflect
  *             port numbers that were used
  * @param upnum number to make unix domain socket names unique
@@ -1107,6 +1169,7 @@ update_config (void *cls,
  */
 static struct GNUNET_CONFIGURATION_Handle *
 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
+             uint32_t off,
              uint16_t * port,
              uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 {
@@ -1114,6 +1177,7 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
   uint16_t orig;
   char *control_host;
   char *allowed_hosts;
+  unsigned long long temp_port;
 
   orig = *port;
   uc.nport = *port;
@@ -1142,20 +1206,28 @@ make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
                                              allowed_hosts);
-      GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "UNIXPATH",
-                                             "");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
                                              "ACCEPT_FROM", allowed_hosts);
-      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport", "UNIXPATH",
-                                             "");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
                                              allowed_hosts);
-      GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "UNIXPATH",
-                                             "");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
                                              "ACCEPT_FROM", allowed_hosts);
-      GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics", "UNIXPATH",
-                                             "");
+
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "UNIXPATH", "");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport", "UNIXPATH", "");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "UNIXPATH", "");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics", "UNIXPATH", "");
+
+
+      if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number(uc.orig, "statistics", "port", &temp_port) &&
+          (temp_port != 0) &&
+          (GNUNET_YES !=
+              GNUNET_CONFIGURATION_get_value_yesno (uc.orig, "testing",
+                                                    "single_statistics_per_host")))
+        {
+          GNUNET_CONFIGURATION_set_value_number (uc.ret, "statistics", "port", temp_port + off);
+        }
+
       GNUNET_free_non_null (control_host);
       GNUNET_free (allowed_hosts);
     }
@@ -1252,6 +1324,7 @@ remove_connections (struct GNUNET_TESTING_PeerGroup *pg,
     break;
   default:
     GNUNET_break(0);
+    return 0;
   }
 
   first_iter = *first_list;
@@ -1365,6 +1438,7 @@ add_connections (struct GNUNET_TESTING_PeerGroup *pg,
     break;
   default:
     GNUNET_break(0);
+    return 0;
   }
 
   add_first = GNUNET_YES;
@@ -2465,11 +2539,15 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
   struct GNUNET_OS_Process **procarr;
   char *arg;
   char *mytemp;
+#if NOT_STUPID
   enum GNUNET_OS_ProcessStatusType type;
   unsigned long return_code;
   int count;
-  int ret;
   int max_wait = 10;
+#endif
+  int ret;
+
+  ret = GNUNET_OK;
 #if OLD
   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
   struct PeerConnection *conn_iter;
@@ -2541,6 +2619,11 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
                                      NULL);
 
+          ret = GNUNET_OS_process_wait(procarr[pg_iter]); /* FIXME: schedule this, throttle! */
+          GNUNET_OS_process_close (procarr[pg_iter]);
+          if (ret != GNUNET_OK)
+            return ret;
+          procarr[pg_iter] = NULL;
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command scp %s %s\n"), mytemp,
@@ -2552,6 +2635,7 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
       GNUNET_free (mytemp);
     }
 
+#if NOT_STUPID
   count = 0;
   ret = GNUNET_SYSERR;
   while ((count < max_wait) && (ret != GNUNET_OK))
@@ -2597,6 +2681,7 @@ create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
 #if VERBOSE_TESTING
   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("Finished copying all friend files!\n"));
+#endif
 #endif
   GNUNET_free (procarr);
   return ret;
@@ -2732,6 +2817,8 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
                                      NULL);
 
+          GNUNET_OS_process_wait(procarr[pg_iter]); /* FIXME: add scheduled blacklist file copy that parallelizes file copying! */
+
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       _("Copying file with command scp %s %s\n"), mytemp,
@@ -2793,6 +2880,41 @@ create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
   return ret;
 }
 
+/* Forward Declaration */
+static void
+schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
+
+/**
+ * Choose a random peer's next connection to create, and
+ * call schedule_connect to set up the connect task.
+ *
+ * @param ct_ctx the overall connection context
+ */
+static void preschedule_connect(struct ConnectTopologyContext *ct_ctx)
+{
+  struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
+  struct PeerConnection *connection_iter;
+  struct ConnectContext *connect_context;
+  uint32_t random_peer;
+
+  if (ct_ctx->remaining_connects_to_schedule == 0)
+    return;
+  random_peer = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, pg->total);
+  while (pg->peers[random_peer].connect_peers_head == NULL)
+    random_peer = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, pg->total);
+
+  connection_iter = pg->peers[random_peer].connect_peers_head;
+  GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Scheduling connection between %d and %d\n", random_peer, connection_iter->index);
+
+  connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
+  connect_context->first = pg->peers[random_peer].daemon;
+  connect_context->second = pg->peers[connection_iter->index].daemon;
+  connect_context->ct_ctx = ct_ctx;
+  GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
+  GNUNET_CONTAINER_DLL_remove(pg->peers[random_peer].connect_peers_head, pg->peers[random_peer].connect_peers_tail, connection_iter);
+  ct_ctx->remaining_connects_to_schedule--;
+}
+
 
 /**
  * Internal notification of a connection, kept so that we can ensure some connections
@@ -2811,7 +2933,7 @@ internal_connect_notify (void *cls,
 {
   struct ConnectTopologyContext *ct_ctx = cls;
   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
-  outstanding_connects--;
+  pg->outstanding_connects--;
   ct_ctx->remaining_connections--;
   if (ct_ctx->remaining_connections == 0)
     {
@@ -2819,6 +2941,8 @@ internal_connect_notify (void *cls,
         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
       GNUNET_free (ct_ctx);
     }
+  else
+    preschedule_connect(ct_ctx);
 
   if (pg->notify_connection != NULL)
     pg->notify_connection (pg->notify_connection_cls, first, second, distance,
@@ -2838,25 +2962,20 @@ static void
 schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct ConnectContext *connect_context = cls;
+  struct GNUNET_TESTING_PeerGroup *pg = connect_context->ct_ctx->pg;
 
   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
     return;
 
-  if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
+  if (pg->outstanding_connects > pg->max_outstanding_connections)
     {
 #if VERBOSE_TESTING > 2
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   _
                   ("Delaying connect, we have too many outstanding connections!\n"));
 #endif
-      if (GNUNET_NO == connect_context->counted)
-        {
-          connect_context->counted = GNUNET_YES;
-          outstanding_scheduled_connects++;
-        }
-      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_add (GNUNET_TIME_relative_multiply
-                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100), GNUNET_TIME_relative_multiply
-                                        (GNUNET_TIME_UNIT_MILLISECONDS, outstanding_scheduled_connects * 2)),
+      GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
+                                    (GNUNET_TIME_UNIT_MILLISECONDS, 100),
                                     &schedule_connect, connect_context);
     }
   else
@@ -2866,12 +2985,12 @@ schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
                   _("Creating connection, outstanding_connections is %d\n"),
                   outstanding_connects);
 #endif
-      outstanding_connects++;
-      outstanding_scheduled_connects--;
+      pg->outstanding_connects++;
+      pg->total_connects_scheduled++;
       GNUNET_TESTING_daemons_connect (connect_context->first,
                                       connect_context->second,
-                                      CONNECT_TIMEOUT,
-                                      CONNECT_ATTEMPTS,
+                                      connect_context->ct_ctx->connect_timeout,
+                                      connect_context->ct_ctx->connect_attempts,
                                       &internal_connect_notify,
                                       connect_context->ct_ctx);
       GNUNET_free (connect_context);
@@ -2982,6 +3101,8 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
  * of each peer in the peer group
  *
  * @param pg the peer group we are dealing with
+ * @param connect_timeout how long try connecting two peers
+ * @param connect_attempts how many times (max) to attempt
  * @param notify_callback callback to notify when finished
  * @param notify_cls closure for notify callback
  *
@@ -2989,6 +3110,8 @@ copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
  */
 static int
 connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
+                  struct GNUNET_TIME_Relative connect_timeout,
+                  unsigned int connect_attempts,
                   GNUNET_TESTING_NotifyCompletion notify_callback,
                   void *notify_cls)
 {
@@ -2997,9 +3120,6 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
   struct ConnectTopologyContext *ct_ctx;
 #if OLD
   struct PeerConnection *connection_iter;
-  struct ConnectContext *connect_context;
-#else
-  int ret;
 #endif
 
   total = 0;
@@ -3028,41 +3148,17 @@ connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_free (ct_ctx);
       return total;
     }
+  ct_ctx->connect_timeout = connect_timeout;
+  ct_ctx->connect_attempts = connect_attempts;
   ct_ctx->remaining_connections = total;
-  total = 0;
+  ct_ctx->remaining_connects_to_schedule = total;
 
-  for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
-    {
-#if OLD
-      connection_iter = pg->peers[pg_iter].connect_peers_head;
-      while (connection_iter != NULL)
-        {
-          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Scheduling connect of peer %d to peer %d\n", pg_iter, connection_iter->index);
-          connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
-          connect_context->first = pg->peers[pg_iter].daemon;
-          connect_context->second = pg->peers[connection_iter->index].daemon;
-          connect_context->ct_ctx = ct_ctx;
-          if (total < MAX_OUTSTANDING_CONNECTIONS)
-            {
-              GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
-            }
-          else
-            {
-              GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 1000 * (total / MAX_OUTSTANDING_CONNECTIONS)), &schedule_connect, connect_context);
-            }
-          connection_iter = connection_iter->next;
-          total++;
-        }
-#else
-      ret =
-        GNUNET_CONTAINER_multihashmap_iterate (pg->
-                                               peers[pg_iter].connect_peers,
-                                               &connect_iterator, ct_ctx);
-      GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
-      total = total + ret;
-#endif
-    }
+  for (pg_iter = 0; pg_iter < pg->max_outstanding_connections; pg_iter++)
+  {
+    preschedule_connect(ct_ctx);
+  }
   return total;
+
 }
 
 
@@ -4029,7 +4125,7 @@ schedule_get_topology (void *cls,
   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
     return;
 
-  if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+  if (topology_context->connected > topology_context->pg->max_outstanding_connections)
     {
 #if VERBOSE_TESTING > 2
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -4077,6 +4173,7 @@ GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
   topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
   topology_context->topology_cb = cb;
   topology_context->cls = cls;
+  topology_context->pg = pg;
   total_count = 0;
   for (i = 0; i < pg->total; i++)
     {
@@ -4171,7 +4268,7 @@ schedule_get_statistics (void *cls,
   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
     return;
 
-  if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
+  if (stats_context->connected > stats_context->pg->max_outstanding_connections)
     {
 #if VERBOSE_TESTING > 2
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
@@ -4309,6 +4406,7 @@ GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
   stats_context->cont = cont;
   stats_context->proc = proc;
   stats_context->cls = cls;
+  stats_context->pg = pg;
   total_count = 0;
 
   for (i = 0; i < pg->total; i++)
@@ -4357,6 +4455,10 @@ GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
  * @param topology which topology to connect the peers in
  * @param options options for connecting the topology
  * @param option_modifier modifier for options that take a parameter
+ * @param connect_timeout how long to wait before giving up on connecting
+ *                        two peers
+ * @param connect_attempts how many times to attempt to connect two peers
+ *                         over the connect_timeout duration
  * @param notify_callback notification to be called once all connections completed
  * @param notify_cls closure for notification callback
  *
@@ -4367,6 +4469,8 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
                                  enum GNUNET_TESTING_Topology topology,
                                  enum GNUNET_TESTING_TopologyOption options,
                                  double option_modifier,
+                                 struct GNUNET_TIME_Relative connect_timeout,
+                                 unsigned int connect_attempts,
                                  GNUNET_TESTING_NotifyCompletion
                                  notify_callback, void *notify_cls)
 {
@@ -4499,9 +4603,66 @@ GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
       break;
     }
 
-  return connect_topology (pg, notify_callback, notify_cls);
+  return connect_topology (pg, connect_timeout, connect_attempts, notify_callback, notify_cls);
+}
+
+/**
+ * Lookup and return the number of SSH connections to a host.
+ *
+ * @param hostname the hostname to lookup in the list
+ * @param pg the peergroup that the host belongs to
+ *
+ * @return the number of current ssh connections to the host
+ */
+static unsigned int
+count_outstanding_at_host(const char *hostname, struct GNUNET_TESTING_PeerGroup *pg)
+{
+  struct OutstandingSSH *pos;
+  pos = pg->ssh_head;
+  while ((pos != NULL) && (strcmp(pos->hostname, hostname) != 0))
+    pos = pos->next;
+  GNUNET_assert(pos != NULL);
+  return pos->outstanding;
+}
+
+
+/**
+ * Increment the number of SSH connections to a host by one.
+ *
+ * @param hostname the hostname to lookup in the list
+ * @param pg the peergroup that the host belongs to
+ *
+ */
+static void
+increment_outstanding_at_host(const char *hostname, struct GNUNET_TESTING_PeerGroup *pg)
+{
+  struct OutstandingSSH *pos;
+  pos = pg->ssh_head;
+  while ((pos != NULL) && (strcmp(pos->hostname, hostname) != 0))
+    pos = pos->next;
+  GNUNET_assert(pos != NULL);
+  pos->outstanding++;
+}
+
+/**
+ * Decrement the number of SSH connections to a host by one.
+ *
+ * @param hostname the hostname to lookup in the list
+ * @param pg the peergroup that the host belongs to
+ *
+ */
+static void
+decrement_outstanding_at_host(const char *hostname, struct GNUNET_TESTING_PeerGroup *pg)
+{
+  struct OutstandingSSH *pos;
+  pos = pg->ssh_head;
+  while ((pos != NULL) && (strcmp(pos->hostname, hostname) != 0))
+    pos = pos->next;
+  GNUNET_assert(pos != NULL);
+  pos->outstanding--;
 }
 
+
 /**
  * Callback that is called whenever a hostkey is generated
  * for a peer.  Call the real callback and decrement the
@@ -4520,6 +4681,8 @@ internal_hostkey_callback (void *cls,
   struct InternalStartContext *internal_context = cls;
   internal_context->peer->pg->starting--;
   internal_context->peer->pg->started++;
+  if (internal_context->hostname != NULL)
+    decrement_outstanding_at_host(internal_context->hostname, internal_context->peer->pg);
   if (internal_context->hostkey_callback != NULL)
     internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
                                         emsg);
@@ -4550,6 +4713,8 @@ internal_startup_callback (void *cls,
 {
   struct InternalStartContext *internal_context = cls;
   internal_context->peer->pg->starting--;
+  if (internal_context->hostname != NULL)
+    decrement_outstanding_at_host(internal_context->hostname, internal_context->peer->pg);
   if (internal_context->start_cb != NULL)
     internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
                                 emsg);
@@ -4566,8 +4731,12 @@ internal_continue_startup (void *cls,
       return;
     }
 
-  if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
+  if ((internal_context->peer->pg->starting < internal_context->peer->pg->max_concurrent_ssh) ||
+      ((internal_context->hostname != NULL) &&
+       (count_outstanding_at_host(internal_context->hostname, internal_context->peer->pg) < internal_context->peer->pg->max_concurrent_ssh)))
     {
+      if (internal_context->hostname != NULL)
+        increment_outstanding_at_host(internal_context->hostname, internal_context->peer->pg);
       internal_context->peer->pg->starting++;
       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
     }
@@ -4649,7 +4818,7 @@ schedule_churn_restart (void *cls,
   struct ChurnRestartContext *startup_ctx =
     peer_restart_ctx->churn_restart_ctx;
 
-  if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
+  if (startup_ctx->outstanding > startup_ctx->pg->max_concurrent_ssh)
     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
                                   &schedule_churn_restart, peer_restart_ctx);
@@ -4663,6 +4832,8 @@ schedule_churn_restart (void *cls,
     }
 }
 
+
+
 static void
 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
@@ -4673,8 +4844,12 @@ internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       return;
     }
 
-  if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
+  if ((internal_context->peer->pg->starting < internal_context->peer->pg->max_concurrent_ssh) ||
+      ((internal_context->hostname != NULL) &&
+       (count_outstanding_at_host(internal_context->hostname, internal_context->peer->pg) < internal_context->peer->pg->max_concurrent_ssh)))
     {
+      if (internal_context->hostname != NULL)
+        increment_outstanding_at_host(internal_context->hostname, internal_context->peer->pg);
       internal_context->peer->pg->starting++;
       internal_context->peer->daemon =
         GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
@@ -4725,6 +4900,10 @@ GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
  *
  * @param cfg configuration template to use
  * @param total number of daemons to start
+ * @param max_concurrent_connections for testing, how many peers can
+*                                   we connect to simultaneously
+ * @param max_concurrent_ssh when starting with ssh, how many ssh
+ *        connections will we allow at once (based on remote hosts allowed!)
  * @param timeout total time allowed for peers to start
  * @param hostkey_callback function to call on each peers hostkey generation
  *        if NULL, peers will be started by this call, if non-null,
@@ -4743,6 +4922,8 @@ GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
 struct GNUNET_TESTING_PeerGroup *
 GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                               unsigned int total,
+                              unsigned int max_concurrent_connections,
+                              unsigned int max_concurrent_ssh,
                               struct GNUNET_TIME_Relative timeout,
                               GNUNET_TESTING_NotifyHostkeyCreated
                               hostkey_callback, void *hostkey_cls,
@@ -4795,6 +4976,8 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
   pg->total = total;
   pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
+  pg->max_outstanding_connections = max_concurrent_connections;
+  pg->max_concurrent_ssh = max_concurrent_ssh;
   if (NULL != hostnames)
     {
       off = 0;
@@ -4883,7 +5066,10 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
                                                                     &baseservicehome));
   for (i = 0; i < pg->num_hosts; i++)
     {
-
+      struct OutstandingSSH *ssh_entry;
+      ssh_entry = GNUNET_malloc(sizeof(struct OutstandingSSH));
+      ssh_entry->hostname = pg->hosts[i].hostname; /* Don't free! */
+      GNUNET_CONTAINER_DLL_insert(pg->ssh_head, pg->ssh_tail, ssh_entry);
       if (NULL != pg->hosts[i].username)
         GNUNET_asprintf (&arg, "%s@%s", pg->hosts[i].username, pg->hosts[i].hostname);
       else
@@ -4925,7 +5111,7 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
           if (GNUNET_YES != GNUNET_DISK_file_size (hostkeys_file, &fs, GNUNET_YES))
             fs = 0;
 
-          GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found file size %llu for hostkeys, expect hostkeys to be size %d\n", fs, HOSTKEYFILESIZE);
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Found file size %llu for hostkeys, expect hostkeys to be size %d\n", fs, HOSTKEYFILESIZE);
 
           if (fs % HOSTKEYFILESIZE != 0)
             {
@@ -4934,7 +5120,7 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
           else
             {
               total_hostkeys = fs / HOSTKEYFILESIZE;
-              GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Will read %llu hostkeys from file\n", total_hostkeys);
+              GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Will read %llu hostkeys from file\n", total_hostkeys);
               pg->hostkey_data = GNUNET_malloc_large (fs);
               GNUNET_assert (fs == GNUNET_DISK_file_read (fd, pg->hostkey_data, fs));
               GNUNET_assert(GNUNET_OK == GNUNET_DISK_file_close(fd));
@@ -4951,6 +5137,7 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
           username = pg->hosts[off % hostcnt].username;
           sshport = pg->hosts[off % hostcnt].sshport;
           pcfg = make_config (cfg,
+                              off,
                               &pg->hosts[off % hostcnt].minport,
                               &upnum, hostname, &fdnum);
         }
@@ -4959,7 +5146,7 @@ GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
           hostname = NULL;
           username = NULL;
           sshport = 0;
-          pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
+          pcfg = make_config (cfg, off, &minport, &upnum, hostname, &fdnum);
         }
 
       if (NULL == pcfg)
@@ -5195,12 +5382,12 @@ schedule_churn_shutdown_task (void *cls,
 {
   struct PeerShutdownContext *peer_shutdown_ctx = cls;
   struct ShutdownContext *shutdown_ctx;
-
+  struct ChurnContext *churn_ctx;
   GNUNET_assert (peer_shutdown_ctx != NULL);
   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
   GNUNET_assert (shutdown_ctx != NULL);
-
-  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+  churn_ctx = (struct ChurnContext *)shutdown_ctx->cb_cls;
+  if (shutdown_ctx->outstanding > churn_ctx->pg->max_concurrent_ssh)
     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
                                   &schedule_churn_shutdown_task,
@@ -5327,6 +5514,7 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
   churn_ctx->num_to_stop = voff;
   churn_ctx->cb = cb;
   churn_ctx->cb_cls = cb_cls;
+  churn_ctx->pg = pg;
 
   for (i = 0; i < pg->total; i++)
     {
@@ -5381,6 +5569,7 @@ GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
       churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
       churn_startup_ctx->churn_ctx = churn_ctx;
       churn_startup_ctx->timeout = timeout;
+      churn_startup_ctx->pg = pg;
     }
   for (i = 0; i < von; i++)
     {
@@ -5505,6 +5694,7 @@ void
 internal_shutdown_callback (void *cls, const char *emsg)
 {
   struct ShutdownContext *shutdown_ctx = cls;
+  unsigned int off;
 
   shutdown_ctx->outstanding--;
   if (emsg == NULL)
@@ -5525,6 +5715,17 @@ internal_shutdown_callback (void *cls, const char *emsg)
                           "Not all peers successfully shut down!");
       else
         shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
+
+      GNUNET_free (shutdown_ctx->pg->peers);
+      GNUNET_free_non_null(shutdown_ctx->pg->hostkey_data);
+      for (off = 0; off < shutdown_ctx->pg->num_hosts; off++)
+        {
+          GNUNET_free (shutdown_ctx->pg->hosts[off].hostname);
+          GNUNET_free_non_null (shutdown_ctx->pg->hosts[off].username);
+        }
+      GNUNET_free_non_null (shutdown_ctx->pg->hosts);
+      GNUNET_free (shutdown_ctx->pg);
+
       GNUNET_free (shutdown_ctx);
     }
 }
@@ -5548,7 +5749,7 @@ schedule_shutdown_task (void *cls,
   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
   GNUNET_assert (shutdown_ctx != NULL);
 
-  if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
+  if (shutdown_ctx->outstanding > shutdown_ctx->pg->max_concurrent_ssh)
     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
                                   &schedule_shutdown_task, peer_shutdown_ctx);
@@ -5591,6 +5792,7 @@ GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
   shutdown_ctx->cb_cls = cb_cls;
   shutdown_ctx->total_peers = pg->total;
   shutdown_ctx->timeout = timeout;
+  shutdown_ctx->pg = pg;
   /* shtudown_ctx->outstanding = 0; */
 
   for (off = 0; off < pg->total; off++)
@@ -5645,15 +5847,7 @@ GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
                                                peers[off].blacklisted_peers);
 #endif
     }
-  GNUNET_free (pg->peers);
-  GNUNET_free_non_null(pg->hostkey_data);
-  for (off = 0; off < pg->num_hosts; off++)
-    {
-      GNUNET_free (pg->hosts[off].hostname);
-      GNUNET_free_non_null (pg->hosts[off].username);
-    }
-  GNUNET_free_non_null (pg->hosts);
-  GNUNET_free (pg);
+
 }