leak
[oweals/gnunet.git] / src / testing / testing_group.c
index 14bf31bbc88b47dbd14d8658ef4c43c1bae90f6a..033ab7d5edc00315a63d50a43ba8eaeddc1d032c 100644 (file)
@@ -26,6 +26,7 @@
  *
  */
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
 #include "gnunet_core_service.h"
 
 #define DEBUG_CHURN GNUNET_NO
 
+#define USE_START_HELPER GNUNET_YES
+
 #define OLD 1
 
+/* Before connecting peers, send all of the HELLOs */
 #define USE_SEND_HELLOS GNUNET_NO
 
 #define TOPOLOGY_HACK GNUNET_YES
@@ -1406,6 +1410,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics", "UNIXPATH",
                                              "");
 
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
       GNUNET_free_non_null (control_host);
       GNUNET_free (allowed_hosts);
     }
@@ -1419,6 +1427,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
                                              hostname);
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "BINDTO",
                                              hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
                                              allowed_hosts);
       GNUNET_free (allowed_hosts);
@@ -1429,6 +1441,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
                                              "127.0.0.1");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "BINDTO",
                                              "127.0.0.1");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
     }
 
   *port = (uint16_t) uc.nport;
@@ -2357,7 +2373,7 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
   connect_attempts = 0;
 
   conn_meter = create_meter ((((pg->total * pg->total) + pg->total) / 2)
-      - pg->total, "Create Clique ", GNUNET_YES);
+      - pg->total, "Create Clique ", GNUNET_NO);
   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
     {
       for (inner_count = outer_count + 1; inner_count < pg->total; inner_count++)
@@ -2371,8 +2387,6 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
           update_meter (conn_meter);
         }
     }
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Meter has %d left\n",
-              conn_meter->total - conn_meter->completed);
   reset_meter (conn_meter);
   free_meter (conn_meter);
   return connect_attempts;
@@ -2422,16 +2436,17 @@ static unsigned int
 copy_allowed(struct GNUNET_TESTING_PeerGroup *pg,
              GNUNET_TESTING_ConnectionProcessor proc)
 {
-  struct UnblacklistContext un_ctx;
   unsigned int count;
   unsigned int total;
   struct PeerConnection *iter;
+#if !OLD
+  struct UnblacklistContext un_ctx;
 
   un_ctx.pg = pg;
+#endif
   total = 0;
   for (count = 0; count < pg->total - 1; count++)
     {
-      un_ctx.first_uid = count;
 #if OLD
       iter = pg->peers[count].allowed_peers_head;
       while (iter != NULL)
@@ -2441,10 +2456,15 @@ copy_allowed(struct GNUNET_TESTING_PeerGroup *pg,
           iter = iter->next;
         }
 #else
-      total += GNUNET_CONTAINER_multihashmap_iterate(pg->peers[count].allowed_peers, &unblacklist_iterator, &un_ctx);
+      un_ctx.first_uid = count;
+      total += GNUNET_CONTAINER_multihashmap_iterate(pg->peers[count].allowed_peers, 
+                                                    &unblacklist_iterator,
+                                                    &un_ctx);
 #endif
     }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Unblacklisted %u peers\n", total);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 
+             "Unblacklisted %u peers\n", 
+             total);
   return total;
 }
 
@@ -2465,16 +2485,16 @@ create_line(struct GNUNET_TESTING_PeerGroup *pg,
             GNUNET_TESTING_ConnectionProcessor proc, enum PeerLists list)
 {
   unsigned int count;
-  int connect_attempts;
+  unsigned int connect_attempts;
 
   connect_attempts = 0;
-
   /* Connect each peer to the next highest numbered peer */
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-          "Connecting peer %d to peer %d\n", count, count + 1);
+                 "Connecting peer %d to peer %d\n", 
+                 count, count + 1);
 #endif
       connect_attempts += proc (pg, count, count + 1, list, GNUNET_YES);
     }
@@ -2502,15 +2522,14 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
   int connect_attempts;
   unsigned int first_peer_index;
   unsigned int second_peer_index;
-  connect_attempts = 0;
   struct stat frstat;
   int count;
   char *data;
-  char *buf;
+  const char *buf;
   unsigned int total_peers;
-
   enum States curr_state;
 
+  connect_attempts = 0;
   if (GNUNET_OK != GNUNET_DISK_file_test (filename))
     GNUNET_DISK_fn_write (filename, NULL, 0, GNUNET_DISK_PERM_USER_READ);
 
@@ -2549,79 +2568,80 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
 
       switch (curr_state)
         {
-      case NUM_PEERS:
-        errno = 0;
-        total_peers = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to read number of peers from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Read %u total peers in topology\n", total_peers);
-        GNUNET_assert(total_peers == pg->total);
-        curr_state = PEER_INDEX;
-        while ((buf[count] != '\n') && (count < frstat.st_size - 1))
-          count++;
-        count++;
-        break;
-      case PEER_INDEX:
-        errno = 0;
-        first_peer_index = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to read peer index from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        while ((buf[count] != ':') && (count < frstat.st_size - 1))
-          count++;
-        count++;
-        curr_state = OTHER_PEER_INDEX;
-        break;
-      case COLON:
-        if (1 == sscanf (&buf[count], ":"))
-          curr_state = OTHER_PEER_INDEX;
-        count++;
-        break;
-      case OTHER_PEER_INDEX:
-        errno = 0;
-        second_peer_index = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to peer index from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        /* Assume file is written with first peer 1, but array index is 0 */
-        connect_attempts += proc (pg, first_peer_index - 1, second_peer_index
-                                  - 1, list, GNUNET_YES);
-        while ((buf[count] != '\n') && (buf[count] != ',') && (count
-            < frstat.st_size - 1))
-          count++;
-        if (buf[count] == '\n')
-          {
-            curr_state = PEER_INDEX;
-          }
-        else if (buf[count] != ',')
-          {
-            curr_state = OTHER_PEER_INDEX;
-          }
-        count++;
-        break;
-      default:
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Found bad data in topology file while in state %d!\n",
-                    curr_state);
-        GNUNET_break(0);
-        return connect_attempts;
+       case NUM_PEERS:
+         errno = 0;
+         total_peers = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to read number of peers from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                     "Read %u total peers in topology\n", total_peers);
+         GNUNET_assert(total_peers == pg->total);
+         curr_state = PEER_INDEX;
+         while ((buf[count] != '\n') && (count < frstat.st_size - 1))
+           count++;
+         count++;
+         break;
+       case PEER_INDEX:
+         errno = 0;
+         first_peer_index = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to read peer index from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         while ((buf[count] != ':') && (count < frstat.st_size - 1))
+           count++;
+         count++;
+         curr_state = OTHER_PEER_INDEX;
+         break;
+       case COLON:
+         if (1 == sscanf (&buf[count], ":"))
+           curr_state = OTHER_PEER_INDEX;
+         count++;
+         break;
+       case OTHER_PEER_INDEX:
+         errno = 0;
+         second_peer_index = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to peer index from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         /* Assume file is written with first peer 1, but array index is 0 */
+         connect_attempts += proc (pg, first_peer_index - 1, second_peer_index
+                                   - 1, list, GNUNET_YES);
+         while ((buf[count] != '\n') && (buf[count] != ',') && (count
+                                                                < frstat.st_size - 1))
+           count++;
+         if (buf[count] == '\n')
+           {
+             curr_state = PEER_INDEX;
+           }
+         else if (buf[count] != ',')
+           {
+             curr_state = OTHER_PEER_INDEX;
+           }
+         count++;
+         break;
+       default:
+         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                     "Found bad data in topology file while in state %d!\n",
+                     curr_state);
+         GNUNET_break(0);
+         GNUNET_free (data);
+         return connect_attempts;
         }
-
     }
+  GNUNET_free (data);
   return connect_attempts;
 }
 
@@ -2811,6 +2831,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv", "mv",
                                                       mytemp, arg, NULL);
+          GNUNET_assert(procarr[pg_iter] != NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("Copying file with command cp %s %s\n"), mytemp, arg);
@@ -2832,7 +2853,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           ret = GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: schedule this, throttle! */
           GNUNET_OS_process_close (procarr[pg_iter]);
           if (ret != GNUNET_OK)
@@ -3030,7 +3051,7 @@ create_and_copy_blacklist_files(struct GNUNET_TESTING_PeerGroup *pg,
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: add scheduled blacklist file copy that parallelizes file copying! */
 
 #if VERBOSE_TESTING
@@ -3229,7 +3250,6 @@ send_core_connect_requests (void *cls, const struct GNUNET_SCHEDULER_TaskContext
         while (conn != NULL)
           {
             GNUNET_CORE_peer_request_connect(send_hello_context->peer->daemon->server,
-                GNUNET_TIME_relative_get_forever(),
                 &send_hello_context->pg->peers[conn->index].daemon->id,
                 NULL,
                 NULL);
@@ -3385,7 +3405,7 @@ hello_sent_callback (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   {
     struct SendHelloContext *send_hello_context = cls;
     //unsigned int pg_iter;
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3443,7 +3463,7 @@ static void schedule_send_hellos (void *cls, const struct GNUNET_SCHEDULER_TaskC
     struct SendHelloContext *send_hello_context = cls;
     struct GNUNET_TESTING_PeerGroup *pg = send_hello_context->pg;
 
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3474,12 +3494,9 @@ static void schedule_send_hellos (void *cls, const struct GNUNET_SCHEDULER_TaskC
         if (send_hello_context->peer->daemon->th == NULL)
           {
             pg->outstanding_connects++; /* Actual TRANSPORT, CORE connections! */
-            send_hello_context->peer->daemon->th = GNUNET_TRANSPORT_connect(send_hello_context->peer->cfg,
-                NULL,
-                send_hello_context,
-                NULL,
-                NULL,
-                NULL);
+            send_hello_context->peer->daemon->th
+                = GNUNET_TRANSPORT_connect (send_hello_context->peer->cfg, NULL,
+                                            send_hello_context, NULL, NULL, NULL);
           }
 #if DEBUG_TESTING
         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -3514,6 +3531,8 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
   struct ConnectTopologyContext *ct_ctx = connect_ctx->ct_ctx;
   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
   struct PeerConnection *connection;
+
+  GNUNET_assert (0 < pg->outstanding_connects);
   pg->outstanding_connects--;
 
   /*
@@ -3535,6 +3554,7 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
 
   if (connection != NULL) /* Can safely remove! */
     {
+      GNUNET_assert (0 < ct_ctx->remaining_connections);
       ct_ctx->remaining_connections--;
       if (pg->notify_connection != NULL) /* Notify of reverse connection */
         pg->notify_connection (pg->notify_connection_cls, second, first,
@@ -3548,7 +3568,10 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
   if (ct_ctx->remaining_connections == 0)
     {
       if (ct_ctx->notify_connections_done != NULL)
-        ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+       {
+         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+         ct_ctx->notify_connections_done = NULL;
+       }
     }
   else
     preschedule_connect (pg);
@@ -3574,7 +3597,7 @@ schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct ConnectContext *connect_context = cls;
   struct GNUNET_TESTING_PeerGroup *pg = connect_context->ct_ctx->pg;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if ((pg->outstanding_connects > pg->max_outstanding_connections)
@@ -3606,7 +3629,7 @@ schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
                                       connect_context->ct_ctx->connect_timeout,
                                       connect_context->ct_ctx->connect_attempts,
 #if USE_SEND_HELLOS
-                                       GNUNET_NO,
+                                      GNUNET_NO,
 #else
                                       GNUNET_YES,
 #endif
@@ -3695,7 +3718,7 @@ copy_allowed_topology(struct GNUNET_TESTING_PeerGroup *pg)
                       "Creating connection between %d and %d\n", pg_iter,
                       iter->index);
           total += add_connections (pg, pg_iter, iter->index, CONNECT,
-                                    GNUNET_NO);
+                                    GNUNET_YES);
           //total += add_actual_connections(pg, pg_iter, iter->index);
           iter = iter->next;
         }
@@ -3741,7 +3764,7 @@ connect_topology(struct GNUNET_TESTING_PeerGroup *pg,
   struct PeerConnection *connection_iter;
 #endif
 #if USE_SEND_HELLOS
-  struct SendHelloContext *send_hello_context
+  struct SendHelloContext *send_hello_context;
 #endif
 
   total = 0;
@@ -3959,6 +3982,8 @@ GNUNET_TESTING_create_topology(struct GNUNET_TESTING_PeerGroup *pg,
   if ((restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE) && (restrict_topology
       != GNUNET_TESTING_TOPOLOGY_FROM_FILE))
     create_clique (pg, &add_connections, BLACKLIST, GNUNET_NO);
+  else
+    return num_connections;
 
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
@@ -4264,22 +4289,17 @@ void
 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg,
                           double percentage)
 {
-  struct RandomContext random_ctx;
   uint32_t pg_iter;
 #if OLD
-  struct PeerConnection *temp_peers;
   struct PeerConnection *conn_iter;
   double random_number;
+#else
+  struct RandomContext random_ctx;
 #endif
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      random_ctx.first_uid = pg_iter;
-      random_ctx.first = &pg->peers[pg_iter];
-      random_ctx.percentage = percentage;
-      random_ctx.pg = pg;
 #if OLD
-      temp_peers = NULL;
       conn_iter = pg->peers[pg_iter].connect_peers_head;
       while (conn_iter != NULL)
         {
@@ -4295,17 +4315,21 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg,
           conn_iter = conn_iter->next;
         }
 #else
-      pg->peers[pg_iter].connect_peers_working_set =
-      GNUNET_CONTAINER_multihashmap_create (pg->total);
+      random_ctx.first_uid = pg_iter;
+      random_ctx.first = &pg->peers[pg_iter];
+      random_ctx.percentage = percentage;
+      random_ctx.pg = pg;
+      pg->peers[pg_iter].connect_peers_working_set 
+       = GNUNET_CONTAINER_multihashmap_create (pg->total);
       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
-          &random_connect_iterator,
-          &random_ctx);
+                                            &random_connect_iterator,
+                                            &random_ctx);
       /* Now remove the old connections */
       GNUNET_CONTAINER_multihashmap_destroy (pg->
-          peers[pg_iter].connect_peers);
+                                            peers[pg_iter].connect_peers);
       /* And replace with the random set */
-      pg->peers[pg_iter].connect_peers =
-      pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers 
+       = pg->peers[pg_iter].connect_peers_working_set;
 #endif
     }
 
@@ -4799,7 +4823,7 @@ schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct CoreContext *core_context = cls;
   struct TopologyIterateContext *topology_context =
       (struct TopologyIterateContext *) core_context->iter_context;
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (topology_context->connected
@@ -4943,7 +4967,7 @@ schedule_get_statistics(void *cls,
   struct StatsIterateContext *stats_context =
       (struct StatsIterateContext *) core_context->iter_context;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (stats_context->connected > stats_context->pg->max_outstanding_connections)
@@ -5425,7 +5449,7 @@ internal_continue_startup(void *cls,
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5532,12 +5556,13 @@ schedule_churn_restart(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     }
 }
 
+
 static void
 internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5556,6 +5581,7 @@ internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       internal_context->peer->daemon
           = GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
                                          internal_context->timeout,
+                                         GNUNET_NO,
                                          internal_context->hostname,
                                          internal_context->username,
                                          internal_context->sshport,
@@ -5574,6 +5600,89 @@ internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
                                     &internal_start, internal_context);
     }
 }
+#if USE_START_HELPER
+
+struct PeerStartHelperContext
+{
+  struct GNUNET_TESTING_PeerGroup *pg;
+
+  struct HostData *host;
+
+  struct GNUNET_OS_Process *proc;
+};
+
+static void
+check_peers_started (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerStartHelperContext *helper = cls;
+  enum GNUNET_OS_ProcessStatusType type;
+  unsigned long code;
+  unsigned int i;
+  GNUNET_TESTING_NotifyDaemonRunning cb;
+
+  if (GNUNET_NO == GNUNET_OS_process_status (helper->proc, &type, &code)) /* Still running, wait some more! */
+  {
+    GNUNET_SCHEDULER_add_delayed(GNUNET_CONSTANTS_EXEC_WAIT, &check_peers_started, helper);
+    return;
+  }
+
+  helper->pg->starting--;
+  if (helper->pg->starting == 0) /* All peers have finished starting! */
+    {
+      /* Call the peer started callback for each peer, set proper FSM state (?) */
+      for (i = 0; i < helper->pg->total; i++)
+        {
+          cb = helper->pg->peers[i].daemon->cb;
+          helper->pg->peers[i].daemon->cb = NULL;
+          helper->pg->peers[i].daemon->running = GNUNET_YES;
+          helper->pg->peers[i].daemon->phase = SP_START_DONE;
+          if (NULL != cb)
+          {
+            if ((type != GNUNET_OS_PROCESS_EXITED) || (code != 0))
+              cb (helper->pg->peers[i].daemon->cb_cls,
+                  &helper->pg->peers[i].daemon->id,
+                  helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
+                  "Failed to execute peerStartHelper.pl, or return code bad!");
+            else
+              cb (helper->pg->peers[i].daemon->cb_cls,
+                  &helper->pg->peers[i].daemon->id,
+                  helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
+                  NULL);
+
+          }
+
+        }
+    }
+  GNUNET_OS_process_close(helper->proc);
+}
+
+static void
+start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerStartHelperContext *helper = cls;
+  char *baseservicehome;
+  char *tempdir;
+  char *arg;
+  /* ssh user@host peerStartHelper /path/to/basedirectory */
+  GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (helper->pg->cfg, "PATHS", "SERVICEHOME",
+                                                                    &baseservicehome));
+  GNUNET_asprintf(&tempdir, "%s/%s/", baseservicehome, helper->host->hostname);
+  if (NULL != helper->host->username)
+    GNUNET_asprintf (&arg, "%s@%s", helper->host->username, helper->host->hostname);
+  else
+    GNUNET_asprintf (&arg, "%s", helper->host->hostname);
+
+  /* FIXME: Doesn't support ssh_port option! */
+  helper->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
+                                  "peerStartHelper.pl", tempdir,  NULL);
+  GNUNET_assert(helper->proc != NULL);
+  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "starting peers with cmd ssh %s %s %s\n", arg, "peerStartHelper.pl", tempdir);
+  GNUNET_SCHEDULER_add_now (&check_peers_started, helper);
+  GNUNET_free (tempdir);
+  GNUNET_free (baseservicehome);
+  GNUNET_free (arg);
+}
+#endif
 
 /**
  * Function which continues a peer group starting up
@@ -5587,15 +5696,58 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int i;
 
+#if USE_START_HELPER
+  if ((pg->num_hosts > 0) && (pg->hostkey_data != NULL))
+    {
+      struct PeerStartHelperContext *helper;
+      pg->starting = pg->num_hosts;
+      for (i = 0; i < pg->num_hosts; i++)
+        {
+          helper = GNUNET_malloc(sizeof(struct PeerStartHelperContext));
+          helper->pg = pg;
+          helper->host = &pg->hosts[i];
+          GNUNET_SCHEDULER_add_now(&start_peer_helper, helper);
+        }
+    }
+  else
+    {
+      pg->starting = 0;
+      for (i = 0; i < pg->total; i++)
+        {
+          GNUNET_SCHEDULER_add_now (&internal_continue_startup,
+                                    &pg->peers[i].internal_context);
+        }
+    }
+#else
   pg->starting = 0;
   for (i = 0; i < pg->total; i++)
     {
       GNUNET_SCHEDULER_add_now (&internal_continue_startup,
                                 &pg->peers[i].internal_context);
-      //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
     }
+#endif
 }
 
+#if USE_START_HELPER
+static void
+call_hostkey_callbacks (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_TESTING_PeerGroup *pg = cls;
+  unsigned int i;
+  for (i = 0; i < pg->total; i++)
+    {
+      if (pg->peers[i].internal_context.hostkey_callback != NULL)
+        pg->peers[i].internal_context.hostkey_callback (pg->peers[i].internal_context.hostkey_cls,
+                                                           &pg->peers[i].daemon->id,
+                                                           pg->peers[i].daemon,
+                                                           NULL);
+    }
+
+  if (pg->peers[0].internal_context.hostkey_callback == NULL)
+    GNUNET_TESTING_daemons_continue_startup (pg);
+}
+#endif
+
 /**
  * Start count gnunet instances with the same set of transports and
  * applications.  The port numbers (any option called "PORT") will be
@@ -5624,8 +5776,7 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
  * @return NULL on error, otherwise handle to control peer group
  */
 struct GNUNET_TESTING_PeerGroup *
-GNUNET_TESTING_daemons_start(
-                             const struct GNUNET_CONFIGURATION_Handle *cfg,
+GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
                              unsigned int total,
                              unsigned int max_concurrent_connections,
                              unsigned int max_concurrent_ssh,
@@ -5640,11 +5791,6 @@ GNUNET_TESTING_daemons_start(
 {
   struct GNUNET_TESTING_PeerGroup *pg;
   const struct GNUNET_TESTING_Host *hostpos;
-#if 0
-  char *pos;
-  const char *rpos;
-  char *start;
-#endif
   const char *hostname;
   const char *username;
   char *baseservicehome;
@@ -5667,6 +5813,7 @@ GNUNET_TESTING_daemons_start(
   uint64_t total_hostkeys;
   struct GNUNET_OS_Process *proc;
 
+  username = NULL;
   if (0 == total)
     {
       GNUNET_break (0);
@@ -5715,51 +5862,6 @@ GNUNET_TESTING_daemons_start(
       hostcnt = off;
       minport = 0;
       pg->num_hosts = off;
-
-#if NO_LL
-      off = 2;
-      /* skip leading spaces */
-      while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
-      hostnames++;
-      rpos = hostnames;
-      while ('\0' != *rpos)
-        {
-          if (isspace ((unsigned char) *rpos))
-          off++;
-          rpos++;
-        }
-      pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
-      off = 0;
-      start = GNUNET_strdup (hostnames);
-      pos = start;
-      while ('\0' != *pos)
-        {
-          if (isspace ((unsigned char) *pos))
-            {
-              *pos = '\0';
-              if (strlen (start) > 0)
-                {
-                  pg->hosts[off].minport = LOW_PORT;
-                  pg->hosts[off++].hostname = start;
-                }
-              start = pos + 1;
-            }
-          pos++;
-        }
-      if (strlen (start) > 0)
-        {
-          pg->hosts[off].minport = LOW_PORT;
-          pg->hosts[off++].hostname = start;
-        }
-      if (off == 0)
-        {
-          GNUNET_free (start);
-          GNUNET_free (pg->hosts);
-          pg->hosts = NULL;
-        }
-      hostcnt = off;
-      minport = 0; /* make gcc happy */
-#endif
     }
   else
     {
@@ -5795,6 +5897,7 @@ GNUNET_TESTING_daemons_start(
       else
         proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
                                         "mkdir -p", tmpdir, NULL);
+      GNUNET_assert(proc != NULL);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Creating remote dir with command ssh %s %s %s\n", arg,
                   " mkdir -p ", tmpdir);
@@ -5810,15 +5913,28 @@ GNUNET_TESTING_daemons_start(
                                                            &hostkeys_file))
     {
       if (GNUNET_YES != GNUNET_DISK_file_test (hostkeys_file))
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Couldn't read hostkeys file!\n");
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 
+                   _("Could not read hostkeys file!\n"));
       else
         {
           /* Check hostkey file size, read entire thing into memory */
-          fd = GNUNET_DISK_file_open (hostkeys_file, GNUNET_DISK_OPEN_READ,
+          fd = GNUNET_DISK_file_open (hostkeys_file,
+                                     GNUNET_DISK_OPEN_READ,
                                       GNUNET_DISK_PERM_NONE);
           if (NULL == fd)
             {
-              GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", hostkeys_file);
+              GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, 
+                                       "open", 
+                                       hostkeys_file);
+             GNUNET_free (hostkeys_file);
+             for (i=0;i<pg->num_hosts;i++)
+               {
+                 GNUNET_free (pg->hosts[i].hostname);
+                 GNUNET_free_non_null (pg->hosts[i].username);
+               }
+             GNUNET_free (pg->peers);
+             GNUNET_free (pg->hosts);
+             GNUNET_free (pg);
               return NULL;
             }
 
@@ -5826,12 +5942,11 @@ GNUNET_TESTING_daemons_start(
                                                    GNUNET_YES))
             fs = 0;
 
-          GNUNET_log (
-                      GNUNET_ERROR_TYPE_WARNING,
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Found file size %llu for hostkeys, expect hostkeys to be size %d\n",
                       fs, HOSTKEYFILESIZE);
 
-          if (fs % HOSTKEYFILESIZE != 0)
+          if (0 != (fs % HOSTKEYFILESIZE))
             {
               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                           "File size %llu seems incorrect for hostkeys...\n",
@@ -5840,12 +5955,13 @@ GNUNET_TESTING_daemons_start(
           else
             {
               total_hostkeys = fs / HOSTKEYFILESIZE;
-              GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                          "Will read %llu hostkeys from file\n", total_hostkeys);
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                          "Will read %llu hostkeys from file\n",
+                         total_hostkeys);
               pg->hostkey_data = GNUNET_malloc_large (fs);
               GNUNET_assert (fs == GNUNET_DISK_file_read (fd, pg->hostkey_data, fs));
-              GNUNET_assert(GNUNET_OK == GNUNET_DISK_file_close(fd));
             }
+         GNUNET_assert(GNUNET_OK == GNUNET_DISK_file_close(fd));
         }
       GNUNET_free(hostkeys_file);
     }
@@ -5885,7 +6001,9 @@ GNUNET_TESTING_daemons_start(
             GNUNET_asprintf (&newservicehome, "%s/%s/%d/", baseservicehome, hostname, off);
           else
             GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
+#if !USE_START_HELPER
           GNUNET_free (baseservicehome);
+#endif
         }
       else
         {
@@ -5902,16 +6020,6 @@ GNUNET_TESTING_daemons_start(
                                              newservicehome);
       GNUNET_free (newservicehome);
       pg->peers[off].cfg = pcfg;
-#if DEFER
-      /* Can we do this later? */
-      pg->peers[off].allowed_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-      pg->peers[off].connect_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-      pg->peers[off].blacklisted_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-
-#endif
       pg->peers[off].pg = pg;
       pg->peers[off].internal_context.peer = &pg->peers[off];
       pg->peers[off].internal_context.timeout = timeout;
@@ -5925,11 +6033,77 @@ GNUNET_TESTING_daemons_start(
       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
       pg->peers[off].internal_context.start_cb = cb;
       pg->peers[off].internal_context.start_cb_cls = cb_cls;
-
+#if !USE_START_HELPER
       GNUNET_SCHEDULER_add_now (&internal_start,
                                 &pg->peers[off].internal_context);
+#else
+      if ((pg->hostkey_data != NULL) && (hostcnt > 0))
+        {
+          pg->peers[off].daemon = GNUNET_TESTING_daemon_start (pcfg,
+                                       timeout,
+                                       GNUNET_YES,
+                                       hostname,
+                                       username,
+                                       sshport,
+                                       pg->peers[off].internal_context.hostkey,
+                                       &internal_hostkey_callback,
+                                       &pg->peers[off].internal_context,
+                                       &internal_startup_callback,
+                                       &pg->peers[off].internal_context);
+          /**
+           * At this point, given that we had a hostkeyfile,
+           * we can call the hostkey callback!
+           * But first, we should copy (rsync) all of the configs
+           * and hostkeys to the remote peers.  Then let topology
+           * creation happen, then call the peer start helper processes,
+           * then set pg->whatever_phase for each peer and let them
+           * enter the fsm to get the HELLO's for peers and start connecting.
+           */
+        }
+      else
+        {
+          GNUNET_SCHEDULER_add_now (&internal_start,
+                                    &pg->peers[off].internal_context);
+        }
 
+#endif
     }
+
+#if USE_START_HELPER /* Now the peergroup has been set up, hostkeys and configs written to files. */
+  if ((pg->hostkey_data != NULL) && (hostcnt > 0))
+    {
+      for (off = 0; off < hostcnt; off++)
+        {
+          GNUNET_asprintf(&newservicehome, "%s/%s/", baseservicehome, pg->hosts[off].hostname);
+
+          if (NULL != username)
+            GNUNET_asprintf (&arg, "%s@%s:%s/%s", username, pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
+          else
+            GNUNET_asprintf (&arg, "%s:%s/%s", pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
+
+          /* FIXME: Doesn't support ssh_port option! */
+          proc = GNUNET_OS_start_process (NULL, NULL, "rsync", "rsync", "-r", newservicehome, arg, NULL);
+
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "copying directory with command rsync -r %s %s\n", newservicehome, arg);
+
+          GNUNET_free (arg);
+          if (NULL == proc)
+            {
+              GNUNET_log (
+                          GNUNET_ERROR_TYPE_ERROR,
+                          _
+                          ("Could not start `%s' process to copy configuration directory.\n"),
+                          "scp");
+              GNUNET_assert(0);
+            }
+          GNUNET_OS_process_wait (proc);
+          GNUNET_OS_process_close (proc);
+        }
+      /* Now all the configuration files and hostkeys are copied to the remote host.  Call the hostkey callback for each peer! */
+      GNUNET_SCHEDULER_add_now(&call_hostkey_callbacks, pg);
+    }
+  GNUNET_free (baseservicehome);
+#endif
   return pg;
 }
 
@@ -5943,8 +6117,7 @@ GNUNET_TESTING_daemon_get(struct GNUNET_TESTING_PeerGroup *pg,
 {
   if (position < pg->total)
     return pg->peers[position].daemon;
-  else
-    return NULL;
+  return NULL;
 }
 
 /*
@@ -5968,7 +6141,6 @@ GNUNET_TESTING_daemon_get_by_id(struct GNUNET_TESTING_PeerGroup *pg,
                        sizeof(struct GNUNET_PeerIdentity)))
         return pg->peers[i].daemon;
     }
-
   return NULL;
 }
 
@@ -5982,7 +6154,7 @@ GNUNET_TESTING_daemon_get_by_id(struct GNUNET_TESTING_PeerGroup *pg,
  * @param d handle to the daemon that was restarted
  * @param emsg NULL on success
  */
-void
+static void
 restart_callback(void *cls, const struct GNUNET_PeerIdentity *id,
                  const struct GNUNET_CONFIGURATION_Handle *cfg,
                  struct GNUNET_TESTING_Daemon *d, const char *emsg)
@@ -6021,7 +6193,7 @@ restart_callback(void *cls, const struct GNUNET_PeerIdentity *id,
  * @param emsg NULL on success, non-NULL on failure
  *
  */
-void
+static void
 churn_stop_callback(void *cls, const char *emsg)
 {
   struct ShutdownContext *shutdown_ctx = cls;
@@ -6125,6 +6297,7 @@ schedule_churn_shutdown_task(void *cls,
     }
 }
 
+
 /**
  * Simulate churn by stopping some peers (and possibly
  * re-starting others if churn is called multiple times).  This
@@ -6167,6 +6340,11 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   unsigned int *running_permute;
   unsigned int *stopped_permute;
 
+  shutdown_ctx = NULL;
+  peer_shutdown_ctx = NULL;
+  peer_restart_ctx = NULL;
+  churn_startup_ctx = NULL;
+
   running = 0;
   stopped = 0;
 
@@ -6315,6 +6493,7 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   GNUNET_free_non_null (stopped_permute);
 }
 
+
 /**
  * Restart all peers in the given group.
  *
@@ -6346,6 +6525,7 @@ GNUNET_TESTING_daemons_restart(struct GNUNET_TESTING_PeerGroup *pg,
     }
 }
 
+
 /**
  * Start or stop an individual peer from the given group.
  *
@@ -6402,13 +6582,14 @@ GNUNET_TESTING_daemons_vary(struct GNUNET_TESTING_PeerGroup *pg,
     GNUNET_break (0);
 }
 
+
 /**
  * Callback for shutting down peers in a peer group.
  *
  * @param cls closure (struct ShutdownContext)
  * @param emsg NULL on success
  */
-void
+static void
 internal_shutdown_callback(void *cls, const char *emsg)
 {
   struct PeerShutdownContext *peer_shutdown_ctx = cls;
@@ -6458,6 +6639,7 @@ internal_shutdown_callback(void *cls, const char *emsg)
   GNUNET_free(peer_shutdown_ctx);
 }
 
+
 /**
  * Task to rate limit the number of outstanding peer shutdown
  * requests.  This is necessary for making sure we don't do
@@ -6497,6 +6679,7 @@ schedule_shutdown_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
 }
 
+
 /**
  * Shutdown all peers started in the given group.
  *
@@ -6537,7 +6720,10 @@ GNUNET_TESTING_daemons_stop(struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
 
       if (NULL != pg->peers[off].cfg)
+      {
         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
+        pg->peers[off].cfg = NULL;
+      }
 #if OLD
       conn_iter = pg->peers[off].allowed_peers_head;
       while (conn_iter != NULL)
@@ -6572,12 +6758,12 @@ GNUNET_TESTING_daemons_stop(struct GNUNET_TESTING_PeerGroup *pg,
         }
 #else
       if (pg->peers[off].allowed_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
       if (pg->peers[off].connect_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
       if (pg->peers[off].blacklisted_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->
-          peers[off].blacklisted_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                              peers[off].blacklisted_peers);
 #endif
     }
 }