shutdown
[oweals/gnunet.git] / src / testing / testing_group.c
index f237ec030d3babfd6a32294fc1b61a4a1447c2fa..be9743b1f5b5ec2af46700e22f113bb4f435ea6a 100644 (file)
  *
  */
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
 #include "gnunet_core_service.h"
 
 #define VERBOSE_TESTING GNUNET_NO
 
-#define VERBOSE_TOPOLOGY GNUNET_YES
+#define VERBOSE_TOPOLOGY GNUNET_NO
 
 #define DEBUG_CHURN GNUNET_NO
 
+#define USE_START_HELPER GNUNET_YES
+
 #define OLD 1
 
+/* Before connecting peers, send all of the HELLOs */
 #define USE_SEND_HELLOS GNUNET_NO
 
 #define TOPOLOGY_HACK GNUNET_YES
 
+
 /**
  * Lowest port used for GNUnet testing.  Should be high enough to not
  * conflict with other applications running on the hosts but be low
@@ -1569,7 +1574,7 @@ add_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   int added;
   int add_first;
   int add_second;
-#if OLD
+
   struct PeerConnection **first_list;
   struct PeerConnection **second_list;
   struct PeerConnection *first_iter;
@@ -1578,15 +1583,7 @@ add_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
   struct PeerConnection *new_second;
   struct PeerConnection **first_tail;
   struct PeerConnection **second_tail;
-#else
-  GNUNET_HashCode hash_first;
-  GNUNET_HashCode hash_second;
-
-  hash_from_uid (first, &hash_first);
-  hash_from_uid (second, &hash_second);
-#endif
 
-#if OLD
   switch (list)
     {
   case ALLOWED:
@@ -1645,59 +1642,22 @@ add_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first,
           second_iter = second_iter->next;
         }
     }
-#else
-  if (GNUNET_NO ==
-      GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].blacklisted_peers,
-          &hash_second))
-    {
-      add_first = GNUNET_YES;
-    }
-
-  if (GNUNET_NO ==
-      GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].blacklisted_peers,
-          &hash_first))
-    {
-      add_second = GNUNET_YES;
-    }
-#endif
 
   added = 0;
   if (add_first)
     {
-#if OLD
       new_first = GNUNET_malloc (sizeof (struct PeerConnection));
       new_first->index = second;
       GNUNET_CONTAINER_DLL_insert(*first_list, *first_tail, new_first);
-#else
-      GNUNET_assert (GNUNET_OK ==
-          GNUNET_CONTAINER_multihashmap_put (pg->
-              peers
-              [first].blacklisted_peers,
-              &hash_second,
-              pg->
-              peers[second].daemon,
-              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-#endif
       pg->peers[first].num_connections++;
       added++;
     }
 
   if (add_second)
     {
-#if OLD
       new_second = GNUNET_malloc (sizeof (struct PeerConnection));
       new_second->index = first;
       GNUNET_CONTAINER_DLL_insert(*second_list, *second_tail, new_second);
-#else
-      GNUNET_assert (GNUNET_OK ==
-          GNUNET_CONTAINER_multihashmap_put (pg->
-              peers
-              [second].blacklisted_peers,
-              &hash_first,
-              pg->
-              peers[first].daemon,
-              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
-#endif
       pg->peers[second].num_connections++;
       added++;
     }
@@ -2401,7 +2361,7 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
   connect_attempts = 0;
 
   conn_meter = create_meter ((((pg->total * pg->total) + pg->total) / 2)
-      - pg->total, "Create Clique ", GNUNET_YES);
+      - pg->total, "Create Clique ", GNUNET_NO);
   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
     {
       for (inner_count = outer_count + 1; inner_count < pg->total; inner_count++)
@@ -2415,8 +2375,6 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
           update_meter (conn_meter);
         }
     }
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Meter has %d left\n",
-              conn_meter->total - conn_meter->completed);
   reset_meter (conn_meter);
   free_meter (conn_meter);
   return connect_attempts;
@@ -2579,6 +2537,7 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
 
   buf = data;
   count = 0;
+  first_peer_index = 0;
   /* First line should contain a single integer, specifying the number of peers */
   /* Each subsequent line should contain this format PEER_INDEX:OTHER_PEER_INDEX[,...] */
   curr_state = NUM_PEERS;
@@ -2593,7 +2552,9 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
       switch (curr_state)
         {
       case NUM_PEERS:
-        if (1 != sscanf (&buf[count], "%u", &total_peers))
+        errno = 0;
+        total_peers = strtoul(&buf[count], NULL, 10);
+        if (errno != 0)
           {
             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                         "Failed to read number of peers from topology file!\n");
@@ -2609,7 +2570,9 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
         count++;
         break;
       case PEER_INDEX:
-        if (1 != sscanf (&buf[count], "%u", &first_peer_index))
+        errno = 0;
+        first_peer_index = strtoul(&buf[count], NULL, 10);
+        if (errno != 0)
           {
             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                         "Failed to read peer index from topology file!\n");
@@ -2627,7 +2590,9 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
         count++;
         break;
       case OTHER_PEER_INDEX:
-        if (1 != sscanf (&buf[count], "%u", &second_peer_index))
+        errno = 0;
+        second_peer_index = strtoul(&buf[count], NULL, 10);
+        if (errno != 0)
           {
             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                         "Failed to peer index from topology file!\n");
@@ -2636,7 +2601,7 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
           }
         /* Assume file is written with first peer 1, but array index is 0 */
         connect_attempts += proc (pg, first_peer_index - 1, second_peer_index
-            - 1, list, GNUNET_YES);
+                                  - 1, list, GNUNET_YES);
         while ((buf[count] != '\n') && (buf[count] != ',') && (count
             < frstat.st_size - 1))
           count++;
@@ -2659,17 +2624,6 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
         }
 
     }
-#if 0
-  /* Connect each peer to the next highest numbered peer */
-  for (count = 0; count < pg->total - 1; count++)
-    {
-#if VERBOSE_TESTING
-      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-          "Connecting peer %d to peer %d\n", first_peer_index, second_peer_index);
-#endif
-      connect_attempts += proc (pg, first_peer_index, second_peer_index);
-    }
-#endif
   return connect_attempts;
 }
 
@@ -2859,6 +2813,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv", "mv",
                                                       mytemp, arg, NULL);
+          GNUNET_assert(procarr[pg_iter] != NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("Copying file with command cp %s %s\n"), mytemp, arg);
@@ -2880,7 +2835,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           ret = GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: schedule this, throttle! */
           GNUNET_OS_process_close (procarr[pg_iter]);
           if (ret != GNUNET_OK)
@@ -3078,7 +3033,7 @@ create_and_copy_blacklist_files(struct GNUNET_TESTING_PeerGroup *pg,
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: add scheduled blacklist file copy that parallelizes file copying! */
 
 #if VERBOSE_TESTING
@@ -3433,7 +3388,7 @@ hello_sent_callback (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   {
     struct SendHelloContext *send_hello_context = cls;
     //unsigned int pg_iter;
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3491,7 +3446,7 @@ static void schedule_send_hellos (void *cls, const struct GNUNET_SCHEDULER_TaskC
     struct SendHelloContext *send_hello_context = cls;
     struct GNUNET_TESTING_PeerGroup *pg = send_hello_context->pg;
 
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3522,12 +3477,9 @@ static void schedule_send_hellos (void *cls, const struct GNUNET_SCHEDULER_TaskC
         if (send_hello_context->peer->daemon->th == NULL)
           {
             pg->outstanding_connects++; /* Actual TRANSPORT, CORE connections! */
-            send_hello_context->peer->daemon->th = GNUNET_TRANSPORT_connect(send_hello_context->peer->cfg,
-                NULL,
-                send_hello_context,
-                NULL,
-                NULL,
-                NULL);
+            send_hello_context->peer->daemon->th
+                = GNUNET_TRANSPORT_connect (send_hello_context->peer->cfg, NULL,
+                                            send_hello_context, NULL, NULL, NULL);
           }
 #if DEBUG_TESTING
         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
@@ -3622,7 +3574,7 @@ schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct ConnectContext *connect_context = cls;
   struct GNUNET_TESTING_PeerGroup *pg = connect_context->ct_ctx->pg;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if ((pg->outstanding_connects > pg->max_outstanding_connections)
@@ -3654,7 +3606,7 @@ schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
                                       connect_context->ct_ctx->connect_timeout,
                                       connect_context->ct_ctx->connect_attempts,
 #if USE_SEND_HELLOS
-                                       GNUNET_NO,
+                                      GNUNET_NO,
 #else
                                       GNUNET_YES,
 #endif
@@ -3743,7 +3695,7 @@ copy_allowed_topology(struct GNUNET_TESTING_PeerGroup *pg)
                       "Creating connection between %d and %d\n", pg_iter,
                       iter->index);
           total += add_connections (pg, pg_iter, iter->index, CONNECT,
-                                    GNUNET_NO);
+                                    GNUNET_YES);
           //total += add_actual_connections(pg, pg_iter, iter->index);
           iter = iter->next;
         }
@@ -3789,7 +3741,7 @@ connect_topology(struct GNUNET_TESTING_PeerGroup *pg,
   struct PeerConnection *connection_iter;
 #endif
 #if USE_SEND_HELLOS
-  struct SendHelloContext *send_hello_context
+  struct SendHelloContext *send_hello_context;
 #endif
 
   total = 0;
@@ -4007,6 +3959,8 @@ GNUNET_TESTING_create_topology(struct GNUNET_TESTING_PeerGroup *pg,
   if ((restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE) && (restrict_topology
       != GNUNET_TESTING_TOPOLOGY_FROM_FILE))
     create_clique (pg, &add_connections, BLACKLIST, GNUNET_NO);
+  else
+    return num_connections;
 
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
@@ -4847,7 +4801,7 @@ schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct CoreContext *core_context = cls;
   struct TopologyIterateContext *topology_context =
       (struct TopologyIterateContext *) core_context->iter_context;
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (topology_context->connected
@@ -4991,7 +4945,7 @@ schedule_get_statistics(void *cls,
   struct StatsIterateContext *stats_context =
       (struct StatsIterateContext *) core_context->iter_context;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (stats_context->connected > stats_context->pg->max_outstanding_connections)
@@ -5473,7 +5427,7 @@ internal_continue_startup(void *cls,
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5580,12 +5534,13 @@ schedule_churn_restart(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
     }
 }
 
+
 static void
 internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5604,6 +5559,7 @@ internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
       internal_context->peer->daemon
           = GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
                                          internal_context->timeout,
+                                         GNUNET_NO,
                                          internal_context->hostname,
                                          internal_context->username,
                                          internal_context->sshport,
@@ -5622,6 +5578,89 @@ internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
                                     &internal_start, internal_context);
     }
 }
+#if USE_START_HELPER
+
+struct PeerStartHelperContext
+{
+  struct GNUNET_TESTING_PeerGroup *pg;
+
+  struct HostData *host;
+
+  struct GNUNET_OS_Process *proc;
+};
+
+static void
+check_peers_started (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerStartHelperContext *helper = cls;
+  enum GNUNET_OS_ProcessStatusType type;
+  unsigned long code;
+  unsigned int i;
+  GNUNET_TESTING_NotifyDaemonRunning cb;
+
+  if (GNUNET_NO == GNUNET_OS_process_status (helper->proc, &type, &code)) /* Still running, wait some more! */
+  {
+    GNUNET_SCHEDULER_add_delayed(GNUNET_CONSTANTS_EXEC_WAIT, &check_peers_started, helper);
+    return;
+  }
+
+  helper->pg->starting--;
+  if (helper->pg->starting == 0) /* All peers have finished starting! */
+    {
+      /* Call the peer started callback for each peer, set proper FSM state (?) */
+      for (i = 0; i < helper->pg->total; i++)
+        {
+          cb = helper->pg->peers[i].daemon->cb;
+          helper->pg->peers[i].daemon->cb = NULL;
+          helper->pg->peers[i].daemon->running = GNUNET_YES;
+          helper->pg->peers[i].daemon->phase = SP_START_DONE;
+          if (NULL != cb)
+          {
+            if ((type != GNUNET_OS_PROCESS_EXITED) || (code != 0))
+              cb (helper->pg->peers[i].daemon->cb_cls,
+                  &helper->pg->peers[i].daemon->id,
+                  helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
+                  "Failed to execute peerStartHelper.pl, or return code bad!");
+            else
+              cb (helper->pg->peers[i].daemon->cb_cls,
+                  &helper->pg->peers[i].daemon->id,
+                  helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
+                  NULL);
+
+          }
+
+        }
+    }
+  GNUNET_OS_process_close(helper->proc);
+}
+
+static void
+start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerStartHelperContext *helper = cls;
+  char *baseservicehome;
+  char *tempdir;
+  char *arg;
+  /* ssh user@host peerStartHelper /path/to/basedirectory */
+  GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (helper->pg->cfg, "PATHS", "SERVICEHOME",
+                                                                    &baseservicehome));
+  GNUNET_asprintf(&tempdir, "%s/%s/", baseservicehome, helper->host->hostname);
+  if (NULL != helper->host->username)
+    GNUNET_asprintf (&arg, "%s@%s", helper->host->username, helper->host->hostname);
+  else
+    GNUNET_asprintf (&arg, "%s", helper->host->hostname);
+
+  /* FIXME: Doesn't support ssh_port option! */
+  helper->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
+                                  "peerStartHelper.pl", tempdir,  NULL);
+  GNUNET_assert(helper->proc != NULL);
+  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "starting peers with cmd ssh %s %s %s\n", arg, "peerStartHelper.pl", tempdir);
+  GNUNET_SCHEDULER_add_now (&check_peers_started, helper);
+  GNUNET_free (tempdir);
+  GNUNET_free (baseservicehome);
+  GNUNET_free (arg);
+}
+#endif
 
 /**
  * Function which continues a peer group starting up
@@ -5635,15 +5674,58 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
 {
   unsigned int i;
 
+#if USE_START_HELPER
+  if ((pg->num_hosts > 0) && (pg->hostkey_data != NULL))
+    {
+      struct PeerStartHelperContext *helper;
+      pg->starting = pg->num_hosts;
+      for (i = 0; i < pg->num_hosts; i++)
+        {
+          helper = GNUNET_malloc(sizeof(struct PeerStartHelperContext));
+          helper->pg = pg;
+          helper->host = &pg->hosts[i];
+          GNUNET_SCHEDULER_add_now(&start_peer_helper, helper);
+        }
+    }
+  else
+    {
+      pg->starting = 0;
+      for (i = 0; i < pg->total; i++)
+        {
+          GNUNET_SCHEDULER_add_now (&internal_continue_startup,
+                                    &pg->peers[i].internal_context);
+        }
+    }
+#else
   pg->starting = 0;
   for (i = 0; i < pg->total; i++)
     {
       GNUNET_SCHEDULER_add_now (&internal_continue_startup,
                                 &pg->peers[i].internal_context);
-      //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
     }
+#endif
 }
 
+#if USE_START_HELPER
+static void
+call_hostkey_callbacks (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct GNUNET_TESTING_PeerGroup *pg = cls;
+  unsigned int i;
+  for (i = 0; i < pg->total; i++)
+    {
+      if (pg->peers[i].internal_context.hostkey_callback != NULL)
+        pg->peers[i].internal_context.hostkey_callback (pg->peers[i].internal_context.hostkey_cls,
+                                                           &pg->peers[i].daemon->id,
+                                                           pg->peers[i].daemon,
+                                                           NULL);
+    }
+
+  if (pg->peers[0].internal_context.hostkey_callback == NULL)
+    GNUNET_TESTING_daemons_continue_startup (pg);
+}
+#endif
+
 /**
  * Start count gnunet instances with the same set of transports and
  * applications.  The port numbers (any option called "PORT") will be
@@ -5672,8 +5754,7 @@ GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
  * @return NULL on error, otherwise handle to control peer group
  */
 struct GNUNET_TESTING_PeerGroup *
-GNUNET_TESTING_daemons_start(
-                             const struct GNUNET_CONFIGURATION_Handle *cfg,
+GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
                              unsigned int total,
                              unsigned int max_concurrent_connections,
                              unsigned int max_concurrent_ssh,
@@ -5688,11 +5769,6 @@ GNUNET_TESTING_daemons_start(
 {
   struct GNUNET_TESTING_PeerGroup *pg;
   const struct GNUNET_TESTING_Host *hostpos;
-#if 0
-  char *pos;
-  const char *rpos;
-  char *start;
-#endif
   const char *hostname;
   const char *username;
   char *baseservicehome;
@@ -5704,6 +5780,7 @@ GNUNET_TESTING_daemons_start(
   struct GNUNET_DISK_FileHandle *fd;
   struct GNUNET_CONFIGURATION_Handle *pcfg;
   unsigned int off;
+  struct OutstandingSSH *ssh_entry;
   unsigned int hostcnt;
   unsigned int i;
   uint16_t minport;
@@ -5714,6 +5791,7 @@ GNUNET_TESTING_daemons_start(
   uint64_t total_hostkeys;
   struct GNUNET_OS_Process *proc;
 
+  username = NULL;
   if (0 == total)
     {
       GNUNET_break (0);
@@ -5762,51 +5840,6 @@ GNUNET_TESTING_daemons_start(
       hostcnt = off;
       minport = 0;
       pg->num_hosts = off;
-
-#if NO_LL
-      off = 2;
-      /* skip leading spaces */
-      while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
-      hostnames++;
-      rpos = hostnames;
-      while ('\0' != *rpos)
-        {
-          if (isspace ((unsigned char) *rpos))
-          off++;
-          rpos++;
-        }
-      pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
-      off = 0;
-      start = GNUNET_strdup (hostnames);
-      pos = start;
-      while ('\0' != *pos)
-        {
-          if (isspace ((unsigned char) *pos))
-            {
-              *pos = '\0';
-              if (strlen (start) > 0)
-                {
-                  pg->hosts[off].minport = LOW_PORT;
-                  pg->hosts[off++].hostname = start;
-                }
-              start = pos + 1;
-            }
-          pos++;
-        }
-      if (strlen (start) > 0)
-        {
-          pg->hosts[off].minport = LOW_PORT;
-          pg->hosts[off++].hostname = start;
-        }
-      if (off == 0)
-        {
-          GNUNET_free (start);
-          GNUNET_free (pg->hosts);
-          pg->hosts = NULL;
-        }
-      hostcnt = off;
-      minport = 0; /* make gcc happy */
-#endif
     }
   else
     {
@@ -5816,13 +5849,13 @@ GNUNET_TESTING_daemons_start(
 
   /* Create the servicehome directory for each remote peer */
   GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "PATHS", "SERVICEHOME",
-          &baseservicehome));
+                                                                    &baseservicehome));
   for (i = 0; i < pg->num_hosts; i++)
     {
-      struct OutstandingSSH *ssh_entry;
       ssh_entry = GNUNET_malloc(sizeof(struct OutstandingSSH));
       ssh_entry->hostname = pg->hosts[i].hostname; /* Don't free! */
       GNUNET_CONTAINER_DLL_insert(pg->ssh_head, pg->ssh_tail, ssh_entry);
+      GNUNET_asprintf(&tmpdir, "%s/%s", baseservicehome, pg->hosts[i].hostname);
       if (NULL != pg->hosts[i].username)
         GNUNET_asprintf (&arg, "%s@%s", pg->hosts[i].username,
                          pg->hosts[i].hostname);
@@ -5836,16 +5869,20 @@ GNUNET_TESTING_daemons_start(
 #if !DEBUG_TESTING
                                           "-q",
 #endif
-                                          arg, "mkdir -p", baseservicehome,
+                                          arg, "mkdir -p", tmpdir,
                                           NULL);
         }
       else
         proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
-                                        "mkdir -p", baseservicehome, NULL);
+                                        "mkdir -p", tmpdir, NULL);
+      GNUNET_assert(proc != NULL);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Creating remote dir with command ssh %s %s %s\n", arg,
-                  " mkdir -p ", baseservicehome);
+                  " mkdir -p ", tmpdir);
+      GNUNET_free(tmpdir);
+      GNUNET_free(arg);
       GNUNET_OS_process_wait (proc);
+      GNUNET_OS_process_close(proc);
     }
   GNUNET_free(baseservicehome);
 
@@ -5871,7 +5908,7 @@ GNUNET_TESTING_daemons_start(
             fs = 0;
 
           GNUNET_log (
-                      GNUNET_ERROR_TYPE_WARNING,
+                      GNUNET_ERROR_TYPE_DEBUG,
                       "Found file size %llu for hostkeys, expect hostkeys to be size %d\n",
                       fs, HOSTKEYFILESIZE);
 
@@ -5884,7 +5921,7 @@ GNUNET_TESTING_daemons_start(
           else
             {
               total_hostkeys = fs / HOSTKEYFILESIZE;
-              GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                           "Will read %llu hostkeys from file\n", total_hostkeys);
               pg->hostkey_data = GNUNET_malloc_large (fs);
               GNUNET_assert (fs == GNUNET_DISK_file_read (fd, pg->hostkey_data, fs));
@@ -5925,30 +5962,29 @@ GNUNET_TESTING_daemons_start(
                                                     "SERVICEHOME",
                                                     &baseservicehome))
         {
-          GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
+          if (hostname != NULL)
+            GNUNET_asprintf (&newservicehome, "%s/%s/%d/", baseservicehome, hostname, off);
+          else
+            GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
+#if !USE_START_HELPER
           GNUNET_free (baseservicehome);
+#endif
         }
       else
         {
           tmpdir = getenv ("TMPDIR");
           tmpdir = tmpdir ? tmpdir : "/tmp";
-          GNUNET_asprintf (&newservicehome, "%s/%s/%d/", tmpdir,
-                           "gnunet-testing-test-test", off);
+          if (hostname != NULL)
+            GNUNET_asprintf (&newservicehome, "%s/%s/%s/%d/", tmpdir, hostname,
+                             "gnunet-testing-test-test", off);
+          else
+            GNUNET_asprintf (&newservicehome, "%s/%s/%d/", tmpdir,
+                             "gnunet-testing-test-test", off);
         }
       GNUNET_CONFIGURATION_set_value_string (pcfg, "PATHS", "SERVICEHOME",
                                              newservicehome);
       GNUNET_free (newservicehome);
       pg->peers[off].cfg = pcfg;
-#if DEFER
-      /* Can we do this later? */
-      pg->peers[off].allowed_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-      pg->peers[off].connect_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-      pg->peers[off].blacklisted_peers =
-      GNUNET_CONTAINER_multihashmap_create (total);
-
-#endif
       pg->peers[off].pg = pg;
       pg->peers[off].internal_context.peer = &pg->peers[off];
       pg->peers[off].internal_context.timeout = timeout;
@@ -5962,11 +5998,77 @@ GNUNET_TESTING_daemons_start(
       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
       pg->peers[off].internal_context.start_cb = cb;
       pg->peers[off].internal_context.start_cb_cls = cb_cls;
-
+#if !USE_START_HELPER
       GNUNET_SCHEDULER_add_now (&internal_start,
                                 &pg->peers[off].internal_context);
+#else
+      if ((pg->hostkey_data != NULL) && (hostcnt > 0))
+        {
+          pg->peers[off].daemon = GNUNET_TESTING_daemon_start (pcfg,
+                                       timeout,
+                                       GNUNET_YES,
+                                       hostname,
+                                       username,
+                                       sshport,
+                                       pg->peers[off].internal_context.hostkey,
+                                       &internal_hostkey_callback,
+                                       &pg->peers[off].internal_context,
+                                       &internal_startup_callback,
+                                       &pg->peers[off].internal_context);
+          /**
+           * At this point, given that we had a hostkeyfile,
+           * we can call the hostkey callback!
+           * But first, we should copy (rsync) all of the configs
+           * and hostkeys to the remote peers.  Then let topology
+           * creation happen, then call the peer start helper processes,
+           * then set pg->whatever_phase for each peer and let them
+           * enter the fsm to get the HELLO's for peers and start connecting.
+           */
+        }
+      else
+        {
+          GNUNET_SCHEDULER_add_now (&internal_start,
+                                    &pg->peers[off].internal_context);
+        }
 
+#endif
     }
+
+#if USE_START_HELPER /* Now the peergroup has been set up, hostkeys and configs written to files. */
+  if ((pg->hostkey_data != NULL) && (hostcnt > 0))
+    {
+      for (off = 0; off < hostcnt; off++)
+        {
+          GNUNET_asprintf(&newservicehome, "%s/%s/", baseservicehome, pg->hosts[off].hostname);
+
+          if (NULL != username)
+            GNUNET_asprintf (&arg, "%s@%s:%s/%s", username, pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
+          else
+            GNUNET_asprintf (&arg, "%s:%s/%s", pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
+
+          /* FIXME: Doesn't support ssh_port option! */
+          proc = GNUNET_OS_start_process (NULL, NULL, "rsync", "rsync", "-r", newservicehome, arg, NULL);
+
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "copying directory with command rsync -r %s %s\n", newservicehome, arg);
+
+          GNUNET_free (arg);
+          if (NULL == proc)
+            {
+              GNUNET_log (
+                          GNUNET_ERROR_TYPE_ERROR,
+                          _
+                          ("Could not start `%s' process to copy configuration directory.\n"),
+                          "scp");
+              GNUNET_assert(0);
+            }
+          GNUNET_OS_process_wait (proc);
+          GNUNET_OS_process_close (proc);
+        }
+      /* Now all the configuration files and hostkeys are copied to the remote host.  Call the hostkey callback for each peer! */
+      GNUNET_SCHEDULER_add_now(&call_hostkey_callbacks, pg);
+    }
+  GNUNET_free (baseservicehome);
+#endif
   return pg;
 }
 
@@ -6204,6 +6306,11 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   unsigned int *running_permute;
   unsigned int *stopped_permute;
 
+  shutdown_ctx = NULL;
+  peer_shutdown_ctx = NULL;
+  peer_restart_ctx = NULL;
+  churn_startup_ctx = NULL;
+
   running = 0;
   stopped = 0;
 
@@ -6449,8 +6556,9 @@ void
 internal_shutdown_callback(void *cls, const char *emsg)
 {
   struct PeerShutdownContext *peer_shutdown_ctx = cls;
-  struct ShutdownContext *shutdown_ctx = cls;
+  struct ShutdownContext *shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
   unsigned int off;
+  struct OutstandingSSH *ssh_pos;
 
   shutdown_ctx->outstanding--;
   if (peer_shutdown_ctx->daemon->hostname != NULL)
@@ -6483,8 +6591,12 @@ internal_shutdown_callback(void *cls, const char *emsg)
           GNUNET_free_non_null (shutdown_ctx->pg->hosts[off].username);
         }
       GNUNET_free_non_null (shutdown_ctx->pg->hosts);
+      while (NULL != (ssh_pos = shutdown_ctx->pg->ssh_head))
+        {
+          GNUNET_CONTAINER_DLL_remove(shutdown_ctx->pg->ssh_head, shutdown_ctx->pg->ssh_tail, ssh_pos);
+          GNUNET_free(ssh_pos);
+        }
       GNUNET_free (shutdown_ctx->pg);
-
       GNUNET_free (shutdown_ctx);
     }
   GNUNET_free(peer_shutdown_ctx);
@@ -6569,7 +6681,10 @@ GNUNET_TESTING_daemons_stop(struct GNUNET_TESTING_PeerGroup *pg,
       GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
 
       if (NULL != pg->peers[off].cfg)
+      {
         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
+        pg->peers[off].cfg = NULL;
+      }
 #if OLD
       conn_iter = pg->peers[off].allowed_peers_head;
       while (conn_iter != NULL)