leak
[oweals/gnunet.git] / src / testing / testing_group.c
index 9fdee3a1383eae944e8da8ac03e4d76d6424eff8..033ab7d5edc00315a63d50a43ba8eaeddc1d032c 100644 (file)
@@ -26,6 +26,7 @@
  *
  */
 #include "platform.h"
+#include "gnunet_constants.h"
 #include "gnunet_arm_service.h"
 #include "gnunet_testing_lib.h"
 #include "gnunet_core_service.h"
@@ -1409,6 +1410,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics", "UNIXPATH",
                                              "");
 
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
       GNUNET_free_non_null (control_host);
       GNUNET_free (allowed_hosts);
     }
@@ -1422,6 +1427,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
                                              hostname);
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "BINDTO",
                                              hostname);
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
                                              allowed_hosts);
       GNUNET_free (allowed_hosts);
@@ -1432,6 +1441,10 @@ make_config(const struct GNUNET_CONFIGURATION_Handle *cfg, uint32_t off,
                                              "127.0.0.1");
       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "BINDTO",
                                              "127.0.0.1");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp", "USE_LOCALADDR",
+                                             "YES");
+      GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp", "USE_LOCALADDR",
+                                             "YES");
     }
 
   *port = (uint16_t) uc.nport;
@@ -2360,7 +2373,7 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
   connect_attempts = 0;
 
   conn_meter = create_meter ((((pg->total * pg->total) + pg->total) / 2)
-      - pg->total, "Create Clique ", GNUNET_YES);
+      - pg->total, "Create Clique ", GNUNET_NO);
   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
     {
       for (inner_count = outer_count + 1; inner_count < pg->total; inner_count++)
@@ -2374,8 +2387,6 @@ create_clique(struct GNUNET_TESTING_PeerGroup *pg,
           update_meter (conn_meter);
         }
     }
-  GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Meter has %d left\n",
-              conn_meter->total - conn_meter->completed);
   reset_meter (conn_meter);
   free_meter (conn_meter);
   return connect_attempts;
@@ -2425,16 +2436,17 @@ static unsigned int
 copy_allowed(struct GNUNET_TESTING_PeerGroup *pg,
              GNUNET_TESTING_ConnectionProcessor proc)
 {
-  struct UnblacklistContext un_ctx;
   unsigned int count;
   unsigned int total;
   struct PeerConnection *iter;
+#if !OLD
+  struct UnblacklistContext un_ctx;
 
   un_ctx.pg = pg;
+#endif
   total = 0;
   for (count = 0; count < pg->total - 1; count++)
     {
-      un_ctx.first_uid = count;
 #if OLD
       iter = pg->peers[count].allowed_peers_head;
       while (iter != NULL)
@@ -2444,10 +2456,15 @@ copy_allowed(struct GNUNET_TESTING_PeerGroup *pg,
           iter = iter->next;
         }
 #else
-      total += GNUNET_CONTAINER_multihashmap_iterate(pg->peers[count].allowed_peers, &unblacklist_iterator, &un_ctx);
+      un_ctx.first_uid = count;
+      total += GNUNET_CONTAINER_multihashmap_iterate(pg->peers[count].allowed_peers, 
+                                                    &unblacklist_iterator,
+                                                    &un_ctx);
 #endif
     }
-  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Unblacklisted %u peers\n", total);
+  GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, 
+             "Unblacklisted %u peers\n", 
+             total);
   return total;
 }
 
@@ -2468,16 +2485,16 @@ create_line(struct GNUNET_TESTING_PeerGroup *pg,
             GNUNET_TESTING_ConnectionProcessor proc, enum PeerLists list)
 {
   unsigned int count;
-  int connect_attempts;
+  unsigned int connect_attempts;
 
   connect_attempts = 0;
-
   /* Connect each peer to the next highest numbered peer */
   for (count = 0; count < pg->total - 1; count++)
     {
 #if VERBOSE_TESTING
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
-          "Connecting peer %d to peer %d\n", count, count + 1);
+                 "Connecting peer %d to peer %d\n", 
+                 count, count + 1);
 #endif
       connect_attempts += proc (pg, count, count + 1, list, GNUNET_YES);
     }
@@ -2505,15 +2522,14 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
   int connect_attempts;
   unsigned int first_peer_index;
   unsigned int second_peer_index;
-  connect_attempts = 0;
   struct stat frstat;
   int count;
   char *data;
-  char *buf;
+  const char *buf;
   unsigned int total_peers;
-
   enum States curr_state;
 
+  connect_attempts = 0;
   if (GNUNET_OK != GNUNET_DISK_file_test (filename))
     GNUNET_DISK_fn_write (filename, NULL, 0, GNUNET_DISK_PERM_USER_READ);
 
@@ -2552,79 +2568,80 @@ create_from_file(struct GNUNET_TESTING_PeerGroup *pg, char *filename,
 
       switch (curr_state)
         {
-      case NUM_PEERS:
-        errno = 0;
-        total_peers = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to read number of peers from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                    "Read %u total peers in topology\n", total_peers);
-        GNUNET_assert(total_peers == pg->total);
-        curr_state = PEER_INDEX;
-        while ((buf[count] != '\n') && (count < frstat.st_size - 1))
-          count++;
-        count++;
-        break;
-      case PEER_INDEX:
-        errno = 0;
-        first_peer_index = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to read peer index from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        while ((buf[count] != ':') && (count < frstat.st_size - 1))
-          count++;
-        count++;
-        curr_state = OTHER_PEER_INDEX;
-        break;
-      case COLON:
-        if (1 == sscanf (&buf[count], ":"))
-          curr_state = OTHER_PEER_INDEX;
-        count++;
-        break;
-      case OTHER_PEER_INDEX:
-        errno = 0;
-        second_peer_index = strtoul(&buf[count], NULL, 10);
-        if (errno != 0)
-          {
-            GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                        "Failed to peer index from topology file!\n");
-            GNUNET_free_non_null(data);
-            return connect_attempts;
-          }
-        /* Assume file is written with first peer 1, but array index is 0 */
-        connect_attempts += proc (pg, first_peer_index - 1, second_peer_index
-                                  - 1, list, GNUNET_YES);
-        while ((buf[count] != '\n') && (buf[count] != ',') && (count
-            < frstat.st_size - 1))
-          count++;
-        if (buf[count] == '\n')
-          {
-            curr_state = PEER_INDEX;
-          }
-        else if (buf[count] != ',')
-          {
-            curr_state = OTHER_PEER_INDEX;
-          }
-        count++;
-        break;
-      default:
-        GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
-                    "Found bad data in topology file while in state %d!\n",
-                    curr_state);
-        GNUNET_break(0);
-        return connect_attempts;
+       case NUM_PEERS:
+         errno = 0;
+         total_peers = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to read number of peers from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                     "Read %u total peers in topology\n", total_peers);
+         GNUNET_assert(total_peers == pg->total);
+         curr_state = PEER_INDEX;
+         while ((buf[count] != '\n') && (count < frstat.st_size - 1))
+           count++;
+         count++;
+         break;
+       case PEER_INDEX:
+         errno = 0;
+         first_peer_index = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to read peer index from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         while ((buf[count] != ':') && (count < frstat.st_size - 1))
+           count++;
+         count++;
+         curr_state = OTHER_PEER_INDEX;
+         break;
+       case COLON:
+         if (1 == sscanf (&buf[count], ":"))
+           curr_state = OTHER_PEER_INDEX;
+         count++;
+         break;
+       case OTHER_PEER_INDEX:
+         errno = 0;
+         second_peer_index = strtoul(&buf[count], NULL, 10);
+         if (errno != 0)
+           {
+             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
+                         "Failed to peer index from topology file!\n");
+             GNUNET_free (data);
+             return connect_attempts;
+           }
+         /* Assume file is written with first peer 1, but array index is 0 */
+         connect_attempts += proc (pg, first_peer_index - 1, second_peer_index
+                                   - 1, list, GNUNET_YES);
+         while ((buf[count] != '\n') && (buf[count] != ',') && (count
+                                                                < frstat.st_size - 1))
+           count++;
+         if (buf[count] == '\n')
+           {
+             curr_state = PEER_INDEX;
+           }
+         else if (buf[count] != ',')
+           {
+             curr_state = OTHER_PEER_INDEX;
+           }
+         count++;
+         break;
+       default:
+         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                     "Found bad data in topology file while in state %d!\n",
+                     curr_state);
+         GNUNET_break(0);
+         GNUNET_free (data);
+         return connect_attempts;
         }
-
     }
+  GNUNET_free (data);
   return connect_attempts;
 }
 
@@ -2814,6 +2831,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv", "mv",
                                                       mytemp, arg, NULL);
+          GNUNET_assert(procarr[pg_iter] != NULL);
 #if VERBOSE_TESTING
           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
               _("Copying file with command cp %s %s\n"), mytemp, arg);
@@ -2835,7 +2853,7 @@ create_and_copy_friend_files(struct GNUNET_TESTING_PeerGroup *pg)
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           ret = GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: schedule this, throttle! */
           GNUNET_OS_process_close (procarr[pg_iter]);
           if (ret != GNUNET_OK)
@@ -3033,7 +3051,7 @@ create_and_copy_blacklist_files(struct GNUNET_TESTING_PeerGroup *pg,
                              temp_service_path);
           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp", "scp",
                                                       mytemp, arg, NULL);
-
+          GNUNET_assert(procarr[pg_iter] != NULL);
           GNUNET_OS_process_wait (procarr[pg_iter]); /* FIXME: add scheduled blacklist file copy that parallelizes file copying! */
 
 #if VERBOSE_TESTING
@@ -3232,7 +3250,6 @@ send_core_connect_requests (void *cls, const struct GNUNET_SCHEDULER_TaskContext
         while (conn != NULL)
           {
             GNUNET_CORE_peer_request_connect(send_hello_context->peer->daemon->server,
-                GNUNET_TIME_relative_get_forever(),
                 &send_hello_context->pg->peers[conn->index].daemon->id,
                 NULL,
                 NULL);
@@ -3388,7 +3405,7 @@ hello_sent_callback (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   {
     struct SendHelloContext *send_hello_context = cls;
     //unsigned int pg_iter;
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3446,7 +3463,7 @@ static void schedule_send_hellos (void *cls, const struct GNUNET_SCHEDULER_TaskC
     struct SendHelloContext *send_hello_context = cls;
     struct GNUNET_TESTING_PeerGroup *pg = send_hello_context->pg;
 
-    if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+    if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
       {
         GNUNET_free(send_hello_context);
         return;
@@ -3514,6 +3531,8 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
   struct ConnectTopologyContext *ct_ctx = connect_ctx->ct_ctx;
   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
   struct PeerConnection *connection;
+
+  GNUNET_assert (0 < pg->outstanding_connects);
   pg->outstanding_connects--;
 
   /*
@@ -3535,6 +3554,7 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
 
   if (connection != NULL) /* Can safely remove! */
     {
+      GNUNET_assert (0 < ct_ctx->remaining_connections);
       ct_ctx->remaining_connections--;
       if (pg->notify_connection != NULL) /* Notify of reverse connection */
         pg->notify_connection (pg->notify_connection_cls, second, first,
@@ -3548,7 +3568,10 @@ internal_connect_notify(void *cls, const struct GNUNET_PeerIdentity *first,
   if (ct_ctx->remaining_connections == 0)
     {
       if (ct_ctx->notify_connections_done != NULL)
-        ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+       {
+         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
+         ct_ctx->notify_connections_done = NULL;
+       }
     }
   else
     preschedule_connect (pg);
@@ -3574,7 +3597,7 @@ schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct ConnectContext *connect_context = cls;
   struct GNUNET_TESTING_PeerGroup *pg = connect_context->ct_ctx->pg;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if ((pg->outstanding_connects > pg->max_outstanding_connections)
@@ -3695,7 +3718,7 @@ copy_allowed_topology(struct GNUNET_TESTING_PeerGroup *pg)
                       "Creating connection between %d and %d\n", pg_iter,
                       iter->index);
           total += add_connections (pg, pg_iter, iter->index, CONNECT,
-                                    GNUNET_NO);
+                                    GNUNET_YES);
           //total += add_actual_connections(pg, pg_iter, iter->index);
           iter = iter->next;
         }
@@ -3959,6 +3982,8 @@ GNUNET_TESTING_create_topology(struct GNUNET_TESTING_PeerGroup *pg,
   if ((restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE) && (restrict_topology
       != GNUNET_TESTING_TOPOLOGY_FROM_FILE))
     create_clique (pg, &add_connections, BLACKLIST, GNUNET_NO);
+  else
+    return num_connections;
 
   unblacklisted_connections = 0;
   /* Un-blacklist connections as per the topology specified */
@@ -4264,22 +4289,17 @@ void
 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg,
                           double percentage)
 {
-  struct RandomContext random_ctx;
   uint32_t pg_iter;
 #if OLD
-  struct PeerConnection *temp_peers;
   struct PeerConnection *conn_iter;
   double random_number;
+#else
+  struct RandomContext random_ctx;
 #endif
 
   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
     {
-      random_ctx.first_uid = pg_iter;
-      random_ctx.first = &pg->peers[pg_iter];
-      random_ctx.percentage = percentage;
-      random_ctx.pg = pg;
 #if OLD
-      temp_peers = NULL;
       conn_iter = pg->peers[pg_iter].connect_peers_head;
       while (conn_iter != NULL)
         {
@@ -4295,17 +4315,21 @@ choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg,
           conn_iter = conn_iter->next;
         }
 #else
-      pg->peers[pg_iter].connect_peers_working_set =
-      GNUNET_CONTAINER_multihashmap_create (pg->total);
+      random_ctx.first_uid = pg_iter;
+      random_ctx.first = &pg->peers[pg_iter];
+      random_ctx.percentage = percentage;
+      random_ctx.pg = pg;
+      pg->peers[pg_iter].connect_peers_working_set 
+       = GNUNET_CONTAINER_multihashmap_create (pg->total);
       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
-          &random_connect_iterator,
-          &random_ctx);
+                                            &random_connect_iterator,
+                                            &random_ctx);
       /* Now remove the old connections */
       GNUNET_CONTAINER_multihashmap_destroy (pg->
-          peers[pg_iter].connect_peers);
+                                            peers[pg_iter].connect_peers);
       /* And replace with the random set */
-      pg->peers[pg_iter].connect_peers =
-      pg->peers[pg_iter].connect_peers_working_set;
+      pg->peers[pg_iter].connect_peers 
+       = pg->peers[pg_iter].connect_peers_working_set;
 #endif
     }
 
@@ -4799,7 +4823,7 @@ schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
   struct CoreContext *core_context = cls;
   struct TopologyIterateContext *topology_context =
       (struct TopologyIterateContext *) core_context->iter_context;
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (topology_context->connected
@@ -4943,7 +4967,7 @@ schedule_get_statistics(void *cls,
   struct StatsIterateContext *stats_context =
       (struct StatsIterateContext *) core_context->iter_context;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     return;
 
   if (stats_context->connected > stats_context->pg->max_outstanding_connections)
@@ -5425,7 +5449,7 @@ internal_continue_startup(void *cls,
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5538,7 +5562,7 @@ internal_start(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct InternalStartContext *internal_context = cls;
 
-  if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
+  if ( (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0)
     {
       return;
     }
@@ -5583,43 +5607,24 @@ struct PeerStartHelperContext
   struct GNUNET_TESTING_PeerGroup *pg;
 
   struct HostData *host;
+
+  struct GNUNET_OS_Process *proc;
 };
 
 static void
-start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+check_peers_started (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 {
   struct PeerStartHelperContext *helper = cls;
-  char *baseservicehome;
-  char *tempdir;
-  struct GNUNET_OS_Process *proc;
-  unsigned int i;
-  char *arg;
   enum GNUNET_OS_ProcessStatusType type;
   unsigned long code;
+  unsigned int i;
   GNUNET_TESTING_NotifyDaemonRunning cb;
-  /* ssh user@host peerStartHelper /path/to/basedirectory */
-
-  GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (helper->pg->cfg, "PATHS", "SERVICEHOME",
-                                                                    &baseservicehome));
-  GNUNET_asprintf(&tempdir, "%s/%s/", baseservicehome, helper->host->hostname);
-  if (NULL != helper->host->username)
-    GNUNET_asprintf (&arg, "%s@%s", helper->host->username, helper->host->hostname);
-  else
-    GNUNET_asprintf (&arg, "%s", helper->host->hostname);
 
-  /* FIXME: Doesn't support ssh_port option! */
-  proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
-                                  "peerStartHelper.pl", tempdir,  NULL);
-
-  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "starting peers with cmd ssh %s %s %s\n", arg, "peerStartHelper.pl", tempdir);
-
-  GNUNET_OS_process_wait (proc);
-  if (GNUNET_OK != GNUNET_OS_process_status (proc, &type, &code))
-    code = 1;
-  GNUNET_OS_process_close(proc);
-  GNUNET_free (tempdir);
-  GNUNET_free (baseservicehome);
-  GNUNET_free (arg);
+  if (GNUNET_NO == GNUNET_OS_process_status (helper->proc, &type, &code)) /* Still running, wait some more! */
+  {
+    GNUNET_SCHEDULER_add_delayed(GNUNET_CONSTANTS_EXEC_WAIT, &check_peers_started, helper);
+    return;
+  }
 
   helper->pg->starting--;
   if (helper->pg->starting == 0) /* All peers have finished starting! */
@@ -5629,16 +5634,15 @@ start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
         {
           cb = helper->pg->peers[i].daemon->cb;
           helper->pg->peers[i].daemon->cb = NULL;
-
+          helper->pg->peers[i].daemon->running = GNUNET_YES;
+          helper->pg->peers[i].daemon->phase = SP_START_DONE;
           if (NULL != cb)
           {
             if ((type != GNUNET_OS_PROCESS_EXITED) || (code != 0))
-              {
-                cb (helper->pg->peers[i].daemon->cb_cls,
-                    &helper->pg->peers[i].daemon->id,
-                    helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
-                    "Failed to execute peerStartHelper.pl, or return code bad!");
-              }
+              cb (helper->pg->peers[i].daemon->cb_cls,
+                  &helper->pg->peers[i].daemon->id,
+                  helper->pg->peers[i].daemon->cfg, helper->pg->peers[i].daemon,
+                  "Failed to execute peerStartHelper.pl, or return code bad!");
             else
               cb (helper->pg->peers[i].daemon->cb_cls,
                   &helper->pg->peers[i].daemon->id,
@@ -5649,7 +5653,34 @@ start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
         }
     }
+  GNUNET_OS_process_close(helper->proc);
+}
 
+static void
+start_peer_helper (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
+{
+  struct PeerStartHelperContext *helper = cls;
+  char *baseservicehome;
+  char *tempdir;
+  char *arg;
+  /* ssh user@host peerStartHelper /path/to/basedirectory */
+  GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (helper->pg->cfg, "PATHS", "SERVICEHOME",
+                                                                    &baseservicehome));
+  GNUNET_asprintf(&tempdir, "%s/%s/", baseservicehome, helper->host->hostname);
+  if (NULL != helper->host->username)
+    GNUNET_asprintf (&arg, "%s@%s", helper->host->username, helper->host->hostname);
+  else
+    GNUNET_asprintf (&arg, "%s", helper->host->hostname);
+
+  /* FIXME: Doesn't support ssh_port option! */
+  helper->proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
+                                  "peerStartHelper.pl", tempdir,  NULL);
+  GNUNET_assert(helper->proc != NULL);
+  GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "starting peers with cmd ssh %s %s %s\n", arg, "peerStartHelper.pl", tempdir);
+  GNUNET_SCHEDULER_add_now (&check_peers_started, helper);
+  GNUNET_free (tempdir);
+  GNUNET_free (baseservicehome);
+  GNUNET_free (arg);
 }
 #endif
 
@@ -5782,6 +5813,7 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
   uint64_t total_hostkeys;
   struct GNUNET_OS_Process *proc;
 
+  username = NULL;
   if (0 == total)
     {
       GNUNET_break (0);
@@ -5865,6 +5897,7 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
       else
         proc = GNUNET_OS_start_process (NULL, NULL, "ssh", "ssh", arg,
                                         "mkdir -p", tmpdir, NULL);
+      GNUNET_assert(proc != NULL);
       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                   "Creating remote dir with command ssh %s %s %s\n", arg,
                   " mkdir -p ", tmpdir);
@@ -5880,15 +5913,28 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
                                                            &hostkeys_file))
     {
       if (GNUNET_YES != GNUNET_DISK_file_test (hostkeys_file))
-        GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Couldn't read hostkeys file!\n");
+        GNUNET_log (GNUNET_ERROR_TYPE_WARNING, 
+                   _("Could not read hostkeys file!\n"));
       else
         {
           /* Check hostkey file size, read entire thing into memory */
-          fd = GNUNET_DISK_file_open (hostkeys_file, GNUNET_DISK_OPEN_READ,
+          fd = GNUNET_DISK_file_open (hostkeys_file,
+                                     GNUNET_DISK_OPEN_READ,
                                       GNUNET_DISK_PERM_NONE);
           if (NULL == fd)
             {
-              GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, "open", hostkeys_file);
+              GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR, 
+                                       "open", 
+                                       hostkeys_file);
+             GNUNET_free (hostkeys_file);
+             for (i=0;i<pg->num_hosts;i++)
+               {
+                 GNUNET_free (pg->hosts[i].hostname);
+                 GNUNET_free_non_null (pg->hosts[i].username);
+               }
+             GNUNET_free (pg->peers);
+             GNUNET_free (pg->hosts);
+             GNUNET_free (pg);
               return NULL;
             }
 
@@ -5896,12 +5942,11 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
                                                    GNUNET_YES))
             fs = 0;
 
-          GNUNET_log (
-                      GNUNET_ERROR_TYPE_WARNING,
+          GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
                       "Found file size %llu for hostkeys, expect hostkeys to be size %d\n",
                       fs, HOSTKEYFILESIZE);
 
-          if (fs % HOSTKEYFILESIZE != 0)
+          if (0 != (fs % HOSTKEYFILESIZE))
             {
               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
                           "File size %llu seems incorrect for hostkeys...\n",
@@ -5910,12 +5955,13 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
           else
             {
               total_hostkeys = fs / HOSTKEYFILESIZE;
-              GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
-                          "Will read %llu hostkeys from file\n", total_hostkeys);
+              GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+                          "Will read %llu hostkeys from file\n",
+                         total_hostkeys);
               pg->hostkey_data = GNUNET_malloc_large (fs);
               GNUNET_assert (fs == GNUNET_DISK_file_read (fd, pg->hostkey_data, fs));
-              GNUNET_assert(GNUNET_OK == GNUNET_DISK_file_close(fd));
             }
+         GNUNET_assert(GNUNET_OK == GNUNET_DISK_file_close(fd));
         }
       GNUNET_free(hostkeys_file);
     }
@@ -5993,7 +6039,6 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
 #else
       if ((pg->hostkey_data != NULL) && (hostcnt > 0))
         {
-          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Have hostkey data and running on remote hosts!\n");
           pg->peers[off].daemon = GNUNET_TESTING_daemon_start (pcfg,
                                        timeout,
                                        GNUNET_YES,
@@ -6029,22 +6074,17 @@ GNUNET_TESTING_daemons_start(const struct GNUNET_CONFIGURATION_Handle *cfg,
     {
       for (off = 0; off < hostcnt; off++)
         {
-          GNUNET_asprintf(&newservicehome, "%s/%s", baseservicehome, pg->hosts[off].hostname);
+          GNUNET_asprintf(&newservicehome, "%s/%s/", baseservicehome, pg->hosts[off].hostname);
 
           if (NULL != username)
-            GNUNET_asprintf (&arg, "%s@%s:%s", username, pg->hosts[off].hostname, baseservicehome);
+            GNUNET_asprintf (&arg, "%s@%s:%s/%s", username, pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
           else
-            GNUNET_asprintf (&arg, "%s:%s", pg->hosts[off].hostname, baseservicehome);
+            GNUNET_asprintf (&arg, "%s:%s/%s", pg->hosts[off].hostname, baseservicehome, pg->hosts[off].hostname);
 
           /* FIXME: Doesn't support ssh_port option! */
-          proc = GNUNET_OS_start_process (NULL, NULL, "scp", "scp", "-r",
-#if !DEBUG_TESTING
-                                               "-q",
-#endif
-                                               newservicehome, arg, NULL);
-
-          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "copying directory with command scp -r %s %s\n", newservicehome, arg);
+          proc = GNUNET_OS_start_process (NULL, NULL, "rsync", "rsync", "-r", newservicehome, arg, NULL);
 
+          GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "copying directory with command rsync -r %s %s\n", newservicehome, arg);
 
           GNUNET_free (arg);
           if (NULL == proc)
@@ -6077,8 +6117,7 @@ GNUNET_TESTING_daemon_get(struct GNUNET_TESTING_PeerGroup *pg,
 {
   if (position < pg->total)
     return pg->peers[position].daemon;
-  else
-    return NULL;
+  return NULL;
 }
 
 /*
@@ -6102,7 +6141,6 @@ GNUNET_TESTING_daemon_get_by_id(struct GNUNET_TESTING_PeerGroup *pg,
                        sizeof(struct GNUNET_PeerIdentity)))
         return pg->peers[i].daemon;
     }
-
   return NULL;
 }
 
@@ -6116,7 +6154,7 @@ GNUNET_TESTING_daemon_get_by_id(struct GNUNET_TESTING_PeerGroup *pg,
  * @param d handle to the daemon that was restarted
  * @param emsg NULL on success
  */
-void
+static void
 restart_callback(void *cls, const struct GNUNET_PeerIdentity *id,
                  const struct GNUNET_CONFIGURATION_Handle *cfg,
                  struct GNUNET_TESTING_Daemon *d, const char *emsg)
@@ -6155,7 +6193,7 @@ restart_callback(void *cls, const struct GNUNET_PeerIdentity *id,
  * @param emsg NULL on success, non-NULL on failure
  *
  */
-void
+static void
 churn_stop_callback(void *cls, const char *emsg)
 {
   struct ShutdownContext *shutdown_ctx = cls;
@@ -6259,6 +6297,7 @@ schedule_churn_shutdown_task(void *cls,
     }
 }
 
+
 /**
  * Simulate churn by stopping some peers (and possibly
  * re-starting others if churn is called multiple times).  This
@@ -6301,6 +6340,11 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   unsigned int *running_permute;
   unsigned int *stopped_permute;
 
+  shutdown_ctx = NULL;
+  peer_shutdown_ctx = NULL;
+  peer_restart_ctx = NULL;
+  churn_startup_ctx = NULL;
+
   running = 0;
   stopped = 0;
 
@@ -6449,6 +6493,7 @@ GNUNET_TESTING_daemons_churn(struct GNUNET_TESTING_PeerGroup *pg,
   GNUNET_free_non_null (stopped_permute);
 }
 
+
 /**
  * Restart all peers in the given group.
  *
@@ -6480,6 +6525,7 @@ GNUNET_TESTING_daemons_restart(struct GNUNET_TESTING_PeerGroup *pg,
     }
 }
 
+
 /**
  * Start or stop an individual peer from the given group.
  *
@@ -6536,13 +6582,14 @@ GNUNET_TESTING_daemons_vary(struct GNUNET_TESTING_PeerGroup *pg,
     GNUNET_break (0);
 }
 
+
 /**
  * Callback for shutting down peers in a peer group.
  *
  * @param cls closure (struct ShutdownContext)
  * @param emsg NULL on success
  */
-void
+static void
 internal_shutdown_callback(void *cls, const char *emsg)
 {
   struct PeerShutdownContext *peer_shutdown_ctx = cls;
@@ -6592,6 +6639,7 @@ internal_shutdown_callback(void *cls, const char *emsg)
   GNUNET_free(peer_shutdown_ctx);
 }
 
+
 /**
  * Task to rate limit the number of outstanding peer shutdown
  * requests.  This is necessary for making sure we don't do
@@ -6631,6 +6679,7 @@ schedule_shutdown_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
 
 }
 
+
 /**
  * Shutdown all peers started in the given group.
  *
@@ -6709,12 +6758,12 @@ GNUNET_TESTING_daemons_stop(struct GNUNET_TESTING_PeerGroup *pg,
         }
 #else
       if (pg->peers[off].allowed_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
       if (pg->peers[off].connect_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
       if (pg->peers[off].blacklisted_peers != NULL)
-      GNUNET_CONTAINER_multihashmap_destroy (pg->
-          peers[off].blacklisted_peers);
+       GNUNET_CONTAINER_multihashmap_destroy (pg->
+                                              peers[off].blacklisted_peers);
 #endif
     }
 }