src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define VERBOSE_TOPOLOGY GNUNET_YES
  36
  37 #define DEBUG_CHURN GNUNET_NO
  38
  39 /**
  40  * Lowest port used for GNUnet testing.  Should be high enough to not
  41  * conflict with other applications running on the hosts but be low
  42  * enough to not conflict with client-ports (typically starting around
  43  * 32k).
  44  */
  45 #define LOW_PORT 10000
  46
  47 /**
  48  * Highest port used for GNUnet testing.  Should be low enough to not
  49  * conflict with the port range for "local" ports (client apps; see
  50  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  51  */
  52 #define HIGH_PORT 56000
  53
  54 #define MAX_OUTSTANDING_CONNECTIONS 40
  55
  56 #define MAX_CONCURRENT_HOSTKEYS 10
  57
  58 #define MAX_CONCURRENT_STARTING 10
  59
  60 #define MAX_CONCURRENT_SHUTDOWN 10
  61
  62 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 200)
  63
  64 #define CONNECT_ATTEMPTS 21
  65
  66 /**
  67  * Prototype of a function called whenever two peers would be connected
  68  * in a certain topology.
  69  */
  70 typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
  71                                                             GNUNET_TESTING_PeerGroup
  72                                                             * pg,
  73                                                             unsigned int
  74                                                             first,
  75                                                             unsigned int
  76                                                             second);
  77
  78
  79 /**
  80  * Context for handling churning a peer group
  81  */
  82 struct ChurnContext
  83 {
  84   /**
  85    * Callback used to notify of churning finished
  86    */
  87   GNUNET_TESTING_NotifyCompletion cb;
  88
  89   /**
  90    * Closure for callback
  91    */
  92   void *cb_cls;
  93
  94   /**
  95    * Number of peers that still need to be started
  96    */
  97   unsigned int num_to_start;
  98
  99   /**
 100    * Number of peers that still need to be stopped
 101    */
 102   unsigned int num_to_stop;
 103
 104   /**
 105    * Number of peers that failed to start
 106    */
 107   unsigned int num_failed_start;
 108
 109   /**
 110    * Number of peers that failed to stop
 111    */
 112   unsigned int num_failed_stop;
 113 };
 114
 115 struct RestartContext
 116 {
 117   /**
 118    * The group of peers being restarted
 119    */
 120   struct GNUNET_TESTING_PeerGroup *peer_group;
 121
 122   /**
 123    * How many peers have been restarted thus far
 124    */
 125   unsigned int peers_restarted;
 126
 127   /**
 128    * How many peers got an error when restarting
 129    */
 130   unsigned int peers_restart_failed;
 131
 132   /**
 133    * The function to call once all peers have been restarted
 134    */
 135   GNUNET_TESTING_NotifyCompletion callback;
 136
 137   /**
 138    * Closure for callback function
 139    */
 140   void *callback_cls;
 141
 142 };
 143
 144
 145 struct ShutdownContext
 146 {
 147   /**
 148    * Total peers to wait for
 149    */
 150   unsigned int total_peers;
 151
 152   /**
 153    * Number of peers successfully shut down
 154    */
 155   unsigned int peers_down;
 156
 157   /**
 158    * Number of peers failed to shut down
 159    */
 160   unsigned int peers_failed;
 161
 162   /**
 163    * Number of peers we have started shutting
 164    * down.  If too many, wait on them.
 165    */
 166   unsigned int outstanding;
 167
 168   /**
 169    * Timeout for shutdown.
 170    */
 171   struct GNUNET_TIME_Relative timeout;
 172
 173   /**
 174    * Callback to call when all peers either
 175    * shutdown or failed to shutdown
 176    */
 177   GNUNET_TESTING_NotifyCompletion cb;
 178
 179   /**
 180    * Closure for cb
 181    */
 182   void *cb_cls;
 183 };
 184
 185 /**
 186  * Individual shutdown context for a particular peer.
 187  */
 188 struct PeerShutdownContext
 189 {
 190   /**
 191    * Pointer to the high level shutdown context.
 192    */
 193   struct ShutdownContext *shutdown_ctx;
 194
 195   /**
 196    * The daemon handle for the peer to shut down.
 197    */
 198   struct GNUNET_TESTING_Daemon *daemon;
 199 };
 200
 201 /**
 202  * Individual shutdown context for a particular peer.
 203  */
 204 struct PeerRestartContext
 205 {
 206   /**
 207    * Pointer to the high level restart context.
 208    */
 209   struct ChurnRestartContext *churn_restart_ctx;
 210
 211   /**
 212    * The daemon handle for the peer to shut down.
 213    */
 214   struct GNUNET_TESTING_Daemon *daemon;
 215 };
 216
 217
 218 struct CreateTopologyContext
 219 {
 220
 221   /**
 222    * Function to call with number of connections
 223    */
 224   GNUNET_TESTING_NotifyConnections cont;
 225
 226   /**
 227    * Closure for connection notification
 228    */
 229   void *cls;
 230 };
 231
 232 #if OLD
 233 struct PeerConnection
 234 {
 235   /*
 236    * Linked list
 237    */
 238   struct PeerConnection *next;
 239
 240   /*
 241    * Pointer to daemon handle
 242    */
 243   struct GNUNET_TESTING_Daemon *daemon;
 244
 245 };
 246 #endif
 247
 248 struct InternalStartContext
 249 {
 250   /**
 251    * Pointer to peerdata
 252    */
 253   struct PeerData *peer;
 254
 255   /**
 256    * Timeout for peer startup
 257    */
 258   struct GNUNET_TIME_Relative timeout;
 259
 260   /**
 261    * Client callback for hostkey notification
 262    */
 263   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 264
 265   /**
 266    * Closure for hostkey_callback
 267    */
 268   void *hostkey_cls;
 269
 270   /**
 271    * Client callback for peer start notification
 272    */
 273   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 274
 275   /**
 276    * Closure for cb
 277    */
 278   void *start_cb_cls;
 279
 280   /**
 281    * Hostname, where to start the peer
 282    */
 283   const char *hostname;
 284
 285   /**
 286    * Username to use when connecting to the
 287    * host via ssh.
 288    */
 289   const char *username;
 290
 291   /**
 292    * Port to use for ssh.
 293    */
 294   uint16_t sshport;
 295
 296 };
 297
 298 struct ChurnRestartContext
 299 {
 300   /**
 301    * Number of restarts currently in flight.
 302    */
 303   unsigned int outstanding;
 304
 305   /**
 306    * Handle to the underlying churn context.
 307    */
 308   struct ChurnContext *churn_ctx;
 309
 310   /**
 311    * How long to allow the operation to take.
 312    */
 313   struct GNUNET_TIME_Relative timeout;
 314 };
 315
 316 /**
 317  * Data we keep per peer.
 318  */
 319 struct PeerData
 320 {
 321   /**
 322    * (Initial) configuration of the host.
 323    * (initial because clients could change
 324    *  it and we would not know about those
 325    *  updates).
 326    */
 327   struct GNUNET_CONFIGURATION_Handle *cfg;
 328
 329   /**
 330    * Handle for controlling the daemon.
 331    */
 332   struct GNUNET_TESTING_Daemon *daemon;
 333
 334   /**
 335    * The peergroup this peer belongs to.
 336    */
 337   struct GNUNET_TESTING_PeerGroup *pg;
 338
 339   /**
 340    * Hash map of allowed peer connections (F2F created topology)
 341    */
 342   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 343
 344   /**
 345    * Hash map of blacklisted peers
 346    */
 347   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 348
 349   /**
 350    * Hash map of peer connections
 351    */
 352   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 353
 354   /**
 355    * Temporary hash map of peer connections
 356    */
 357   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 358
 359   /**
 360    * Temporary variable for topology creation, should be reset before
 361    * creating any topology so the count is valid once finished.
 362    */
 363   int num_connections;
 364
 365   /**
 366    * Context to keep track of peers being started, to
 367    * stagger hostkey generation and peer startup.
 368    */
 369   struct InternalStartContext internal_context;
 370 };
 371
 372
 373 /**
 374  * Linked list of per-host data.
 375  */
 376 struct HostData
 377 {
 378   /**
 379    * Name of the host.
 380    */
 381   char *hostname;
 382
 383   /**
 384    * SSH username to use when connecting to this host.
 385    */
 386   char *username;
 387
 388   /**
 389    * SSH port to use when connecting to this host.
 390    */
 391   uint16_t sshport;
 392
 393   /**
 394    * Lowest port that we have not yet used
 395    * for GNUnet.
 396    */
 397   uint16_t minport;
 398 };
 399
 400 struct TopologyIterateContext
 401 {
 402   /**
 403    * Callback for notifying of two connected peers.
 404    */
 405   GNUNET_TESTING_NotifyTopology topology_cb;
 406
 407   /**
 408    * Closure for topology_cb
 409    */
 410   void *cls;
 411
 412   /**
 413    * Number of peers currently connected to.
 414    */
 415   unsigned int connected;
 416
 417   /**
 418    * Number of peers we have finished iterating.
 419    */
 420   unsigned int completed;
 421
 422   /**
 423    * Number of peers total.
 424    */
 425   unsigned int total;
 426 };
 427
 428 struct StatsIterateContext
 429 {
 430   /**
 431    * Continuation to call once all stats information has been retrieved.
 432    */
 433   GNUNET_STATISTICS_Callback cont;
 434
 435   /**
 436    * Proc function to call on each value received.
 437    */
 438   GNUNET_TESTING_STATISTICS_Iterator proc;
 439
 440   /**
 441    * Closure for topology_cb
 442    */
 443   void *cls;
 444
 445   /**
 446    * Number of peers currently connected to.
 447    */
 448   unsigned int connected;
 449
 450   /**
 451    * Number of peers we have finished iterating.
 452    */
 453   unsigned int completed;
 454
 455   /**
 456    * Number of peers total.
 457    */
 458   unsigned int total;
 459 };
 460
 461 struct CoreContext
 462 {
 463   void *iter_context;
 464   struct GNUNET_TESTING_Daemon *daemon;
 465 };
 466
 467 struct StatsCoreContext
 468 {
 469   void *iter_context;
 470   struct GNUNET_TESTING_Daemon *daemon;
 471   /**
 472    * Handle to the statistics service.
 473    */
 474   struct GNUNET_STATISTICS_Handle *stats_handle;
 475
 476   /**
 477    * Handle for getting statistics.
 478    */
 479   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 480 };
 481
 482 /**
 483  * Handle to a group of GNUnet peers.
 484  */
 485 struct GNUNET_TESTING_PeerGroup
 486 {
 487   /**
 488    * Configuration template.
 489    */
 490   const struct GNUNET_CONFIGURATION_Handle *cfg;
 491
 492   /**
 493    * Function to call on each started daemon.
 494    */
 495   //GNUNET_TESTING_NotifyDaemonRunning cb;
 496
 497   /**
 498    * Closure for cb.
 499    */
 500   //void *cb_cls;
 501
 502   /*
 503    * Function to call on each topology connection created
 504    */
 505   GNUNET_TESTING_NotifyConnection notify_connection;
 506
 507   /*
 508    * Callback for notify_connection
 509    */
 510   void *notify_connection_cls;
 511
 512   /**
 513    * Array of information about hosts.
 514    */
 515   struct HostData *hosts;
 516
 517   /**
 518    * Number of hosts (size of HostData)
 519    */
 520   unsigned int num_hosts;
 521
 522   /**
 523    * Array of "total" peers.
 524    */
 525   struct PeerData *peers;
 526
 527   /**
 528    * Number of peers in this group.
 529    */
 530   unsigned int total;
 531
 532   /**
 533    * At what time should we fail the peer startup process?
 534    */
 535   struct GNUNET_TIME_Absolute max_timeout;
 536
 537   /**
 538    * How many peers are being started right now?
 539    */
 540   unsigned int starting;
 541
 542   /**
 543    * How many peers have already been started?
 544    */
 545   unsigned int started;
 546 };
 547
 548 struct UpdateContext
 549 {
 550   struct GNUNET_CONFIGURATION_Handle *ret;
 551   const struct GNUNET_CONFIGURATION_Handle *orig;
 552   const char *hostname;
 553   unsigned int nport;
 554   unsigned int upnum;
 555   unsigned int fdnum;
 556 };
 557
 558 struct ConnectTopologyContext
 559 {
 560   /**
 561    * How many connections are left to create.
 562    */
 563   unsigned int remaining_connections;
 564
 565   /**
 566    * Handle to group of peers.
 567    */
 568   struct GNUNET_TESTING_PeerGroup *pg;
 569
 570   /**
 571    * Temp value set for each iteration.
 572    */
 573   struct PeerData *first;
 574
 575   /**
 576    * Notification that all peers are connected.
 577    */
 578   GNUNET_TESTING_NotifyCompletion notify_connections_done;
 579
 580   /**
 581    * Closure for notify.
 582    */
 583   void *notify_cls;
 584 };
 585
 586 struct ConnectContext
 587 {
 588   /**
 589    * Peer to connect second to.
 590    */
 591   struct GNUNET_TESTING_Daemon *first;
 592
 593   /**
 594    * Peer to connect first to.
 595    */
 596   struct GNUNET_TESTING_Daemon *second;
 597
 598   /**
 599    * Higher level topology connection context.
 600    */
 601   struct ConnectTopologyContext *ct_ctx;
 602 };
 603
 604 /**
 605  * Convert unique ID to hash code.
 606  *
 607  * @param uid unique ID to convert
 608  * @param hash set to uid (extended with zeros)
 609  */
 610 static void
 611 hash_from_uid (uint32_t uid, GNUNET_HashCode * hash)
 612 {
 613   memset (hash, 0, sizeof (GNUNET_HashCode));
 614   *((uint32_t *) hash) = uid;
 615 }
 616
 617 /**
 618  * Convert hash code to unique ID.
 619  *
 620  * @param uid unique ID to convert
 621  * @param hash set to uid (extended with zeros)
 622  */
 623 static void
 624 uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 625 {
 626   memcpy (uid, hash, sizeof (uint32_t));
 627 }
 628
 629 /**
 630  * Number of connects we are waiting on, allows us to rate limit
 631  * connect attempts.
 632  */
 633 static int outstanding_connects;
 634
 635 /**
 636  * Get a topology from a string input.
 637  *
 638  * @param topology where to write the retrieved topology
 639  * @param topology_string The string to attempt to
 640  *        get a configuration value from
 641  * @return GNUNET_YES if topology string matched a
 642  *         known topology, GNUNET_NO if not
 643  */
 644 int
 645 GNUNET_TESTING_topology_get (enum GNUNET_TESTING_Topology *topology,
 646                              const char *topology_string)
 647 {
 648   /**
 649    * Strings representing topologies in enum
 650    */
 651   static const char *topology_strings[] = {
 652       /**
 653        * A clique (everyone connected to everyone else).
 654        */
 655     "CLIQUE",
 656
 657       /**
 658        * Small-world network (2d torus plus random links).
 659        */
 660     "SMALL_WORLD",
 661
 662       /**
 663        * Small-world network (ring plus random links).
 664        */
 665     "SMALL_WORLD_RING",
 666
 667       /**
 668        * Ring topology.
 669        */
 670     "RING",
 671
 672       /**
 673        * 2-d torus.
 674        */
 675     "2D_TORUS",
 676
 677       /**
 678        * Random graph.
 679        */
 680     "ERDOS_RENYI",
 681
 682       /**
 683        * Certain percentage of peers are unable to communicate directly
 684        * replicating NAT conditions
 685        */
 686     "INTERNAT",
 687
 688       /**
 689        * Scale free topology.
 690        */
 691     "SCALE_FREE",
 692
 693       /**
 694        * Straight line topology.
 695        */
 696     "LINE",
 697
 698       /**
 699        * All peers are disconnected.
 700        */
 701     "NONE",
 702
 703     NULL
 704   };
 705
 706   int curr = 0;
 707   if (topology_string == NULL)
 708     return GNUNET_NO;
 709   while (topology_strings[curr] != NULL)
 710     {
 711       if (strcasecmp (topology_strings[curr], topology_string) == 0)
 712         {
 713           *topology = curr;
 714           return GNUNET_YES;
 715         }
 716       curr++;
 717     }
 718   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 719   return GNUNET_NO;
 720 }
 721
 722
 723 /**
 724  * Get connect topology option from string input.
 725  *
 726  * @param topology_option where to write the retrieved topology
 727  * @param topology_string The string to attempt to
 728  *        get a configuration value from
 729  * @return GNUNET_YES if string matched a known
 730  *         topology option, GNUNET_NO if not
 731  */
 732 int
 733 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption
 734                                     *topology_option,
 735                                     const char *topology_string)
 736 {
 737   /**
 738    * Options for connecting a topology as strings.
 739    */
 740   static const char *topology_option_strings[] = {
 741       /**
 742        * Try to connect all peers specified in the topology.
 743        */
 744     "CONNECT_ALL",
 745
 746       /**
 747        * Choose a random subset of connections to create.
 748        */
 749     "CONNECT_RANDOM_SUBSET",
 750
 751       /**
 752        * Create at least X connections for each peer.
 753        */
 754     "CONNECT_MINIMUM",
 755
 756       /**
 757        * Using a depth first search, create one connection
 758        * per peer.  If any are missed (graph disconnected)
 759        * start over at those peers until all have at least one
 760        * connection.
 761        */
 762     "CONNECT_DFS",
 763
 764       /**
 765        * Find the N closest peers to each allowed peer in the
 766        * topology and make sure a connection to those peers
 767        * exists in the connect topology.
 768        */
 769     "CONNECT_CLOSEST",
 770
 771       /**
 772        * No options specified.
 773        */
 774     "CONNECT_NONE",
 775
 776     NULL
 777   };
 778   int curr = 0;
 779
 780   if (topology_string == NULL)
 781     return GNUNET_NO;
 782   while (NULL != topology_option_strings[curr])
 783     {
 784       if (strcasecmp (topology_option_strings[curr], topology_string) == 0)
 785         {
 786           *topology_option = curr;
 787           return GNUNET_YES;
 788         }
 789       curr++;
 790     }
 791   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 792   return GNUNET_NO;
 793 }
 794
 795 /**
 796  * Function to iterate over options.  Copies
 797  * the options to the target configuration,
 798  * updating PORT values as needed.
 799  *
 800  * @param cls closure
 801  * @param section name of the section
 802  * @param option name of the option
 803  * @param value value of the option
 804  */
 805 static void
 806 update_config (void *cls,
 807                const char *section, const char *option, const char *value)
 808 {
 809   struct UpdateContext *ctx = cls;
 810   unsigned int ival;
 811   char cval[12];
 812   char uval[128];
 813   char *single_variable;
 814   char *per_host_variable;
 815   unsigned long long num_per_host;
 816
 817   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 818     {
 819       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 820       if ((ival != 0)
 821           && (GNUNET_YES !=
 822               GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 823                                                     single_variable)))
 824         {
 825           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 826           value = cval;
 827         }
 828
 829       GNUNET_free (single_variable);
 830     }
 831
 832   if (0 == strcmp (option, "UNIXPATH"))
 833     {
 834       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 835       GNUNET_asprintf (&per_host_variable, "num_%s_per_host", section);
 836       if (GNUNET_YES !=
 837           GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 838                                                 single_variable))
 839         {
 840           GNUNET_snprintf (uval,
 841                            sizeof (uval),
 842                            "/tmp/test-service-%s-%u", section, ctx->upnum++);
 843           value = uval;
 844         }
 845       else if ((GNUNET_YES ==
 846                GNUNET_CONFIGURATION_get_value_number (ctx->orig, "testing",
 847                                                       per_host_variable,
 848                                                       &num_per_host)) && (num_per_host > 0))
 849
 850         {
 851           GNUNET_snprintf (uval,
 852                            sizeof (uval),
 853                            "/tmp/test-service-%s-%u",
 854                            section, ctx->fdnum % num_per_host);
 855           value = uval;
 856         }
 857       GNUNET_free (single_variable);
 858       GNUNET_free (per_host_variable);
 859
 860     }
 861
 862   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 863     {
 864       value = ctx->hostname;
 865     }
 866
 867   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 868 }
 869
 870
 871 /**
 872  * Create a new configuration using the given configuration
 873  * as a template; however, each PORT in the existing cfg
 874  * must be renumbered by incrementing "*port".  If we run
 875  * out of "*port" numbers, return NULL.
 876  *
 877  * @param cfg template configuration
 878  * @param port port numbers to use, update to reflect
 879  *             port numbers that were used
 880  * @param upnum number to make unix domain socket names unique
 881  * @param hostname hostname of the controlling host, to allow control connections from
 882  * @param fdnum number used to offset the unix domain socket for grouped processes
 883  *              (such as statistics or peerinfo, which can be shared among others)
 884  *
 885  * @return new configuration, NULL on error
 886  */
 887 static struct GNUNET_CONFIGURATION_Handle *
 888 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 889              uint16_t * port,
 890              uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 891 {
 892   struct UpdateContext uc;
 893   uint16_t orig;
 894   char *control_host;
 895   char *allowed_hosts;
 896
 897   orig = *port;
 898   uc.nport = *port;
 899   uc.upnum = *upnum;
 900   uc.fdnum = *fdnum;
 901   uc.ret = GNUNET_CONFIGURATION_create ();
 902   uc.hostname = hostname;
 903   uc.orig = cfg;
 904
 905   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 906   if (uc.nport >= HIGH_PORT)
 907     {
 908       *port = orig;
 909       GNUNET_CONFIGURATION_destroy (uc.ret);
 910       return NULL;
 911     }
 912
 913   if (GNUNET_CONFIGURATION_get_value_string
 914       (cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 915     {
 916       if (hostname != NULL)
 917         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1; %s;", control_host,
 918                          hostname);
 919       else
 920         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", control_host);
 921
 922       GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
 923                                              allowed_hosts);
 924       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
 925                                              "ACCEPT_FROM", allowed_hosts);
 926       GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
 927                                              allowed_hosts);
 928       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
 929                                              "ACCEPT_FROM", allowed_hosts);
 930       GNUNET_free_non_null (control_host);
 931       GNUNET_free (allowed_hosts);
 932     }
 933
 934
 935   /* arm needs to know to allow connections from the host on which it is running,
 936    * otherwise gnunet-arm is unable to connect to it in some instances */
 937   if (hostname != NULL)
 938     {
 939       GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", hostname);
 940       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 941                                              "BINDTO", hostname);
 942       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 943                                              "BINDTO", hostname);
 944       GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
 945                                              allowed_hosts);
 946       GNUNET_free (allowed_hosts);
 947     }
 948   else
 949     {
 950       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 951                                              "BINDTO", "127.0.0.1");
 952       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 953                                              "BINDTO", "127.0.0.1");
 954     }
 955
 956   *port = (uint16_t) uc.nport;
 957   *upnum = uc.upnum;
 958   uc.fdnum++;
 959   *fdnum = uc.fdnum;
 960   return uc.ret;
 961 }
 962
 963
 964 /*
 965  * Add entries to the peers connect list
 966  *
 967  * @param pg the peer group we are working with
 968  * @param first index of the first peer
 969  * @param second index of the second peer
 970  *
 971  * @return the number of connections added
 972  *         technically should only be 0 or 2
 973  *
 974  */
 975 static unsigned int
 976 add_actual_connections (struct GNUNET_TESTING_PeerGroup *pg,
 977                         unsigned int first, unsigned int second)
 978 {
 979   int added;
 980   int add_first;
 981   int add_second;
 982
 983   GNUNET_HashCode hash_first;
 984   GNUNET_HashCode hash_second;
 985
 986   hash_from_uid (first, &hash_first);
 987   hash_from_uid (second, &hash_second);
 988
 989   add_first = GNUNET_NO;
 990   if (GNUNET_NO ==
 991       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].connect_peers,
 992                                               &hash_second))
 993     {
 994       add_first = GNUNET_YES;
 995     }
 996
 997   add_second = GNUNET_NO;
 998   if (GNUNET_NO ==
 999       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].connect_peers,
1000                                               &hash_first))
1001     {
1002       add_second = GNUNET_YES;
1003     }
1004
1005   added = 0;
1006   if (add_first)
1007     {
1008       GNUNET_assert (GNUNET_OK ==
1009                      GNUNET_CONTAINER_multihashmap_put (pg->
1010                                                         peers
1011                                                         [first].connect_peers,
1012                                                         &hash_second,
1013                                                         pg->
1014                                                         peers[second].daemon,
1015                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1016       pg->peers[first].num_connections++;
1017       added++;
1018     }
1019
1020   if (add_second)
1021     {
1022       GNUNET_assert (GNUNET_OK ==
1023                      GNUNET_CONTAINER_multihashmap_put (pg->
1024                                                         peers
1025                                                         [second].connect_peers,
1026                                                         &hash_first,
1027                                                         pg->
1028                                                         peers[first].daemon,
1029                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1030       pg->peers[second].num_connections++;
1031       added++;
1032     }
1033
1034   return added;
1035 }
1036
1037
1038 /*
1039  * Add entries to the peers allowed connections list
1040  *
1041  * @param pg the peer group we are working with
1042  * @param first index of the first peer
1043  * @param second index of the second peer
1044  *
1045  * @return the number of connections added (can be 0, 1 or 2)
1046  *         technically should only be 0 or 2, but the small price
1047  *         of iterating over the lists (hashmaps in the future)
1048  *         for being sure doesn't bother me!
1049  *
1050  */
1051 static unsigned int
1052 add_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg,
1053                          unsigned int first, unsigned int second)
1054 {
1055   int added;
1056 #if OLD
1057   struct PeerConnection *first_iter;
1058   struct PeerConnection *second_iter;
1059   struct PeerConnection *new_first;
1060   struct PeerConnection *new_second;
1061 #endif
1062   int add_first;
1063   int add_second;
1064
1065   GNUNET_HashCode hash_first;
1066   GNUNET_HashCode hash_second;
1067
1068   hash_from_uid (first, &hash_first);
1069   hash_from_uid (second, &hash_second);
1070
1071   add_first = GNUNET_NO;
1072   if (GNUNET_NO ==
1073       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].allowed_peers,
1074                                               &hash_second))
1075     {
1076       add_first = GNUNET_YES;
1077     }
1078
1079   add_second = GNUNET_NO;
1080   if (GNUNET_NO ==
1081       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].allowed_peers,
1082                                               &hash_first))
1083     {
1084       add_second = GNUNET_YES;
1085     }
1086 #if OLD
1087   first_iter = pg->peers[first].connected_peers;
1088   while (first_iter != NULL)
1089     {
1090       if (first_iter->daemon == pg->peers[second].daemon)
1091         add_first = GNUNET_NO;
1092       first_iter = first_iter->next;
1093     }
1094
1095   second_iter = pg->peers[second].connected_peers;
1096   add_second = GNUNET_YES;
1097   while (second_iter != NULL)
1098     {
1099       if (second_iter->daemon == pg->peers[first].daemon)
1100         add_second = GNUNET_NO;
1101       second_iter = second_iter->next;
1102     }
1103 #endif
1104
1105   added = 0;
1106   if (add_first)
1107     {
1108       GNUNET_assert (GNUNET_OK ==
1109                      GNUNET_CONTAINER_multihashmap_put (pg->
1110                                                         peers
1111                                                         [first].allowed_peers,
1112                                                         &hash_second,
1113                                                         pg->
1114                                                         peers[second].daemon,
1115                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1116 #if OLD
1117       new_first = GNUNET_malloc (sizeof (struct PeerConnection));
1118       new_first->daemon = pg->peers[second].daemon;
1119       new_first->next = pg->peers[first].connected_peers;
1120       pg->peers[first].connected_peers = new_first;
1121 #endif
1122       pg->peers[first].num_connections++;
1123       added++;
1124     }
1125
1126   if (add_second)
1127     {
1128       GNUNET_assert (GNUNET_OK ==
1129                      GNUNET_CONTAINER_multihashmap_put (pg->
1130                                                         peers
1131                                                         [second].allowed_peers,
1132                                                         &hash_first,
1133                                                         pg->
1134                                                         peers[first].daemon,
1135                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1136 #if OLD
1137       new_second = GNUNET_malloc (sizeof (struct PeerConnection));
1138       new_second->daemon = pg->peers[first].daemon;
1139       new_second->next = pg->peers[second].connected_peers;
1140       pg->peers[second].connected_peers = new_second;
1141       pg->peers[first].num_connections++;
1142 #endif
1143       pg->peers[second].num_connections++;
1144       added++;
1145     }
1146
1147   return added;
1148 }
1149
1150 /*
1151  * Add entries to the peers blacklisted list
1152  *
1153  * @param pg the peer group we are working with
1154  * @param first index of the first peer
1155  * @param second index of the second peer
1156  *
1157  * @return the number of connections added (can be 0, 1 or 2)
1158  *
1159  */
1160 static unsigned int
1161 blacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1162                        unsigned int first, unsigned int second)
1163 {
1164   int added;
1165   int add_first;
1166   int add_second;
1167   GNUNET_HashCode hash_first;
1168   GNUNET_HashCode hash_second;
1169
1170   hash_from_uid (first, &hash_first);
1171   hash_from_uid (second, &hash_second);
1172
1173   add_first = GNUNET_NO;
1174   if (GNUNET_NO ==
1175       GNUNET_CONTAINER_multihashmap_contains (pg->
1176                                               peers[first].blacklisted_peers,
1177                                               &hash_second))
1178     {
1179       add_first = GNUNET_YES;
1180     }
1181
1182   add_second = GNUNET_NO;
1183   if (GNUNET_NO ==
1184       GNUNET_CONTAINER_multihashmap_contains (pg->
1185                                               peers[second].blacklisted_peers,
1186                                               &hash_first))
1187     {
1188       add_second = GNUNET_YES;
1189     }
1190
1191   added = 0;
1192   if (add_first)
1193     {
1194       GNUNET_assert (GNUNET_OK ==
1195                      GNUNET_CONTAINER_multihashmap_put (pg->
1196                                                         peers
1197                                                         [first].blacklisted_peers,
1198                                                         &hash_second,
1199                                                         pg->
1200                                                         peers[second].daemon,
1201                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1202       pg->peers[first].num_connections++;
1203       added++;
1204     }
1205
1206   if (add_second)
1207     {
1208       GNUNET_assert (GNUNET_OK ==
1209                      GNUNET_CONTAINER_multihashmap_put (pg->
1210                                                         peers
1211                                                         [second].blacklisted_peers,
1212                                                         &hash_first,
1213                                                         pg->
1214                                                         peers[first].daemon,
1215                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1216       pg->peers[second].num_connections++;
1217       added++;
1218     }
1219
1220   return added;
1221 }
1222
1223 /*
1224  * Remove entries from the peers blacklisted list
1225  *
1226  * @param pg the peer group we are working with
1227  * @param first index of the first peer
1228  * @param second index of the second peer
1229  *
1230  * @return the number of connections removed (can be 0, 1 or 2)
1231  *
1232  */
1233 static unsigned int
1234 unblacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1235                          unsigned int first, unsigned int second)
1236 {
1237   int removed;
1238   int remove_first;
1239   int remove_second;
1240   GNUNET_HashCode hash_first;
1241   GNUNET_HashCode hash_second;
1242
1243   hash_from_uid (first, &hash_first);
1244   hash_from_uid (second, &hash_second);
1245
1246   remove_first =
1247     GNUNET_CONTAINER_multihashmap_contains (pg->
1248                                             peers[first].blacklisted_peers,
1249                                             &hash_second);
1250   remove_second =
1251     GNUNET_CONTAINER_multihashmap_contains (pg->
1252                                             peers[second].blacklisted_peers,
1253                                             &hash_first);
1254
1255   removed = 0;
1256   if (remove_first)
1257     {
1258       GNUNET_assert (GNUNET_YES ==
1259                      GNUNET_CONTAINER_multihashmap_remove (pg->
1260                                                            peers
1261                                                            [first].blacklisted_peers,
1262                                                            &hash_second,
1263                                                            pg->
1264                                                            peers
1265                                                            [second].daemon));
1266       removed++;
1267     }
1268
1269   if (remove_second)
1270     {
1271       GNUNET_assert (GNUNET_YES ==
1272                      GNUNET_CONTAINER_multihashmap_remove (pg->
1273                                                            peers
1274                                                            [second].blacklisted_peers,
1275                                                            &hash_first,
1276                                                            pg->
1277                                                            peers
1278                                                            [first].daemon));
1279       removed++;
1280     }
1281
1282   return removed;
1283 }
1284
1285 /**
1286  * Scale free network construction as described in:
1287  *
1288  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1289  *
1290  * Start with a network of "one" peer, then progressively add
1291  * peers up to the total number.  At each step, iterate over
1292  * all possible peers and connect new peer based on number of
1293  * existing connections of the target peer.
1294  *
1295  * @param pg the peer group we are dealing with
1296  * @param proc the connection processor to use
1297  *
1298  * @return the number of connections created
1299  */
1300 static unsigned int
1301 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg,
1302                    GNUNET_TESTING_ConnectionProcessor proc)
1303 {
1304
1305   unsigned int total_connections;
1306   unsigned int outer_count;
1307   unsigned int i;
1308   unsigned int previous_total_connections;
1309   double random;
1310   double probability;
1311
1312   GNUNET_assert (pg->total > 1);
1313
1314   /* Add a connection between the first two nodes */
1315   total_connections = proc (pg, 0, 1);
1316
1317   for (outer_count = 1; outer_count < pg->total; outer_count++)
1318     {
1319       previous_total_connections = total_connections;
1320       for (i = 0; i < outer_count; i++)
1321         {
1322           probability =
1323             pg->peers[i].num_connections /
1324             (double) previous_total_connections;
1325           random =
1326             ((double)
1327              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1328                                        UINT64_MAX)) / ((double) UINT64_MAX);
1329 #if VERBOSE_TESTING
1330           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1331                       "Considering connecting peer %d to peer %d\n",
1332                       outer_count, i);
1333 #endif
1334           if (random < probability)
1335             {
1336 #if VERBOSE_TESTING
1337               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1338                           "Connecting peer %d to peer %d\n", outer_count, i);
1339 #endif
1340               total_connections += proc (pg, outer_count, i);
1341             }
1342         }
1343     }
1344
1345   return total_connections;
1346 }
1347
1348 /**
1349  * Create a topology given a peer group (set of running peers)
1350  * and a connection processor.
1351  *
1352  * @param pg the peergroup to create the topology on
1353  * @param proc the connection processor to call to actually set
1354  *        up connections between two peers
1355  *
1356  * @return the number of connections that were set up
1357  *
1358  */
1359 static unsigned int
1360 create_small_world_ring (struct GNUNET_TESTING_PeerGroup *pg,
1361                          GNUNET_TESTING_ConnectionProcessor proc)
1362 {
1363   unsigned int i, j;
1364   int nodeToConnect;
1365   unsigned int natLog;
1366   unsigned int randomPeer;
1367   double random, logNModifier, percentage;
1368   unsigned int smallWorldConnections;
1369   int connsPerPeer;
1370   char *p_string;
1371   int max;
1372   int min;
1373   unsigned int useAnd;
1374   int connect_attempts;
1375
1376   logNModifier = 0.5;           /* FIXME: default value? */
1377   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1378                                                           "TESTING",
1379                                                           "LOGNMODIFIER",
1380                                                           &p_string))
1381     {
1382       if (sscanf (p_string, "%lf", &logNModifier) != 1)
1383         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1384                     _
1385                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1386                     p_string, "LOGNMODIFIER", "TESTING");
1387       GNUNET_free (p_string);
1388     }
1389   percentage = 0.5;             /* FIXME: default percentage? */
1390   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1391                                                           "TESTING",
1392                                                           "PERCENTAGE",
1393                                                           &p_string))
1394     {
1395       if (sscanf (p_string, "%lf", &percentage) != 1)
1396         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1397                     _
1398                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1399                     p_string, "PERCENTAGE", "TESTING");
1400       GNUNET_free (p_string);
1401     }
1402   natLog = log (pg->total);
1403   connsPerPeer = ceil (natLog * logNModifier);
1404
1405   if (connsPerPeer % 2 == 1)
1406     connsPerPeer += 1;
1407
1408   smallWorldConnections = 0;
1409   connect_attempts = 0;
1410   for (i = 0; i < pg->total; i++)
1411     {
1412       useAnd = 0;
1413       max = i + connsPerPeer / 2;
1414       min = i - connsPerPeer / 2;
1415
1416       if (max > pg->total - 1)
1417         {
1418           max = max - pg->total;
1419           useAnd = 1;
1420         }
1421
1422       if (min < 0)
1423         {
1424           min = pg->total - 1 + min;
1425           useAnd = 1;
1426         }
1427
1428       for (j = 0; j < connsPerPeer / 2; j++)
1429         {
1430           random =
1431             ((double)
1432              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1433                                        UINT64_MAX) / ((double) UINT64_MAX));
1434           if (random < percentage)
1435             {
1436               /* Connect to uniformly selected random peer */
1437               randomPeer =
1438                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1439                                           pg->total);
1440               while ((((randomPeer < max) && (randomPeer > min))
1441                       && (useAnd == 0)) || (((randomPeer > min)
1442                                              || (randomPeer < max))
1443                                             && (useAnd == 1)))
1444                 {
1445                   randomPeer =
1446                     GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1447                                               pg->total);
1448                 }
1449               smallWorldConnections += proc (pg, i, randomPeer);
1450             }
1451           else
1452             {
1453               nodeToConnect = i + j + 1;
1454               if (nodeToConnect > pg->total - 1)
1455                 {
1456                   nodeToConnect = nodeToConnect - pg->total;
1457                 }
1458               connect_attempts += proc (pg, i, nodeToConnect);
1459             }
1460         }
1461
1462     }
1463
1464   connect_attempts += smallWorldConnections;
1465
1466   return connect_attempts;
1467 }
1468
1469 /**
1470  * Create a topology given a peer group (set of running peers)
1471  * and a connection processor.
1472  *
1473  * @param pg the peergroup to create the topology on
1474  * @param proc the connection processor to call to actually set
1475  *        up connections between two peers
1476  *
1477  * @return the number of connections that were set up
1478  *
1479  */
1480 static unsigned int
1481 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg,
1482                        GNUNET_TESTING_ConnectionProcessor proc)
1483 {
1484   unsigned int outer_count, inner_count;
1485   unsigned int cutoff;
1486   int connect_attempts;
1487   double nat_percentage;
1488   char *p_string;
1489
1490   nat_percentage = 0.6;         /* FIXME: default percentage? */
1491   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1492                                                           "TESTING",
1493                                                           "PERCENTAGE",
1494                                                           &p_string))
1495     {
1496       if (sscanf (p_string, "%lf", &nat_percentage) != 1)
1497         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1498                     _
1499                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1500                     p_string, "PERCENTAGE", "TESTING");
1501       GNUNET_free (p_string);
1502     }
1503
1504
1505
1506   cutoff = (unsigned int) (nat_percentage * pg->total);
1507
1508   connect_attempts = 0;
1509
1510   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1511     {
1512       for (inner_count = outer_count + 1; inner_count < pg->total;
1513            inner_count++)
1514         {
1515           if ((outer_count > cutoff) || (inner_count > cutoff))
1516             {
1517 #if VERBOSE_TESTING
1518               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1519                           "Connecting peer %d to peer %d\n",
1520                           outer_count, inner_count);
1521 #endif
1522               connect_attempts += proc (pg, outer_count, inner_count);
1523             }
1524         }
1525     }
1526
1527   return connect_attempts;
1528
1529 }
1530
1531 /**
1532  * Create a topology given a peer group (set of running peers)
1533  * and a connection processor.
1534  *
1535  * @param pg the peergroup to create the topology on
1536  * @param proc the connection processor to call to actually set
1537  *        up connections between two peers
1538  *
1539  * @return the number of connections that were set up
1540  *
1541  */
1542 static unsigned int
1543 create_small_world (struct GNUNET_TESTING_PeerGroup *pg,
1544                     GNUNET_TESTING_ConnectionProcessor proc)
1545 {
1546   unsigned int i, j, k;
1547   unsigned int square;
1548   unsigned int rows;
1549   unsigned int cols;
1550   unsigned int toggle = 1;
1551   unsigned int nodeToConnect;
1552   unsigned int natLog;
1553   unsigned int node1Row;
1554   unsigned int node1Col;
1555   unsigned int node2Row;
1556   unsigned int node2Col;
1557   unsigned int distance;
1558   double probability, random, percentage;
1559   unsigned int smallWorldConnections;
1560   unsigned int small_world_it;
1561   char *p_string;
1562   int connect_attempts;
1563   square = floor (sqrt (pg->total));
1564   rows = square;
1565   cols = square;
1566
1567   percentage = 0.5;             /* FIXME: default percentage? */
1568   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1569                                                           "TESTING",
1570                                                           "PERCENTAGE",
1571                                                           &p_string))
1572     {
1573       if (sscanf (p_string, "%lf", &percentage) != 1)
1574         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1575                     _
1576                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1577                     p_string, "PERCENTAGE", "TESTING");
1578       GNUNET_free (p_string);
1579     }
1580   if (percentage < 0.0)
1581     {
1582       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1583                   _
1584                   ("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1585                   "PERCENTAGE", "TESTING", percentage);
1586       percentage = 0.5;
1587     }
1588   probability = 0.5;            /* FIXME: default percentage? */
1589   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1590                                                           "TESTING",
1591                                                           "PROBABILITY",
1592                                                           &p_string))
1593     {
1594       if (sscanf (p_string, "%lf", &probability) != 1)
1595         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1596                     _
1597                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1598                     p_string, "PROBABILITY", "TESTING");
1599       GNUNET_free (p_string);
1600     }
1601   if (square * square != pg->total)
1602     {
1603       while (rows * cols < pg->total)
1604         {
1605           if (toggle % 2 == 0)
1606             rows++;
1607           else
1608             cols++;
1609
1610           toggle++;
1611         }
1612     }
1613 #if VERBOSE_TESTING
1614   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1615               _
1616               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1617               rows, cols);
1618 #endif
1619
1620   connect_attempts = 0;
1621   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1622    * to the node to its right and above.  Once this is over, we'll have our torus!
1623    * Special case for the last node (if the rows and columns are not equal), connect
1624    * to the first in the row to maintain topology.
1625    */
1626   for (i = 0; i < pg->total; i++)
1627     {
1628       /* First connect to the node to the right */
1629       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1630         nodeToConnect = i + 1;
1631       else if (i + 1 == pg->total)
1632         nodeToConnect = rows * cols - cols;
1633       else
1634         nodeToConnect = i - cols + 1;
1635
1636       connect_attempts += proc (pg, i, nodeToConnect);
1637
1638       if (i < cols)
1639         nodeToConnect = (rows * cols) - cols + i;
1640       else
1641         nodeToConnect = i - cols;
1642
1643       if (nodeToConnect < pg->total)
1644         connect_attempts += proc (pg, i, nodeToConnect);
1645     }
1646   natLog = log (pg->total);
1647 #if VERBOSE_TESTING > 2
1648   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1649               _("natural log of %d is %d, will run %d iterations\n"),
1650               pg->total, natLog, (int) (natLog * percentage));
1651   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1652               _("Total connections added thus far: %u!\n"), connect_attempts);
1653 #endif
1654   smallWorldConnections = 0;
1655   small_world_it = (unsigned int) (natLog * percentage);
1656   if (small_world_it < 1)
1657     small_world_it = 1;
1658   GNUNET_assert (small_world_it > 0 && small_world_it < (unsigned int) -1);
1659   for (i = 0; i < small_world_it; i++)
1660     {
1661       for (j = 0; j < pg->total; j++)
1662         {
1663           /* Determine the row and column of node at position j on the 2d torus */
1664           node1Row = j / cols;
1665           node1Col = j - (node1Row * cols);
1666           for (k = 0; k < pg->total; k++)
1667             {
1668               /* Determine the row and column of node at position k on the 2d torus */
1669               node2Row = k / cols;
1670               node2Col = k - (node2Row * cols);
1671               /* Simple Cartesian distance */
1672               distance =
1673                 abs (node1Row - node2Row) + abs (node1Col - node2Col);
1674               if (distance > 1)
1675                 {
1676                   /* Calculate probability as 1 over the square of the distance */
1677                   probability = 1.0 / (distance * distance);
1678                   /* Choose a random value between 0 and 1 */
1679                   random =
1680                     ((double)
1681                      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1682                                                UINT64_MAX)) /
1683                     ((double) UINT64_MAX);
1684                   /* If random < probability, then connect the two nodes */
1685                   if (random < probability)
1686                     smallWorldConnections += proc (pg, j, k);
1687
1688                 }
1689             }
1690         }
1691     }
1692   connect_attempts += smallWorldConnections;
1693 #if VERBOSE_TESTING > 2
1694   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1695               _("Total connections added for small world: %d!\n"),
1696               smallWorldConnections);
1697 #endif
1698   return connect_attempts;
1699 }
1700
1701 /**
1702  * Create a topology given a peer group (set of running peers)
1703  * and a connection processor.
1704  *
1705  * @param pg the peergroup to create the topology on
1706  * @param proc the connection processor to call to actually set
1707  *        up connections between two peers
1708  *
1709  * @return the number of connections that were set up
1710  *
1711  */
1712 static unsigned int
1713 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg,
1714                     GNUNET_TESTING_ConnectionProcessor proc)
1715 {
1716   double temp_rand;
1717   unsigned int outer_count;
1718   unsigned int inner_count;
1719   int connect_attempts;
1720   double probability;
1721   char *p_string;
1722
1723   probability = 0.5;            /* FIXME: default percentage? */
1724   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1725                                                           "TESTING",
1726                                                           "PROBABILITY",
1727                                                           &p_string))
1728     {
1729       if (sscanf (p_string, "%lf", &probability) != 1)
1730         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1731                     _
1732                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1733                     p_string, "PROBABILITY", "TESTING");
1734       GNUNET_free (p_string);
1735     }
1736   connect_attempts = 0;
1737   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1738     {
1739       for (inner_count = outer_count + 1; inner_count < pg->total;
1740            inner_count++)
1741         {
1742           temp_rand =
1743             ((double)
1744              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1745                                        UINT64_MAX)) / ((double) UINT64_MAX);
1746 #if VERBOSE_TESTING
1747           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1748                       _("rand is %f probability is %f\n"), temp_rand,
1749                       probability);
1750 #endif
1751           if (temp_rand < probability)
1752             {
1753               connect_attempts += proc (pg, outer_count, inner_count);
1754             }
1755         }
1756     }
1757
1758   return connect_attempts;
1759 }
1760
1761 /**
1762  * Create a topology given a peer group (set of running peers)
1763  * and a connection processor.  This particular function creates
1764  * the connections for a 2d-torus, plus additional "closest"
1765  * connections per peer.
1766  *
1767  * @param pg the peergroup to create the topology on
1768  * @param proc the connection processor to call to actually set
1769  *        up connections between two peers
1770  *
1771  * @return the number of connections that were set up
1772  *
1773  */
1774 static unsigned int
1775 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg,
1776                  GNUNET_TESTING_ConnectionProcessor proc)
1777 {
1778   unsigned int i;
1779   unsigned int square;
1780   unsigned int rows;
1781   unsigned int cols;
1782   unsigned int toggle = 1;
1783   unsigned int nodeToConnect;
1784   int connect_attempts;
1785
1786   connect_attempts = 0;
1787
1788   square = floor (sqrt (pg->total));
1789   rows = square;
1790   cols = square;
1791
1792   if (square * square != pg->total)
1793     {
1794       while (rows * cols < pg->total)
1795         {
1796           if (toggle % 2 == 0)
1797             rows++;
1798           else
1799             cols++;
1800
1801           toggle++;
1802         }
1803     }
1804 #if VERBOSE_TESTING
1805   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1806               _
1807               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1808               rows, cols);
1809 #endif
1810   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1811    * to the node to its right and above.  Once this is over, we'll have our torus!
1812    * Special case for the last node (if the rows and columns are not equal), connect
1813    * to the first in the row to maintain topology.
1814    */
1815   for (i = 0; i < pg->total; i++)
1816     {
1817       /* First connect to the node to the right */
1818       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1819         nodeToConnect = i + 1;
1820       else if (i + 1 == pg->total)
1821         nodeToConnect = rows * cols - cols;
1822       else
1823         nodeToConnect = i - cols + 1;
1824 #if VERBOSE_TESTING
1825       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1826                   "Connecting peer %d to peer %d\n", i, nodeToConnect);
1827 #endif
1828       connect_attempts += proc (pg, i, nodeToConnect);
1829
1830       /* Second connect to the node immediately above */
1831       if (i < cols)
1832         nodeToConnect = (rows * cols) - cols + i;
1833       else
1834         nodeToConnect = i - cols;
1835
1836       if (nodeToConnect < pg->total)
1837         {
1838 #if VERBOSE_TESTING
1839           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1840                       "Connecting peer %d to peer %d\n", i, nodeToConnect);
1841 #endif
1842           connect_attempts += proc (pg, i, nodeToConnect);
1843         }
1844
1845     }
1846
1847   return connect_attempts;
1848 }
1849
1850
1851 /**
1852  * Create a topology given a peer group (set of running peers)
1853  * and a connection processor.
1854  *
1855  * @param pg the peergroup to create the topology on
1856  * @param proc the connection processor to call to actually set
1857  *        up connections between two peers
1858  *
1859  * @return the number of connections that were set up
1860  *
1861  */
1862 static unsigned int
1863 create_clique (struct GNUNET_TESTING_PeerGroup *pg,
1864                GNUNET_TESTING_ConnectionProcessor proc)
1865 {
1866   unsigned int outer_count;
1867   unsigned int inner_count;
1868   int connect_attempts;
1869
1870   connect_attempts = 0;
1871
1872   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1873     {
1874       for (inner_count = outer_count + 1; inner_count < pg->total;
1875            inner_count++)
1876         {
1877 #if VERBOSE_TESTING
1878           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1879                       "Connecting peer %d to peer %d\n",
1880                       outer_count, inner_count);
1881 #endif
1882           connect_attempts += proc (pg, outer_count, inner_count);
1883         }
1884     }
1885
1886   return connect_attempts;
1887 }
1888
1889 /**
1890  * Create a topology given a peer group (set of running peers)
1891  * and a connection processor.
1892  *
1893  * @param pg the peergroup to create the topology on
1894  * @param proc the connection processor to call to actually set
1895  *        up connections between two peers
1896  *
1897  * @return the number of connections that were set up
1898  *
1899  */
1900 static unsigned int
1901 create_line (struct GNUNET_TESTING_PeerGroup *pg,
1902              GNUNET_TESTING_ConnectionProcessor proc)
1903 {
1904   unsigned int count;
1905   int connect_attempts;
1906
1907   connect_attempts = 0;
1908
1909   /* Connect each peer to the next highest numbered peer */
1910   for (count = 0; count < pg->total - 1; count++)
1911     {
1912 #if VERBOSE_TESTING
1913       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1914                   "Connecting peer %d to peer %d\n", count, count + 1);
1915 #endif
1916       connect_attempts += proc (pg, count, count + 1);
1917     }
1918
1919   return connect_attempts;
1920 }
1921
1922 /**
1923  * Create a topology given a peer group (set of running peers)
1924  * and a connection processor.
1925  *
1926  * @param pg the peergroup to create the topology on
1927  * @param proc the connection processor to call to actually set
1928  *        up connections between two peers
1929  *
1930  * @return the number of connections that were set up
1931  *
1932  */
1933 static unsigned int
1934 create_ring (struct GNUNET_TESTING_PeerGroup *pg,
1935              GNUNET_TESTING_ConnectionProcessor proc)
1936 {
1937   unsigned int count;
1938   int connect_attempts;
1939
1940   connect_attempts = 0;
1941
1942   /* Connect each peer to the next highest numbered peer */
1943   for (count = 0; count < pg->total - 1; count++)
1944     {
1945 #if VERBOSE_TESTING
1946       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1947                   "Connecting peer %d to peer %d\n", count, count + 1);
1948 #endif
1949       connect_attempts += proc (pg, count, count + 1);
1950     }
1951
1952   /* Connect the last peer to the first peer */
1953   connect_attempts += proc (pg, pg->total - 1, 0);
1954
1955   return connect_attempts;
1956 }
1957
1958
1959 /**
1960  * Iterator for writing friends of a peer to a file.
1961  *
1962  * @param cls closure, an open writable file handle
1963  * @param key the key the daemon was stored under
1964  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1965  *
1966  * @return GNUNET_YES to continue iteration
1967  *
1968  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1969  *       with a single file_iterator that takes a closure which contains
1970  *       the prefix to write before the peer.  Then this could be used
1971  *       for blacklisting multiple transports and writing the friend
1972  *       file.  I'm sure *someone* will complain loudly about other
1973  *       things that negate these functions even existing so no point in
1974  *       "fixing" now.
1975  */
1976 static int
1977 friend_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
1978 {
1979   FILE *temp_friend_handle = cls;
1980   struct GNUNET_TESTING_Daemon *peer = value;
1981   struct GNUNET_PeerIdentity *temppeer;
1982   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1983
1984   temppeer = &peer->id;
1985   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
1986   fprintf (temp_friend_handle, "%s\n", (char *) &peer_enc);
1987
1988   return GNUNET_YES;
1989 }
1990
1991 struct BlacklistContext
1992 {
1993   /*
1994    * The (open) file handle to write to
1995    */
1996   FILE *temp_file_handle;
1997
1998   /*
1999    * The transport that this peer will be blacklisted on.
2000    */
2001   char *transport;
2002 };
2003
2004 /**
2005  * Iterator for writing blacklist data to appropriate files.
2006  *
2007  * @param cls closure, an open writable file handle
2008  * @param key the key the daemon was stored under
2009  * @param value the GNUNET_TESTING_Daemon that needs to be written.
2010  *
2011  * @return GNUNET_YES to continue iteration
2012  */
2013 static int
2014 blacklist_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2015 {
2016   struct BlacklistContext *blacklist_ctx = cls;
2017   //FILE *temp_blacklist_handle = cls;
2018   struct GNUNET_TESTING_Daemon *peer = value;
2019   struct GNUNET_PeerIdentity *temppeer;
2020   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
2021
2022   temppeer = &peer->id;
2023   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
2024   fprintf (blacklist_ctx->temp_file_handle, "%s:%s\n",
2025            blacklist_ctx->transport, (char *) &peer_enc);
2026
2027   return GNUNET_YES;
2028 }
2029
2030 /*
2031  * Create the friend files based on the PeerConnection's
2032  * of each peer in the peer group, and copy the files
2033  * to the appropriate place
2034  *
2035  * @param pg the peer group we are dealing with
2036  */
2037 static int
2038 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
2039 {
2040   FILE *temp_friend_handle;
2041   unsigned int pg_iter;
2042   char *temp_service_path;
2043   struct GNUNET_OS_Process **procarr;
2044   char *arg;
2045   char *mytemp;
2046   enum GNUNET_OS_ProcessStatusType type;
2047   unsigned long return_code;
2048   int count;
2049   int ret;
2050   int max_wait = 10;
2051
2052   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2053   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2054     {
2055       mytemp = GNUNET_DISK_mktemp ("friends");
2056       GNUNET_assert (mytemp != NULL);
2057       temp_friend_handle = fopen (mytemp, "wt");
2058       GNUNET_assert (temp_friend_handle != NULL);
2059       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].allowed_peers,
2060                                              &friend_file_iterator,
2061                                              temp_friend_handle);
2062       fclose (temp_friend_handle);
2063
2064       if (GNUNET_OK !=
2065           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2066                                                  daemon->cfg, "PATHS",
2067                                                  "SERVICEHOME",
2068                                                  &temp_service_path))
2069         {
2070           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2071                       _
2072                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2073                       "SERVICEHOME", "PATHS");
2074           if (UNLINK (mytemp) != 0)
2075             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2076                                       mytemp);
2077           GNUNET_free (mytemp);
2078           break;
2079         }
2080
2081       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2082         {
2083           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
2084           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2085                                                       "mv", mytemp, arg,
2086                                                       NULL);
2087 #if VERBOSE_TESTING
2088           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2089                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2090 #endif
2091
2092           GNUNET_free (arg);
2093         }
2094       else                      /* Remote, scp the file to the correct place */
2095         {
2096           if (NULL != pg->peers[pg_iter].daemon->username)
2097             GNUNET_asprintf (&arg, "%s@%s:%s/friends",
2098                              pg->peers[pg_iter].daemon->username,
2099                              pg->peers[pg_iter].daemon->hostname,
2100                              temp_service_path);
2101           else
2102             GNUNET_asprintf (&arg, "%s:%s/friends",
2103                              pg->peers[pg_iter].daemon->hostname,
2104                              temp_service_path);
2105           procarr[pg_iter] =
2106             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2107                                      NULL);
2108
2109 #if VERBOSE_TESTING
2110           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2111                       _("Copying file with command scp %s %s\n"), mytemp,
2112                       arg);
2113 #endif
2114           GNUNET_free (arg);
2115         }
2116       GNUNET_free (temp_service_path);
2117       GNUNET_free (mytemp);
2118     }
2119
2120   count = 0;
2121   ret = GNUNET_SYSERR;
2122   while ((count < max_wait) && (ret != GNUNET_OK))
2123     {
2124       ret = GNUNET_OK;
2125       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2126         {
2127 #if VERBOSE_TESTING
2128           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2129                       _("Checking copy status of file %d\n"), pg_iter);
2130 #endif
2131           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2132             {
2133               if (GNUNET_OS_process_status
2134                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2135                 {
2136                   ret = GNUNET_SYSERR;
2137                 }
2138               else if ((type != GNUNET_OS_PROCESS_EXITED)
2139                        || (return_code != 0))
2140                 {
2141                   ret = GNUNET_SYSERR;
2142                 }
2143               else
2144                 {
2145                   GNUNET_OS_process_close (procarr[pg_iter]);
2146                   procarr[pg_iter] = NULL;
2147 #if VERBOSE_TESTING
2148                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2149                               _("File %d copied\n"), pg_iter);
2150 #endif
2151                 }
2152             }
2153         }
2154       count++;
2155       if (ret == GNUNET_SYSERR)
2156         {
2157           /* FIXME: why sleep here? -CG */
2158           sleep (1);
2159         }
2160     }
2161
2162 #if VERBOSE_TESTING
2163   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2164               _("Finished copying all friend files!\n"));
2165 #endif
2166   GNUNET_free (procarr);
2167   return ret;
2168 }
2169
2170
2171 /*
2172  * Create the blacklist files based on the PeerConnection's
2173  * of each peer in the peer group, and copy the files
2174  * to the appropriate place.
2175  *
2176  * @param pg the peer group we are dealing with
2177  * @param transports space delimited list of transports to blacklist
2178  */
2179 static int
2180 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
2181                                  const char *transports)
2182 {
2183   FILE *temp_file_handle;
2184   static struct BlacklistContext blacklist_ctx;
2185   unsigned int pg_iter;
2186   char *temp_service_path;
2187   struct GNUNET_OS_Process **procarr;
2188   char *arg;
2189   char *mytemp;
2190   enum GNUNET_OS_ProcessStatusType type;
2191   unsigned long return_code;
2192   int count;
2193   int ret;
2194   int max_wait = 10;
2195   int transport_len;
2196   unsigned int i;
2197   char *pos;
2198   char *temp_transports;
2199
2200   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2201   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2202     {
2203       mytemp = GNUNET_DISK_mktemp ("blacklist");
2204       GNUNET_assert (mytemp != NULL);
2205       temp_file_handle = fopen (mytemp, "wt");
2206       GNUNET_assert (temp_file_handle != NULL);
2207       temp_transports = GNUNET_strdup (transports);
2208       blacklist_ctx.temp_file_handle = temp_file_handle;
2209       transport_len = strlen (temp_transports) + 1;
2210       pos = NULL;
2211
2212       for (i = 0; i < transport_len; i++)
2213         {
2214           if ((temp_transports[i] == ' ') && (pos == NULL))
2215             continue;           /* At start of string (whitespace) */
2216           else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
2217             {
2218               temp_transports[i] = '\0';
2219               blacklist_ctx.transport = pos;
2220               GNUNET_CONTAINER_multihashmap_iterate (pg->
2221                                                      peers
2222                                                      [pg_iter].blacklisted_peers,
2223                                                      &blacklist_file_iterator,
2224                                                      &blacklist_ctx);
2225               pos = NULL;
2226             }                   /* At beginning of actual string */
2227           else if (pos == NULL)
2228             {
2229               pos = &temp_transports[i];
2230             }
2231         }
2232
2233       GNUNET_free (temp_transports);
2234       fclose (temp_file_handle);
2235
2236       if (GNUNET_OK !=
2237           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2238                                                  daemon->cfg, "PATHS",
2239                                                  "SERVICEHOME",
2240                                                  &temp_service_path))
2241         {
2242           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2243                       _
2244                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2245                       "SERVICEHOME", "PATHS");
2246           if (UNLINK (mytemp) != 0)
2247             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2248                                       mytemp);
2249           GNUNET_free (mytemp);
2250           break;
2251         }
2252
2253       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2254         {
2255           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
2256           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2257                                                       "mv", mytemp, arg,
2258                                                       NULL);
2259 #if VERBOSE_TESTING
2260           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2261                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2262 #endif
2263
2264           GNUNET_free (arg);
2265         }
2266       else                      /* Remote, scp the file to the correct place */
2267         {
2268           if (NULL != pg->peers[pg_iter].daemon->username)
2269             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist",
2270                              pg->peers[pg_iter].daemon->username,
2271                              pg->peers[pg_iter].daemon->hostname,
2272                              temp_service_path);
2273           else
2274             GNUNET_asprintf (&arg, "%s:%s/blacklist",
2275                              pg->peers[pg_iter].daemon->hostname,
2276                              temp_service_path);
2277           procarr[pg_iter] =
2278             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2279                                      NULL);
2280
2281 #if VERBOSE_TESTING
2282           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2283                       _("Copying file with command scp %s %s\n"), mytemp,
2284                       arg);
2285 #endif
2286           GNUNET_free (arg);
2287         }
2288       GNUNET_free (temp_service_path);
2289       GNUNET_free (mytemp);
2290     }
2291
2292   count = 0;
2293   ret = GNUNET_SYSERR;
2294   while ((count < max_wait) && (ret != GNUNET_OK))
2295     {
2296       ret = GNUNET_OK;
2297       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2298         {
2299 #if VERBOSE_TESTING
2300           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2301                       _("Checking copy status of file %d\n"), pg_iter);
2302 #endif
2303           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2304             {
2305               if (GNUNET_OS_process_status
2306                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2307                 {
2308                   ret = GNUNET_SYSERR;
2309                 }
2310               else if ((type != GNUNET_OS_PROCESS_EXITED)
2311                        || (return_code != 0))
2312                 {
2313                   ret = GNUNET_SYSERR;
2314                 }
2315               else
2316                 {
2317                   GNUNET_OS_process_close (procarr[pg_iter]);
2318                   procarr[pg_iter] = NULL;
2319 #if VERBOSE_TESTING
2320                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2321                               _("File %d copied\n"), pg_iter);
2322 #endif
2323                 }
2324             }
2325         }
2326       count++;
2327       if (ret == GNUNET_SYSERR)
2328         {
2329           /* FIXME: why sleep here? -CG */
2330           sleep (1);
2331         }
2332     }
2333
2334 #if VERBOSE_TESTING
2335   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2336               _("Finished copying all blacklist files!\n"));
2337 #endif
2338   GNUNET_free (procarr);
2339   return ret;
2340 }
2341
2342
2343 /**
2344  * Internal notification of a connection, kept so that we can ensure some connections
2345  * happen instead of flooding all testing daemons with requests to connect.
2346  */
2347 static void
2348 internal_connect_notify (void *cls,
2349                          const struct GNUNET_PeerIdentity *first,
2350                          const struct GNUNET_PeerIdentity *second,
2351                          uint32_t distance,
2352                          const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2353                          const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2354                          struct GNUNET_TESTING_Daemon *first_daemon,
2355                          struct GNUNET_TESTING_Daemon *second_daemon,
2356                          const char *emsg)
2357 {
2358   struct ConnectTopologyContext *ct_ctx = cls;
2359   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
2360   outstanding_connects--;
2361   ct_ctx->remaining_connections--;
2362   if (ct_ctx->remaining_connections == 0)
2363     {
2364       if (ct_ctx->notify_connections_done != NULL)
2365         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
2366       GNUNET_free (ct_ctx);
2367     }
2368
2369   if (pg->notify_connection != NULL)
2370     pg->notify_connection (pg->notify_connection_cls, first, second, distance,
2371                            first_cfg, second_cfg, first_daemon, second_daemon,
2372                            emsg);
2373 }
2374
2375
2376 /**
2377  * Either delay a connection (because there are too many outstanding)
2378  * or schedule it for right now.
2379  *
2380  * @param cls a connection context
2381  * @param tc the task runtime context
2382  */
2383 static void
2384 schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2385 {
2386   struct ConnectContext *connect_context = cls;
2387
2388   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2389     return;
2390
2391   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2392     {
2393 #if VERBOSE_TESTING > 2
2394       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2395                   _
2396                   ("Delaying connect, we have too many outstanding connections!\n"));
2397 #endif
2398       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
2399                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
2400                                     &schedule_connect, connect_context);
2401     }
2402   else
2403     {
2404 #if VERBOSE_TESTING > 2
2405       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2406                   _("Creating connection, outstanding_connections is %d\n"),
2407                   outstanding_connects);
2408 #endif
2409       outstanding_connects++;
2410       GNUNET_TESTING_daemons_connect (connect_context->first,
2411                                       connect_context->second,
2412                                       CONNECT_TIMEOUT,
2413                                       CONNECT_ATTEMPTS,
2414                                       &internal_connect_notify,
2415                                       connect_context->ct_ctx);
2416       GNUNET_free (connect_context);
2417     }
2418 }
2419
2420
2421 /**
2422  * Iterator for actually scheduling connections to be created
2423  * between two peers.
2424  *
2425  * @param cls closure, a GNUNET_TESTING_Daemon
2426  * @param key the key the second Daemon was stored under
2427  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2428  *
2429  * @return GNUNET_YES to continue iteration
2430  */
2431 static int
2432 connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2433 {
2434   struct ConnectTopologyContext *ct_ctx = cls;
2435   struct PeerData *first = ct_ctx->first;
2436   struct GNUNET_TESTING_Daemon *second = value;
2437   struct ConnectContext *connect_context;
2438
2439   connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2440   connect_context->first = first->daemon;
2441   connect_context->second = second;
2442   connect_context->ct_ctx = ct_ctx;
2443   GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2444
2445   return GNUNET_YES;
2446 }
2447
2448
2449 /**
2450  * Iterator for copying all entries in the allowed hashmap to the
2451  * connect hashmap.
2452  *
2453  * @param cls closure, a GNUNET_TESTING_Daemon
2454  * @param key the key the second Daemon was stored under
2455  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2456  *
2457  * @return GNUNET_YES to continue iteration
2458  */
2459 static int
2460 copy_topology_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2461 {
2462   struct PeerData *first = cls;
2463
2464   GNUNET_assert (GNUNET_OK ==
2465                  GNUNET_CONTAINER_multihashmap_put (first->connect_peers, key,
2466                                                     value,
2467                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2468
2469   return GNUNET_YES;
2470 }
2471
2472 /**
2473  * Make the peers to connect the same as those that are allowed to be
2474  * connected.
2475  *
2476  * @param pg the peer group
2477  */
2478 static int
2479 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2480 {
2481   unsigned int pg_iter;
2482   int ret;
2483   int total;
2484
2485   total = 0;
2486   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2487     {
2488       ret =
2489         GNUNET_CONTAINER_multihashmap_iterate (pg->
2490                                                peers[pg_iter].allowed_peers,
2491                                                &copy_topology_iterator,
2492                                                &pg->peers[pg_iter]);
2493       if (GNUNET_SYSERR == ret)
2494         return GNUNET_SYSERR;
2495
2496       total = total + ret;
2497     }
2498
2499   return total;
2500 }
2501
2502
2503 /**
2504  * Connect the topology as specified by the PeerConnection's
2505  * of each peer in the peer group
2506  *
2507  * @param pg the peer group we are dealing with
2508  * @return the number of connections that will be attempted
2509  */
2510 static int
2511 connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
2512                   GNUNET_TESTING_NotifyCompletion notify_callback,
2513                   void *notify_cls)
2514 {
2515   unsigned int pg_iter;
2516   int ret;
2517   unsigned int total;
2518   struct ConnectTopologyContext *ct_ctx;
2519 #if OLD
2520   struct PeerConnection *connection_iter;
2521   struct ConnectContext *connect_context;
2522 #endif
2523
2524   total = 0;
2525   ct_ctx = GNUNET_malloc (sizeof (struct ConnectTopologyContext));
2526   ct_ctx->notify_connections_done = notify_callback;
2527   ct_ctx->notify_cls = notify_cls;
2528   ct_ctx->pg = pg;
2529
2530   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2531     {
2532       total +=
2533         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
2534     }
2535
2536   if (total == 0)
2537     {
2538       GNUNET_free (ct_ctx);
2539       return total;
2540     }
2541   ct_ctx->remaining_connections = total;
2542   total = 0;
2543
2544   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2545     {
2546       ct_ctx->first = &pg->peers[pg_iter];
2547       ret =
2548         GNUNET_CONTAINER_multihashmap_iterate (pg->
2549                                                peers[pg_iter].connect_peers,
2550                                                &connect_iterator, ct_ctx);
2551       GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
2552       total = total + ret;
2553
2554 #if OLD
2555       connection_iter = FIXME;
2556       while (connection_iter != NULL)
2557         {
2558           connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2559           connect_context->pg = pg;
2560           connect_context->first = FIXME;
2561           connect_context->second = connection_iter->daemon;
2562           GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2563           connection_iter = connection_iter->next;
2564         }
2565 #endif
2566     }
2567   return total;
2568 }
2569
2570
2571 /**
2572  * Takes a peer group and creates a topology based on the
2573  * one specified.  Creates a topology means generates friend
2574  * files for the peers so they can only connect to those allowed
2575  * by the topology.  This will only have an effect once peers
2576  * are started if the FRIENDS_ONLY option is set in the base
2577  * config.  Also takes an optional restrict topology which
2578  * disallows connections based on particular transports
2579  * UNLESS they are specified in the restricted topology.
2580  *
2581  * @param pg the peer group struct representing the running peers
2582  * @param topology which topology to connect the peers in
2583  * @param restrict_topology disallow restrict_transports transport
2584  *                          connections to peers NOT in this topology
2585  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2586  * @param restrict_transports space delimited list of transports to blacklist
2587  *                            to create restricted topology
2588  *
2589  * @return the maximum number of connections were all allowed peers
2590  *         connected to each other
2591  */
2592 unsigned int
2593 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2594                                 enum GNUNET_TESTING_Topology topology,
2595                                 enum GNUNET_TESTING_Topology
2596                                 restrict_topology,
2597                                 const char *restrict_transports)
2598 {
2599   int ret;
2600   unsigned int num_connections;
2601   int unblacklisted_connections;
2602
2603   switch (topology)
2604     {
2605     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2606 #if VERBOSE_TESTING
2607       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating clique topology\n"));
2608 #endif
2609       num_connections = create_clique (pg, &add_allowed_connections);
2610       break;
2611     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2612 #if VERBOSE_TESTING
2613       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2614                   _("Creating small world (ring) topology\n"));
2615 #endif
2616       num_connections =
2617         create_small_world_ring (pg, &add_allowed_connections);
2618       break;
2619     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2620 #if VERBOSE_TESTING
2621       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2622                   _("Creating small world (2d-torus) topology\n"));
2623 #endif
2624       num_connections = create_small_world (pg, &add_allowed_connections);
2625       break;
2626     case GNUNET_TESTING_TOPOLOGY_RING:
2627 #if VERBOSE_TESTING
2628       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating ring topology\n"));
2629 #endif
2630       num_connections = create_ring (pg, &add_allowed_connections);
2631       break;
2632     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2633 #if VERBOSE_TESTING
2634       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating 2d torus topology\n"));
2635 #endif
2636       num_connections = create_2d_torus (pg, &add_allowed_connections);
2637       break;
2638     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2639 #if VERBOSE_TESTING
2640       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2641                   _("Creating Erdos-Renyi topology\n"));
2642 #endif
2643       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2644       break;
2645     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2646 #if VERBOSE_TESTING
2647       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating InterNAT topology\n"));
2648 #endif
2649       num_connections = create_nated_internet (pg, &add_allowed_connections);
2650       break;
2651     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2652 #if VERBOSE_TESTING
2653       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2654                   _("Creating Scale Free topology\n"));
2655 #endif
2656       num_connections = create_scale_free (pg, &add_allowed_connections);
2657       break;
2658     case GNUNET_TESTING_TOPOLOGY_LINE:
2659 #if VERBOSE_TESTING
2660       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2661                   _("Creating straight line topology\n"));
2662 #endif
2663       num_connections = create_line (pg, &add_allowed_connections);
2664       break;
2665     case GNUNET_TESTING_TOPOLOGY_NONE:
2666 #if VERBOSE_TESTING
2667       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2668                   _
2669                   ("Creating no allowed topology (all peers can connect at core level)\n"));
2670 #endif
2671       num_connections = 0;
2672       break;
2673     default:
2674       num_connections = 0;
2675       break;
2676     }
2677
2678   if (GNUNET_YES ==
2679       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2680     {
2681       ret = create_and_copy_friend_files (pg);
2682       if (ret != GNUNET_OK)
2683         {
2684 #if VERBOSE_TESTING
2685           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2686                       _("Failed during friend file copying!\n"));
2687 #endif
2688           return GNUNET_SYSERR;
2689         }
2690       else
2691         {
2692 #if VERBOSE_TESTING
2693           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2694                       _("Friend files created/copied successfully!\n"));
2695 #endif
2696         }
2697     }
2698
2699   /* Use the create clique method to initially set all connections as blacklisted. */
2700   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2701     create_clique (pg, &blacklist_connections);
2702
2703   unblacklisted_connections = 0;
2704   /* Un-blacklist connections as per the topology specified */
2705   switch (restrict_topology)
2706     {
2707     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2708 #if VERBOSE_TESTING
2709       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2710                   _("Blacklisting all but clique topology\n"));
2711 #endif
2712       unblacklisted_connections =
2713         create_clique (pg, &unblacklist_connections);
2714       break;
2715     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2716 #if VERBOSE_TESTING
2717       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2718                   _("Blacklisting all but small world (ring) topology\n"));
2719 #endif
2720       unblacklisted_connections =
2721         create_small_world_ring (pg, &unblacklist_connections);
2722       break;
2723     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2724 #if VERBOSE_TESTING
2725       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2726                   _
2727                   ("Blacklisting all but small world (2d-torus) topology\n"));
2728 #endif
2729       unblacklisted_connections =
2730         create_small_world (pg, &unblacklist_connections);
2731       break;
2732     case GNUNET_TESTING_TOPOLOGY_RING:
2733 #if VERBOSE_TESTING
2734       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2735                   _("Blacklisting all but ring topology\n"));
2736 #endif
2737       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2738       break;
2739     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2740 #if VERBOSE_TESTING
2741       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2742                   _("Blacklisting all but 2d torus topology\n"));
2743 #endif
2744       unblacklisted_connections =
2745         create_2d_torus (pg, &unblacklist_connections);
2746       break;
2747     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2748 #if VERBOSE_TESTING
2749       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2750                   _("Blacklisting all but Erdos-Renyi topology\n"));
2751 #endif
2752       unblacklisted_connections =
2753         create_erdos_renyi (pg, &unblacklist_connections);
2754       break;
2755     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2756 #if VERBOSE_TESTING
2757       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2758                   _("Blacklisting all but InterNAT topology\n"));
2759 #endif
2760       unblacklisted_connections =
2761         create_nated_internet (pg, &unblacklist_connections);
2762       break;
2763     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2764 #if VERBOSE_TESTING
2765       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2766                   _("Blacklisting all but Scale Free topology\n"));
2767 #endif
2768       unblacklisted_connections =
2769         create_scale_free (pg, &unblacklist_connections);
2770       break;
2771     case GNUNET_TESTING_TOPOLOGY_LINE:
2772 #if VERBOSE_TESTING
2773       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2774                   _("Blacklisting all but straight line topology\n"));
2775 #endif
2776       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2777       break;
2778     case GNUNET_TESTING_TOPOLOGY_NONE:
2779 #if VERBOSE_TESTING
2780       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2781                   _
2782                   ("Creating no blacklist topology (all peers can connect at transport level)\n"));
2783 #endif
2784     default:
2785       break;
2786     }
2787
2788   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2789     {
2790       ret = create_and_copy_blacklist_files (pg, restrict_transports);
2791       if (ret != GNUNET_OK)
2792         {
2793 #if VERBOSE_TESTING
2794           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2795                       _("Failed during blacklist file copying!\n"));
2796 #endif
2797           return 0;
2798         }
2799       else
2800         {
2801 #if VERBOSE_TESTING
2802           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2803                       _("Blacklist files created/copied successfully!\n"));
2804 #endif
2805         }
2806     }
2807   return num_connections;
2808 }
2809
2810 struct RandomContext
2811 {
2812   /**
2813    * The peergroup
2814    */
2815   struct GNUNET_TESTING_PeerGroup *pg;
2816
2817   /**
2818    * uid of the first peer
2819    */
2820   uint32_t first_uid;
2821
2822   /**
2823    * Peer data for first peer.
2824    */
2825   struct PeerData *first;
2826
2827   /**
2828    * Random percentage to use
2829    */
2830   double percentage;
2831 };
2832
2833 struct MinimumContext
2834 {
2835   /**
2836    * The peergroup
2837    */
2838   struct GNUNET_TESTING_PeerGroup *pg;
2839
2840   /**
2841    * uid of the first peer
2842    */
2843   uint32_t first_uid;
2844
2845   /**
2846    * Peer data for first peer.
2847    */
2848   struct PeerData *first;
2849
2850   /**
2851    * Number of conns per peer
2852    */
2853   unsigned int num_to_add;
2854
2855   /**
2856    * Permuted array of all possible connections.  Only add the Nth
2857    * peer if it's in the Nth position.
2858    */
2859   unsigned int *pg_array;
2860
2861   /**
2862    * What number is the current element we are iterating over?
2863    */
2864   unsigned int current;
2865 };
2866
2867 struct DFSContext
2868 {
2869   /**
2870    * The peergroup
2871    */
2872   struct GNUNET_TESTING_PeerGroup *pg;
2873
2874   /**
2875    * uid of the first peer
2876    */
2877   uint32_t first_uid;
2878
2879   /**
2880    * uid of the second peer
2881    */
2882   uint32_t second_uid;
2883
2884   /**
2885    * Peer data for first peer.
2886    */
2887   struct PeerData *first;
2888
2889   /**
2890    * Which peer has been chosen as the one to add?
2891    */
2892   unsigned int chosen;
2893
2894   /**
2895    * What number is the current element we are iterating over?
2896    */
2897   unsigned int current;
2898 };
2899
2900 /**
2901  * Iterator for choosing random peers to connect.
2902  *
2903  * @param cls closure, a RandomContext
2904  * @param key the key the second Daemon was stored under
2905  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2906  *
2907  * @return GNUNET_YES to continue iteration
2908  */
2909 static int
2910 random_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2911 {
2912   struct RandomContext *random_ctx = cls;
2913   double random_number;
2914   uint32_t second_pos;
2915   GNUNET_HashCode first_hash;
2916   random_number =
2917     ((double)
2918      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
2919                                UINT64_MAX)) / ((double) UINT64_MAX);
2920   if (random_number < random_ctx->percentage)
2921     {
2922       GNUNET_assert (GNUNET_OK ==
2923                      GNUNET_CONTAINER_multihashmap_put (random_ctx->
2924                                                         first->connect_peers_working_set,
2925                                                         key, value,
2926                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2927     }
2928   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2929   uid_from_hash (key, &second_pos);
2930   hash_from_uid (random_ctx->first_uid, &first_hash);
2931   GNUNET_assert (random_ctx->pg->total > second_pos);
2932   GNUNET_assert (GNUNET_YES ==
2933                  GNUNET_CONTAINER_multihashmap_remove (random_ctx->
2934                                                        pg->peers
2935                                                        [second_pos].connect_peers,
2936                                                        &first_hash,
2937                                                        random_ctx->
2938                                                        first->daemon));
2939
2940   return GNUNET_YES;
2941 }
2942
2943 /**
2944  * Iterator for adding at least X peers to a peers connection set.
2945  *
2946  * @param cls closure, MinimumContext
2947  * @param key the key the second Daemon was stored under
2948  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2949  *
2950  * @return GNUNET_YES to continue iteration
2951  */
2952 static int
2953 minimum_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2954 {
2955   struct MinimumContext *min_ctx = cls;
2956   uint32_t second_pos;
2957   GNUNET_HashCode first_hash;
2958   unsigned int i;
2959
2960   if (GNUNET_CONTAINER_multihashmap_size
2961       (min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2962     {
2963       for (i = 0; i < min_ctx->num_to_add; i++)
2964         {
2965           if (min_ctx->pg_array[i] == min_ctx->current)
2966             {
2967               GNUNET_assert (GNUNET_OK ==
2968                              GNUNET_CONTAINER_multihashmap_put
2969                              (min_ctx->first->connect_peers_working_set, key,
2970                               value,
2971                               GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2972               uid_from_hash (key, &second_pos);
2973               hash_from_uid (min_ctx->first_uid, &first_hash);
2974               GNUNET_assert (min_ctx->pg->total > second_pos);
2975               GNUNET_assert (GNUNET_OK ==
2976                              GNUNET_CONTAINER_multihashmap_put (min_ctx->
2977                                                                 pg->peers
2978                                                                 [second_pos].connect_peers_working_set,
2979                                                                 &first_hash,
2980                                                                 min_ctx->first->
2981                                                                 daemon,
2982                                                                 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2983               /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2984               GNUNET_assert (GNUNET_YES ==
2985                              GNUNET_CONTAINER_multihashmap_remove
2986                              (min_ctx->pg->peers[second_pos].connect_peers,
2987                               &first_hash, min_ctx->first->daemon));
2988             }
2989         }
2990       min_ctx->current++;
2991       return GNUNET_YES;
2992     }
2993   else
2994     return GNUNET_NO;           /* We can stop iterating, we have enough peers! */
2995
2996 }
2997
2998
2999 /**
3000  * Iterator for adding peers to a connection set based on a depth first search.
3001  *
3002  * @param cls closure, MinimumContext
3003  * @param key the key the second daemon was stored under
3004  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
3005  *
3006  * @return GNUNET_YES to continue iteration
3007  */
3008 static int
3009 dfs_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
3010 {
3011   struct DFSContext *dfs_ctx = cls;
3012   GNUNET_HashCode first_hash;
3013
3014   if (dfs_ctx->current == dfs_ctx->chosen)
3015     {
3016       GNUNET_assert (GNUNET_OK ==
3017                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3018                                                         first->connect_peers_working_set,
3019                                                         key, value,
3020                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3021       uid_from_hash (key, &dfs_ctx->second_uid);
3022       hash_from_uid (dfs_ctx->first_uid, &first_hash);
3023       GNUNET_assert (GNUNET_OK ==
3024                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3025                                                         pg->peers
3026                                                         [dfs_ctx->second_uid].connect_peers_working_set,
3027                                                         &first_hash,
3028                                                         dfs_ctx->
3029                                                         first->daemon,
3030                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3031       GNUNET_assert (GNUNET_YES ==
3032                      GNUNET_CONTAINER_multihashmap_remove (dfs_ctx->
3033                                                            pg->peers
3034                                                            [dfs_ctx->second_uid].connect_peers,
3035                                                            &first_hash,
3036                                                            dfs_ctx->
3037                                                            first->daemon));
3038       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
3039       return GNUNET_NO;         /* We have found our peer, don't iterate more */
3040     }
3041
3042   dfs_ctx->current++;
3043   return GNUNET_YES;
3044 }
3045
3046
3047 /**
3048  * From the set of connections possible, choose percentage percent of connections
3049  * to actually connect.
3050  *
3051  * @param pg the peergroup we are dealing with
3052  * @param percentage what percent of total connections to make
3053  */
3054 void
3055 choose_random_connections (struct GNUNET_TESTING_PeerGroup *pg,
3056                            double percentage)
3057 {
3058   struct RandomContext random_ctx;
3059   uint32_t pg_iter;
3060
3061   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3062     {
3063       random_ctx.first_uid = pg_iter;
3064       random_ctx.first = &pg->peers[pg_iter];
3065       random_ctx.percentage = percentage;
3066       random_ctx.pg = pg;
3067       pg->peers[pg_iter].connect_peers_working_set =
3068         GNUNET_CONTAINER_multihashmap_create (pg->total);
3069       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3070                                              &random_connect_iterator,
3071                                              &random_ctx);
3072       /* Now remove the old connections */
3073       GNUNET_CONTAINER_multihashmap_destroy (pg->
3074                                              peers[pg_iter].connect_peers);
3075       /* And replace with the random set */
3076       pg->peers[pg_iter].connect_peers =
3077         pg->peers[pg_iter].connect_peers_working_set;
3078     }
3079 }
3080
3081 /**
3082  * From the set of connections possible, choose at least num connections per
3083  * peer.
3084  *
3085  * @param pg the peergroup we are dealing with
3086  * @param num how many connections at least should each peer have (if possible)?
3087  */
3088 static void
3089 choose_minimum (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3090 {
3091   struct MinimumContext minimum_ctx;
3092   uint32_t pg_iter;
3093
3094   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3095     {
3096       pg->peers[pg_iter].connect_peers_working_set =
3097         GNUNET_CONTAINER_multihashmap_create (num);
3098     }
3099
3100   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3101     {
3102       minimum_ctx.first_uid = pg_iter;
3103       minimum_ctx.pg_array =
3104         GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK,
3105                                       GNUNET_CONTAINER_multihashmap_size
3106                                       (pg->peers[pg_iter].connect_peers));
3107       minimum_ctx.first = &pg->peers[pg_iter];
3108       minimum_ctx.pg = pg;
3109       minimum_ctx.num_to_add = num;
3110       minimum_ctx.current = 0;
3111       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3112                                              &minimum_connect_iterator,
3113                                              &minimum_ctx);
3114     }
3115
3116   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3117     {
3118       /* Remove the "old" connections */
3119       GNUNET_CONTAINER_multihashmap_destroy (pg->
3120                                              peers[pg_iter].connect_peers);
3121       /* And replace with the working set */
3122       pg->peers[pg_iter].connect_peers =
3123         pg->peers[pg_iter].connect_peers_working_set;
3124     }
3125
3126 }
3127
3128
3129 static unsigned int
3130 count_workingset_connections (struct GNUNET_TESTING_PeerGroup *pg)
3131 {
3132   unsigned int count;
3133   unsigned int pg_iter;
3134
3135   count = 0;
3136
3137   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3138     {
3139       count +=
3140         GNUNET_CONTAINER_multihashmap_size (pg->
3141                                             peers
3142                                             [pg_iter].connect_peers_working_set);
3143     }
3144
3145   return count;
3146 }
3147
3148
3149 static unsigned int
3150 count_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg)
3151 {
3152   unsigned int count;
3153   unsigned int pg_iter;
3154
3155   count = 0;
3156
3157   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3158     {
3159       count +=
3160         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
3161     }
3162
3163   return count;
3164 }
3165
3166
3167 struct FindClosestContext
3168 {
3169   /**
3170    * The currently known closest peer.
3171    */
3172   struct GNUNET_TESTING_Daemon *closest;
3173
3174   /**
3175    * The info for the peer we are adding connections for.
3176    */
3177   struct PeerData *curr_peer;
3178
3179   /**
3180    * The distance (bits) between the current
3181    * peer and the currently known closest.
3182    */
3183   unsigned int closest_dist;
3184
3185   /**
3186    * The offset of the closest known peer in
3187    * the peer group.
3188    */
3189   unsigned int closest_num;
3190 };
3191
3192 /**
3193  * Iterator over hash map entries of the allowed
3194  * peer connections.  Find the closest, not already
3195  * connected peer and return it.
3196  *
3197  * @param cls closure (struct FindClosestContext)
3198  * @param key current key code (hash of offset in pg)
3199  * @param value value in the hash map - a GNUNET_TESTING_Daemon
3200  * @return GNUNET_YES if we should continue to
3201  *         iterate,
3202  *         GNUNET_NO if not.
3203  */
3204 static int
3205 find_closest_peers (void *cls, const GNUNET_HashCode * key, void *value)
3206 {
3207   struct FindClosestContext *closest_ctx = cls;
3208   struct GNUNET_TESTING_Daemon *daemon = value;
3209
3210   if (((closest_ctx->closest == NULL) ||
3211        (GNUNET_CRYPTO_hash_matching_bits
3212         (&daemon->id.hashPubKey,
3213          &closest_ctx->curr_peer->daemon->id.hashPubKey) >
3214         closest_ctx->closest_dist))
3215       && (GNUNET_YES !=
3216           GNUNET_CONTAINER_multihashmap_contains (closest_ctx->
3217                                                   curr_peer->connect_peers,
3218                                                   key)))
3219     {
3220       closest_ctx->closest_dist =
3221         GNUNET_CRYPTO_hash_matching_bits (&daemon->id.hashPubKey,
3222                                           &closest_ctx->curr_peer->daemon->
3223                                           id.hashPubKey);
3224       closest_ctx->closest = daemon;
3225       uid_from_hash (key, &closest_ctx->closest_num);
3226     }
3227   return GNUNET_YES;
3228 }
3229
3230 /**
3231  * From the set of connections possible, choose at num connections per
3232  * peer based on depth which are closest out of those allowed.  Guaranteed
3233  * to add num peers to connect to, provided there are that many peers
3234  * in the underlay topology to connect to.
3235  *
3236  * @param pg the peergroup we are dealing with
3237  * @param num how many connections at least should each peer have (if possible)?
3238  * @param proc processor to actually add the connections
3239  */
3240 void
3241 add_closest (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num,
3242              GNUNET_TESTING_ConnectionProcessor proc)
3243 {
3244   struct FindClosestContext closest_ctx;
3245   uint32_t pg_iter;
3246   uint32_t i;
3247
3248   for (i = 0; i < num; i++)     /* Each time find a closest peer (from those available) */
3249     {
3250       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3251         {
3252           closest_ctx.curr_peer = &pg->peers[pg_iter];
3253           closest_ctx.closest = NULL;
3254           closest_ctx.closest_dist = 0;
3255           closest_ctx.closest_num = 0;
3256           GNUNET_CONTAINER_multihashmap_iterate (pg->
3257                                                  peers[pg_iter].allowed_peers,
3258                                                  &find_closest_peers,
3259                                                  &closest_ctx);
3260           if (closest_ctx.closest != NULL)
3261             {
3262               GNUNET_assert (closest_ctx.closest_num < pg->total);
3263               proc (pg, pg_iter, closest_ctx.closest_num);
3264             }
3265         }
3266     }
3267 }
3268
3269 /**
3270  * From the set of connections possible, choose at least num connections per
3271  * peer based on depth first traversal of peer connections.  If DFS leaves
3272  * peers unconnected, ensure those peers get connections.
3273  *
3274  * @param pg the peergroup we are dealing with
3275  * @param num how many connections at least should each peer have (if possible)?
3276  */
3277 void
3278 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3279 {
3280   struct DFSContext dfs_ctx;
3281   uint32_t pg_iter;
3282   uint32_t dfs_count;
3283   uint32_t starting_peer;
3284   uint32_t least_connections;
3285   GNUNET_HashCode second_hash;
3286
3287   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3288     {
3289       pg->peers[pg_iter].connect_peers_working_set =
3290         GNUNET_CONTAINER_multihashmap_create (num);
3291     }
3292
3293   starting_peer = 0;
3294   dfs_count = 0;
3295   while ((count_workingset_connections (pg) < num * pg->total)
3296          && (count_allowed_connections (pg) > 0))
3297     {
3298       if (dfs_count % pg->total == 0)   /* Restart the DFS at some weakly connected peer */
3299         {
3300           least_connections = -1;       /* Set to very high number */
3301           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3302             {
3303               if (GNUNET_CONTAINER_multihashmap_size
3304                   (pg->peers[pg_iter].connect_peers_working_set) <
3305                   least_connections)
3306                 {
3307                   starting_peer = pg_iter;
3308                   least_connections =
3309                     GNUNET_CONTAINER_multihashmap_size (pg->
3310                                                         peers
3311                                                         [pg_iter].connect_peers_working_set);
3312                 }
3313             }
3314         }
3315
3316       if (GNUNET_CONTAINER_multihashmap_size (pg->peers[starting_peer].connect_peers) == 0)     /* Ensure there is at least one peer left to connect! */
3317         {
3318           dfs_count = 0;
3319           continue;
3320         }
3321
3322       /* Choose a random peer from the chosen peers set of connections to add */
3323       dfs_ctx.chosen =
3324         GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
3325                                   GNUNET_CONTAINER_multihashmap_size
3326                                   (pg->peers[starting_peer].connect_peers));
3327       dfs_ctx.first_uid = starting_peer;
3328       dfs_ctx.first = &pg->peers[starting_peer];
3329       dfs_ctx.pg = pg;
3330       dfs_ctx.current = 0;
3331
3332       GNUNET_CONTAINER_multihashmap_iterate (pg->
3333                                              peers
3334                                              [starting_peer].connect_peers,
3335                                              &dfs_connect_iterator, &dfs_ctx);
3336       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
3337       hash_from_uid (dfs_ctx.second_uid, &second_hash);
3338       GNUNET_assert (GNUNET_YES ==
3339                      GNUNET_CONTAINER_multihashmap_remove (pg->peers
3340                                                            [starting_peer].connect_peers,
3341                                                            &second_hash,
3342                                                            pg->
3343                                                            peers
3344                                                            [dfs_ctx.second_uid].daemon));
3345       starting_peer = dfs_ctx.second_uid;
3346     }
3347
3348   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3349     {
3350       /* Remove the "old" connections */
3351       GNUNET_CONTAINER_multihashmap_destroy (pg->
3352                                              peers[pg_iter].connect_peers);
3353       /* And replace with the working set */
3354       pg->peers[pg_iter].connect_peers =
3355         pg->peers[pg_iter].connect_peers_working_set;
3356     }
3357 }
3358
3359 /**
3360  * Internal callback for topology information for a particular peer.
3361  */
3362 static void
3363 internal_topology_callback (void *cls,
3364                             const struct GNUNET_PeerIdentity *peer,
3365                             const struct GNUNET_TRANSPORT_ATS_Information
3366                             *atsi)
3367 {
3368   struct CoreContext *core_ctx = cls;
3369   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
3370
3371   if (peer == NULL)             /* Either finished, or something went wrong */
3372     {
3373       iter_ctx->completed++;
3374       iter_ctx->connected--;
3375       /* One core context allocated per iteration, must free! */
3376       GNUNET_free (core_ctx);
3377     }
3378   else
3379     {
3380       iter_ctx->topology_cb (iter_ctx->cls, &core_ctx->daemon->id,
3381                              peer, NULL);
3382     }
3383
3384   if (iter_ctx->completed == iter_ctx->total)
3385     {
3386       iter_ctx->topology_cb (iter_ctx->cls, NULL, NULL, NULL);
3387       /* Once all are done, free the iteration context */
3388       GNUNET_free (iter_ctx);
3389     }
3390 }
3391
3392
3393 /**
3394  * Check running topology iteration tasks, if below max start a new one, otherwise
3395  * schedule for some time in the future.
3396  */
3397 static void
3398 schedule_get_topology (void *cls,
3399                        const struct GNUNET_SCHEDULER_TaskContext *tc)
3400 {
3401   struct CoreContext *core_context = cls;
3402   struct TopologyIterateContext *topology_context =
3403     (struct TopologyIterateContext *) core_context->iter_context;
3404   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3405     return;
3406
3407   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3408     {
3409 #if VERBOSE_TESTING > 2
3410       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3411                   _
3412                   ("Delaying connect, we have too many outstanding connections!\n"));
3413 #endif
3414       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3415                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3416                                     &schedule_get_topology, core_context);
3417     }
3418   else
3419     {
3420 #if VERBOSE_TESTING > 2
3421       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3422                   _("Creating connection, outstanding_connections is %d\n"),
3423                   outstanding_connects);
3424 #endif
3425       topology_context->connected++;
3426
3427       if (GNUNET_OK !=
3428           GNUNET_CORE_iterate_peers (core_context->daemon->cfg,
3429                                      &internal_topology_callback,
3430                                      core_context))
3431         {
3432           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Topology iteration failed.\n");
3433           internal_topology_callback (core_context, NULL, NULL);
3434         }
3435     }
3436 }
3437
3438 /**
3439  * Iterate over all (running) peers in the peer group, retrieve
3440  * all connections that each currently has.
3441  */
3442 void
3443 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
3444                              GNUNET_TESTING_NotifyTopology cb, void *cls)
3445 {
3446   struct TopologyIterateContext *topology_context;
3447   struct CoreContext *core_ctx;
3448   unsigned int i;
3449   unsigned int total_count;
3450
3451   /* Allocate a single topology iteration context */
3452   topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
3453   topology_context->topology_cb = cb;
3454   topology_context->cls = cls;
3455   total_count = 0;
3456   for (i = 0; i < pg->total; i++)
3457     {
3458       if (pg->peers[i].daemon->running == GNUNET_YES)
3459         {
3460           /* Allocate one core context per core we need to connect to */
3461           core_ctx = GNUNET_malloc (sizeof (struct CoreContext));
3462           core_ctx->daemon = pg->peers[i].daemon;
3463           /* Set back pointer to topology iteration context */
3464           core_ctx->iter_context = topology_context;
3465           GNUNET_SCHEDULER_add_now (&schedule_get_topology, core_ctx);
3466           total_count++;
3467         }
3468     }
3469   if (total_count == 0)
3470     {
3471       cb (cls, NULL, NULL, "Cannot iterate over topology, no running peers!");
3472       GNUNET_free (topology_context);
3473     }
3474   else
3475     topology_context->total = total_count;
3476   return;
3477 }
3478
3479 /**
3480  * Callback function to process statistic values.
3481  * This handler is here only really to insert a peer
3482  * identity (or daemon) so the statistics can be uniquely
3483  * tied to a single running peer.
3484  *
3485  * @param cls closure
3486  * @param subsystem name of subsystem that created the statistic
3487  * @param name the name of the datum
3488  * @param value the current value
3489  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
3490  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
3491  */
3492 static int
3493 internal_stats_callback (void *cls,
3494                          const char *subsystem,
3495                          const char *name, uint64_t value, int is_persistent)
3496 {
3497   struct StatsCoreContext *core_context = cls;
3498   struct StatsIterateContext *stats_context =
3499     (struct StatsIterateContext *) core_context->iter_context;
3500
3501   return stats_context->proc (stats_context->cls, &core_context->daemon->id,
3502                               subsystem, name, value, is_persistent);
3503 }
3504
3505 /**
3506  * Internal continuation call for statistics iteration.
3507  *
3508  * @param cls closure, the CoreContext for this iteration
3509  * @param success whether or not the statistics iterations
3510  *        was canceled or not (we don't care)
3511  */
3512 static void
3513 internal_stats_cont (void *cls, int success)
3514 {
3515   struct StatsCoreContext *core_context = cls;
3516   struct StatsIterateContext *stats_context =
3517     (struct StatsIterateContext *) core_context->iter_context;
3518
3519   stats_context->connected--;
3520   stats_context->completed++;
3521
3522   if (stats_context->completed == stats_context->total)
3523     {
3524       stats_context->cont (stats_context->cls, GNUNET_YES);
3525       GNUNET_free (stats_context);
3526     }
3527
3528   if (core_context->stats_handle != NULL)
3529     GNUNET_STATISTICS_destroy (core_context->stats_handle, GNUNET_NO);
3530
3531   GNUNET_free (core_context);
3532 }
3533
3534 /**
3535  * Check running topology iteration tasks, if below max start a new one, otherwise
3536  * schedule for some time in the future.
3537  */
3538 static void
3539 schedule_get_statistics (void *cls,
3540                          const struct GNUNET_SCHEDULER_TaskContext *tc)
3541 {
3542   struct StatsCoreContext *core_context = cls;
3543   struct StatsIterateContext *stats_context =
3544     (struct StatsIterateContext *) core_context->iter_context;
3545
3546   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3547     return;
3548
3549   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3550     {
3551 #if VERBOSE_TESTING > 2
3552       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3553                   _
3554                   ("Delaying connect, we have too many outstanding connections!\n"));
3555 #endif
3556       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3557                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3558                                     &schedule_get_statistics, core_context);
3559     }
3560   else
3561     {
3562 #if VERBOSE_TESTING > 2
3563       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3564                   _("Creating connection, outstanding_connections is %d\n"),
3565                   outstanding_connects);
3566 #endif
3567
3568       stats_context->connected++;
3569       core_context->stats_handle =
3570         GNUNET_STATISTICS_create ("testing", core_context->daemon->cfg);
3571       if (core_context->stats_handle == NULL)
3572         {
3573           internal_stats_cont (core_context, GNUNET_NO);
3574           return;
3575         }
3576
3577       core_context->stats_get_handle =
3578         GNUNET_STATISTICS_get (core_context->stats_handle, NULL, NULL,
3579                                GNUNET_TIME_relative_get_forever (),
3580                                &internal_stats_cont, &internal_stats_callback,
3581                                core_context);
3582       if (core_context->stats_get_handle == NULL)
3583         internal_stats_cont (core_context, GNUNET_NO);
3584
3585     }
3586 }
3587
3588 struct DuplicateStats
3589 {
3590   /**
3591    * Next item in the list
3592    */
3593   struct DuplicateStats *next;
3594
3595   /**
3596    * Nasty string, concatenation of relevant information.
3597    */
3598   char *unique_string;
3599 };
3600
3601 /**
3602  * Check whether the combination of port/host/unix domain socket
3603  * already exists in the list of peers being checked for statistics.
3604  *
3605  * @param pg the peergroup in question
3606  * @param specific_peer the peer we're concerned with
3607  * @param stats_list the list to return to the caller
3608  *
3609  * @return GNUNET_YES if the statistics instance has been seen already,
3610  *         GNUNET_NO if not (and we may have added it to the list)
3611  */
3612 static int
3613 stats_check_existing (struct GNUNET_TESTING_PeerGroup *pg,
3614                       struct PeerData *specific_peer,
3615                       struct DuplicateStats **stats_list)
3616 {
3617   struct DuplicateStats *pos;
3618   char *unix_domain_socket;
3619   unsigned long long port;
3620   char *to_match;
3621   if (GNUNET_YES !=
3622       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "testing",
3623                                             "single_statistics_per_host"))
3624     return GNUNET_NO;           /* Each peer has its own statistics instance, do nothing! */
3625
3626   pos = *stats_list;
3627   if (GNUNET_OK !=
3628       GNUNET_CONFIGURATION_get_value_string (specific_peer->cfg, "statistics",
3629                                              "unixpath", &unix_domain_socket))
3630     return GNUNET_NO;
3631
3632   if (GNUNET_OK !=
3633       GNUNET_CONFIGURATION_get_value_number (specific_peer->cfg, "statistics",
3634                                              "port", &port))
3635     {
3636       GNUNET_free(unix_domain_socket);
3637       return GNUNET_NO;
3638     }
3639
3640   if (specific_peer->daemon->hostname != NULL)
3641     GNUNET_asprintf (&to_match, "%s%s%llu", specific_peer->daemon->hostname,
3642                      unix_domain_socket, port);
3643   else
3644     GNUNET_asprintf (&to_match, "%s%llu", unix_domain_socket, port);
3645
3646   while (pos != NULL)
3647     {
3648       if (0 == strcmp (to_match, pos->unique_string))
3649         {
3650           GNUNET_free (unix_domain_socket);
3651           GNUNET_free (to_match);
3652           return GNUNET_YES;
3653         }
3654       pos = pos->next;
3655     }
3656   pos = GNUNET_malloc (sizeof (struct DuplicateStats));
3657   pos->unique_string = to_match;
3658   pos->next = *stats_list;
3659   *stats_list = pos;
3660   GNUNET_free (unix_domain_socket);
3661   return GNUNET_NO;
3662 }
3663
3664 /**
3665  * Iterate over all (running) peers in the peer group, retrieve
3666  * all statistics from each.
3667  */
3668 void
3669 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3670                                GNUNET_STATISTICS_Callback cont,
3671                                GNUNET_TESTING_STATISTICS_Iterator proc,
3672                                void *cls)
3673 {
3674   struct StatsIterateContext *stats_context;
3675   struct StatsCoreContext *core_ctx;
3676   unsigned int i;
3677   unsigned int total_count;
3678   struct DuplicateStats *stats_list;
3679   struct DuplicateStats *pos;
3680   stats_list = NULL;
3681
3682   /* Allocate a single stats iteration context */
3683   stats_context = GNUNET_malloc (sizeof (struct StatsIterateContext));
3684   stats_context->cont = cont;
3685   stats_context->proc = proc;
3686   stats_context->cls = cls;
3687   total_count = 0;
3688
3689   for (i = 0; i < pg->total; i++)
3690     {
3691       if ((pg->peers[i].daemon->running == GNUNET_YES)
3692           && (GNUNET_NO ==
3693               stats_check_existing (pg, &pg->peers[i], &stats_list)))
3694         {
3695           /* Allocate one core context per core we need to connect to */
3696           core_ctx = GNUNET_malloc (sizeof (struct StatsCoreContext));
3697           core_ctx->daemon = pg->peers[i].daemon;
3698           /* Set back pointer to topology iteration context */
3699           core_ctx->iter_context = stats_context;
3700           GNUNET_SCHEDULER_add_now (&schedule_get_statistics, core_ctx);
3701           total_count++;
3702         }
3703     }
3704
3705   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3706               "Retrieving stats from %u total instances.\n", total_count);
3707   stats_context->total = total_count;
3708   if (stats_list != NULL)
3709     {
3710       pos = stats_list;
3711       while (pos != NULL)
3712         {
3713           GNUNET_free (pos->unique_string);
3714           stats_list = pos->next;
3715           GNUNET_free (pos);
3716           pos = stats_list->next;
3717         }
3718     }
3719   return;
3720 }
3721
3722 /**
3723  * There are many ways to connect peers that are supported by this function.
3724  * To connect peers in the same topology that was created via the
3725  * GNUNET_TESTING_create_topology, the topology variable must be set to
3726  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3727  * a new instance of that topology will be generated and attempted to be
3728  * connected.  This could result in some connections being impossible,
3729  * because some topologies are non-deterministic.
3730  *
3731  * @param pg the peer group struct representing the running peers
3732  * @param topology which topology to connect the peers in
3733  * @param options options for connecting the topology
3734  * @param option_modifier modifier for options that take a parameter
3735  * @param notify_callback notification to be called once all connections completed
3736  * @param notify_cls closure for notification callback
3737  *
3738  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3739  */
3740 int
3741 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3742                                  enum GNUNET_TESTING_Topology topology,
3743                                  enum GNUNET_TESTING_TopologyOption options,
3744                                  double option_modifier,
3745                                  GNUNET_TESTING_NotifyCompletion
3746                                  notify_callback, void *notify_cls)
3747 {
3748   switch (topology)
3749     {
3750     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3751 #if VERBOSE_TOPOLOGY
3752       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3753                   _("Creating clique CONNECT topology\n"));
3754 #endif
3755       create_clique (pg, &add_actual_connections);
3756       break;
3757     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3758 #if VERBOSE_TOPOLOGY
3759       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3760                   _("Creating small world (ring) CONNECT topology\n"));
3761 #endif
3762       create_small_world_ring (pg, &add_actual_connections);
3763       break;
3764     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3765 #if VERBOSE_TOPOLOGY
3766       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3767                   _("Creating small world (2d-torus) CONNECT topology\n"));
3768 #endif
3769       create_small_world (pg, &add_actual_connections);
3770       break;
3771     case GNUNET_TESTING_TOPOLOGY_RING:
3772 #if VERBOSE_TOPOLOGY
3773       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3774                   _("Creating ring CONNECT topology\n"));
3775 #endif
3776       create_ring (pg, &add_actual_connections);
3777       break;
3778     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3779 #if VERBOSE_TOPOLOGY
3780       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3781                   _("Creating 2d torus CONNECT topology\n"));
3782 #endif
3783       create_2d_torus (pg, &add_actual_connections);
3784       break;
3785     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3786 #if VERBOSE_TOPOLOGY
3787       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3788                   _("Creating Erdos-Renyi CONNECT topology\n"));
3789 #endif
3790       create_erdos_renyi (pg, &add_actual_connections);
3791       break;
3792     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3793 #if VERBOSE_TOPOLOGY
3794       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3795                   _("Creating InterNAT CONNECT topology\n"));
3796 #endif
3797       create_nated_internet (pg, &add_actual_connections);
3798       break;
3799     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3800 #if VERBOSE_TOPOLOGY
3801       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3802                   _("Creating Scale Free CONNECT topology\n"));
3803 #endif
3804       create_scale_free (pg, &add_actual_connections);
3805       break;
3806     case GNUNET_TESTING_TOPOLOGY_LINE:
3807 #if VERBOSE_TOPOLOGY
3808       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3809                   _("Creating straight line CONNECT topology\n"));
3810 #endif
3811       create_line (pg, &add_actual_connections);
3812       break;
3813     case GNUNET_TESTING_TOPOLOGY_NONE:
3814 #if VERBOSE_TOPOLOGY
3815       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3816                   _("Creating no CONNECT topology\n"));
3817 #endif
3818       copy_allowed_topology (pg);
3819       break;
3820     default:
3821       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3822                   _
3823                   ("Unknown topology specification, can't connect peers!\n"));
3824       return GNUNET_SYSERR;
3825     }
3826
3827   switch (options)
3828     {
3829     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3830 #if VERBOSE_TOPOLOGY
3831       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3832                   _
3833                   ("Connecting random subset (%'.2f percent) of possible peers\n"),
3834                   100 * option_modifier);
3835 #endif
3836       choose_random_connections (pg, option_modifier);
3837       break;
3838     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3839 #if VERBOSE_TOPOLOGY
3840       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3841                   _("Connecting a minimum of %u peers each (if possible)\n"),
3842                   (unsigned int) option_modifier);
3843 #endif
3844       choose_minimum (pg, (unsigned int) option_modifier);
3845       break;
3846     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3847 #if VERBOSE_TOPOLOGY
3848       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3849                   _
3850                   ("Using DFS to connect a minimum of %u peers each (if possible)\n"),
3851                   (unsigned int) option_modifier);
3852 #endif
3853       perform_dfs (pg, (int) option_modifier);
3854       break;
3855     case GNUNET_TESTING_TOPOLOGY_OPTION_ADD_CLOSEST:
3856 #if VERBOSE_TOPOLOGY
3857       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3858                   _
3859                   ("Finding additional %u closest peers each (if possible)\n"),
3860                   (unsigned int) option_modifier);
3861 #endif
3862       add_closest (pg, (unsigned int) option_modifier,
3863                    &add_actual_connections);
3864       break;
3865     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3866       break;
3867     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3868       break;
3869     default:
3870       break;
3871     }
3872
3873   return connect_topology (pg, notify_callback, notify_cls);
3874 }
3875
3876 /**
3877  * Callback that is called whenever a hostkey is generated
3878  * for a peer.  Call the real callback and decrement the
3879  * starting counter for the peergroup.
3880  *
3881  * @param cls closure
3882  * @param id identifier for the daemon, NULL on error
3883  * @param d handle for the daemon
3884  * @param emsg error message (NULL on success)
3885  */
3886 static void
3887 internal_hostkey_callback (void *cls,
3888                            const struct GNUNET_PeerIdentity *id,
3889                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3890 {
3891   struct InternalStartContext *internal_context = cls;
3892   internal_context->peer->pg->starting--;
3893   internal_context->peer->pg->started++;
3894   if (internal_context->hostkey_callback != NULL)
3895     internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
3896                                         emsg);
3897   else if (internal_context->peer->pg->started ==
3898            internal_context->peer->pg->total)
3899     {
3900       internal_context->peer->pg->started = 0;  /* Internal startup may use this counter! */
3901       GNUNET_TESTING_daemons_continue_startup (internal_context->peer->pg);
3902     }
3903 }
3904
3905 /**
3906  * Callback that is called whenever a peer has finished starting.
3907  * Call the real callback and decrement the starting counter
3908  * for the peergroup.
3909  *
3910  * @param cls closure
3911  * @param id identifier for the daemon, NULL on error
3912  * @param d handle for the daemon
3913  * @param emsg error message (NULL on success)
3914  */
3915 static void
3916 internal_startup_callback (void *cls,
3917                            const struct GNUNET_PeerIdentity *id,
3918                            const struct GNUNET_CONFIGURATION_Handle *cfg,
3919                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3920 {
3921   struct InternalStartContext *internal_context = cls;
3922   internal_context->peer->pg->starting--;
3923   if (internal_context->start_cb != NULL)
3924     internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
3925                                 emsg);
3926 }
3927
3928 static void
3929 internal_continue_startup (void *cls,
3930                            const struct GNUNET_SCHEDULER_TaskContext *tc)
3931 {
3932   struct InternalStartContext *internal_context = cls;
3933
3934   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3935     {
3936       return;
3937     }
3938
3939   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3940     {
3941       internal_context->peer->pg->starting++;
3942       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3943     }
3944   else
3945     {
3946       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3947                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3948                                     &internal_continue_startup,
3949                                     internal_context);
3950     }
3951 }
3952
3953
3954 /**
3955  * Callback for informing us about a successful
3956  * or unsuccessful churn start call.
3957  *
3958  * @param cls a ChurnContext
3959  * @param id the peer identity of the started peer
3960  * @param cfg the handle to the configuration of the peer
3961  * @param d handle to the daemon for the peer
3962  * @param emsg NULL on success, non-NULL on failure
3963  *
3964  */
3965 void
3966 churn_start_callback (void *cls,
3967                       const struct GNUNET_PeerIdentity *id,
3968                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3969                       struct GNUNET_TESTING_Daemon *d, const char *emsg)
3970 {
3971   struct ChurnRestartContext *startup_ctx = cls;
3972   struct ChurnContext *churn_ctx = startup_ctx->churn_ctx;
3973
3974   unsigned int total_left;
3975   char *error_message;
3976
3977   error_message = NULL;
3978   if (emsg != NULL)
3979     {
3980       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3981                   "Churn stop callback failed with error `%s'\n", emsg);
3982       churn_ctx->num_failed_start++;
3983     }
3984   else
3985     {
3986       churn_ctx->num_to_start--;
3987     }
3988
3989 #if DEBUG_CHURN
3990   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3991               "Started peer, %d left.\n", churn_ctx->num_to_start);
3992 #endif
3993
3994   total_left =
3995     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
3996     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3997
3998   if (total_left == 0)
3999     {
4000       if ((churn_ctx->num_failed_stop > 0)
4001           || (churn_ctx->num_failed_start > 0))
4002         GNUNET_asprintf (&error_message,
4003                          "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4004                          churn_ctx->num_failed_start,
4005                          churn_ctx->num_failed_stop);
4006       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4007       GNUNET_free_non_null (error_message);
4008       GNUNET_free (churn_ctx);
4009       GNUNET_free (startup_ctx);
4010     }
4011 }
4012
4013
4014 static void
4015 schedule_churn_restart (void *cls,
4016                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4017 {
4018   struct PeerRestartContext *peer_restart_ctx = cls;
4019   struct ChurnRestartContext *startup_ctx =
4020     peer_restart_ctx->churn_restart_ctx;
4021
4022   if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
4023     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4024                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4025                                   &schedule_churn_restart, peer_restart_ctx);
4026   else
4027     {
4028       GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon,
4029                                            startup_ctx->timeout,
4030                                            &churn_start_callback,
4031                                            startup_ctx);
4032       GNUNET_free (peer_restart_ctx);
4033     }
4034 }
4035
4036 static void
4037 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
4038 {
4039   struct InternalStartContext *internal_context = cls;
4040
4041   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
4042     {
4043       return;
4044     }
4045
4046   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
4047     {
4048       internal_context->peer->pg->starting++;
4049       internal_context->peer->daemon =
4050         GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
4051                                      internal_context->timeout,
4052                                      internal_context->hostname,
4053                                      internal_context->username,
4054                                      internal_context->sshport,
4055                                      &internal_hostkey_callback,
4056                                      internal_context,
4057                                      &internal_startup_callback,
4058                                      internal_context);
4059     }
4060   else
4061     {
4062       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4063                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4064                                     &internal_start, internal_context);
4065     }
4066 }
4067
4068 /**
4069  * Function which continues a peer group starting up
4070  * after successfully generating hostkeys for each peer.
4071  *
4072  * @param pg the peer group to continue starting
4073  *
4074  */
4075 void
4076 GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
4077 {
4078   unsigned int i;
4079
4080   pg->starting = 0;
4081   for (i = 0; i < pg->total; i++)
4082     {
4083       GNUNET_SCHEDULER_add_now (&internal_continue_startup,
4084                                 &pg->peers[i].internal_context);
4085       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
4086     }
4087 }
4088
4089 /**
4090  * Start count gnunet instances with the same set of transports and
4091  * applications.  The port numbers (any option called "PORT") will be
4092  * adjusted to ensure that no two peers running on the same system
4093  * have the same port(s) in their respective configurations.
4094  *
4095  * @param cfg configuration template to use
4096  * @param total number of daemons to start
4097  * @param timeout total time allowed for peers to start
4098  * @param hostkey_callback function to call on each peers hostkey generation
4099  *        if NULL, peers will be started by this call, if non-null,
4100  *        GNUNET_TESTING_daemons_continue_startup must be called after
4101  *        successful hostkey generation
4102  * @param hostkey_cls closure for hostkey callback
4103  * @param cb function to call on each daemon that was started
4104  * @param cb_cls closure for cb
4105  * @param connect_callback function to call each time two hosts are connected
4106  * @param connect_callback_cls closure for connect_callback
4107  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
4108  *
4109  * @return NULL on error, otherwise handle to control peer group
4110  */
4111 struct GNUNET_TESTING_PeerGroup *
4112 GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
4113                               unsigned int total,
4114                               struct GNUNET_TIME_Relative timeout,
4115                               GNUNET_TESTING_NotifyHostkeyCreated
4116                               hostkey_callback, void *hostkey_cls,
4117                               GNUNET_TESTING_NotifyDaemonRunning cb,
4118                               void *cb_cls,
4119                               GNUNET_TESTING_NotifyConnection
4120                               connect_callback, void *connect_callback_cls,
4121                               const struct GNUNET_TESTING_Host *hostnames)
4122 {
4123   struct GNUNET_TESTING_PeerGroup *pg;
4124   const struct GNUNET_TESTING_Host *hostpos;
4125 #if 0
4126   char *pos;
4127   const char *rpos;
4128   char *start;
4129 #endif
4130   const char *hostname;
4131   const char *username;
4132   char *baseservicehome;
4133   char *newservicehome;
4134   char *tmpdir;
4135   struct GNUNET_CONFIGURATION_Handle *pcfg;
4136   unsigned int off;
4137   unsigned int hostcnt;
4138   uint16_t minport;
4139   uint16_t sshport;
4140   uint32_t upnum;
4141   uint32_t fdnum;
4142
4143   if (0 == total)
4144     {
4145       GNUNET_break (0);
4146       return NULL;
4147     }
4148   upnum = 0;
4149   fdnum = 0;
4150   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
4151   pg->cfg = cfg;
4152   pg->notify_connection = connect_callback;
4153   pg->notify_connection_cls = connect_callback_cls;
4154   pg->total = total;
4155   pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
4156   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
4157   if (NULL != hostnames)
4158     {
4159       off = 0;
4160       hostpos = hostnames;
4161       while (hostpos != NULL)
4162         {
4163           hostpos = hostpos->next;
4164           off++;
4165         }
4166       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4167       off = 0;
4168
4169       hostpos = hostnames;
4170       while (hostpos != NULL)
4171         {
4172           pg->hosts[off].minport = LOW_PORT;
4173           pg->hosts[off].hostname = GNUNET_strdup (hostpos->hostname);
4174           if (hostpos->username != NULL)
4175             pg->hosts[off].username = GNUNET_strdup (hostpos->username);
4176           pg->hosts[off].sshport = hostpos->port;
4177           hostpos = hostpos->next;
4178           off++;
4179         }
4180
4181       if (off == 0)
4182         {
4183           pg->hosts = NULL;
4184         }
4185       hostcnt = off;
4186       minport = 0;
4187       pg->num_hosts = off;
4188
4189 #if NO_LL
4190       off = 2;
4191       /* skip leading spaces */
4192       while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
4193         hostnames++;
4194       rpos = hostnames;
4195       while ('\0' != *rpos)
4196         {
4197           if (isspace ((unsigned char) *rpos))
4198             off++;
4199           rpos++;
4200         }
4201       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4202       off = 0;
4203       start = GNUNET_strdup (hostnames);
4204       pos = start;
4205       while ('\0' != *pos)
4206         {
4207           if (isspace ((unsigned char) *pos))
4208             {
4209               *pos = '\0';
4210               if (strlen (start) > 0)
4211                 {
4212                   pg->hosts[off].minport = LOW_PORT;
4213                   pg->hosts[off++].hostname = start;
4214                 }
4215               start = pos + 1;
4216             }
4217           pos++;
4218         }
4219       if (strlen (start) > 0)
4220         {
4221           pg->hosts[off].minport = LOW_PORT;
4222           pg->hosts[off++].hostname = start;
4223         }
4224       if (off == 0)
4225         {
4226           GNUNET_free (start);
4227           GNUNET_free (pg->hosts);
4228           pg->hosts = NULL;
4229         }
4230       hostcnt = off;
4231       minport = 0;              /* make gcc happy */
4232 #endif
4233     }
4234   else
4235     {
4236       hostcnt = 0;
4237       minport = LOW_PORT;
4238     }
4239   for (off = 0; off < total; off++)
4240     {
4241       if (hostcnt > 0)
4242         {
4243           hostname = pg->hosts[off % hostcnt].hostname;
4244           username = pg->hosts[off % hostcnt].username;
4245           sshport = pg->hosts[off % hostcnt].sshport;
4246           pcfg = make_config (cfg,
4247                               &pg->hosts[off % hostcnt].minport,
4248                               &upnum, hostname, &fdnum);
4249         }
4250       else
4251         {
4252           hostname = NULL;
4253           username = NULL;
4254           sshport = 0;
4255           pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
4256         }
4257
4258       if (NULL == pcfg)
4259         {
4260           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4261                       _
4262                       ("Could not create configuration for peer number %u on `%s'!\n"),
4263                       off, hostname == NULL ? "localhost" : hostname);
4264           continue;
4265         }
4266
4267       if (GNUNET_YES ==
4268           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
4269                                                  &baseservicehome))
4270         {
4271           GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
4272           GNUNET_free (baseservicehome);
4273         }
4274       else
4275         {
4276           tmpdir = getenv ("TMPDIR");
4277           tmpdir = tmpdir ? tmpdir : "/tmp";
4278           GNUNET_asprintf (&newservicehome,
4279                            "%s/%s/%d/",
4280                            tmpdir, "gnunet-testing-test-test", off);
4281         }
4282       GNUNET_CONFIGURATION_set_value_string (pcfg,
4283                                              "PATHS",
4284                                              "SERVICEHOME", newservicehome);
4285       GNUNET_free (newservicehome);
4286       pg->peers[off].cfg = pcfg;
4287       pg->peers[off].allowed_peers =
4288         GNUNET_CONTAINER_multihashmap_create (total);
4289       pg->peers[off].connect_peers =
4290         GNUNET_CONTAINER_multihashmap_create (total);
4291       pg->peers[off].blacklisted_peers =
4292         GNUNET_CONTAINER_multihashmap_create (total);
4293       pg->peers[off].pg = pg;
4294
4295       pg->peers[off].internal_context.peer = &pg->peers[off];
4296       pg->peers[off].internal_context.timeout = timeout;
4297       pg->peers[off].internal_context.hostname = hostname;
4298       pg->peers[off].internal_context.username = username;
4299       pg->peers[off].internal_context.sshport = sshport;
4300       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
4301       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
4302       pg->peers[off].internal_context.start_cb = cb;
4303       pg->peers[off].internal_context.start_cb_cls = cb_cls;
4304
4305       GNUNET_SCHEDULER_add_now (&internal_start,
4306                                 &pg->peers[off].internal_context);
4307
4308     }
4309   return pg;
4310 }
4311
4312 /*
4313  * Get a daemon by number, so callers don't have to do nasty
4314  * offsetting operation.
4315  */
4316 struct GNUNET_TESTING_Daemon *
4317 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg,
4318                            unsigned int position)
4319 {
4320   if (position < pg->total)
4321     return pg->peers[position].daemon;
4322   else
4323     return NULL;
4324 }
4325
4326 /*
4327  * Get a daemon by peer identity, so callers can
4328  * retrieve the daemon without knowing it's offset.
4329  *
4330  * @param pg the peer group to retrieve the daemon from
4331  * @param peer_id the peer identity of the daemon to retrieve
4332  *
4333  * @return the daemon on success, or NULL if no such peer identity is found
4334  */
4335 struct GNUNET_TESTING_Daemon *
4336 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg,
4337                                  struct GNUNET_PeerIdentity *peer_id)
4338 {
4339   unsigned int i;
4340
4341   for (i = 0; i < pg->total; i++)
4342     {
4343       if (0 ==
4344           memcmp (&pg->peers[i].daemon->id, peer_id,
4345                   sizeof (struct GNUNET_PeerIdentity)))
4346         return pg->peers[i].daemon;
4347     }
4348
4349   return NULL;
4350 }
4351
4352 /**
4353  * Prototype of a function that will be called when a
4354  * particular operation was completed the testing library.
4355  *
4356  * @param cls closure (a struct RestartContext)
4357  * @param id id of the peer that was restarted
4358  * @param cfg handle to the configuration of the peer
4359  * @param d handle to the daemon that was restarted
4360  * @param emsg NULL on success
4361  */
4362 void
4363 restart_callback (void *cls,
4364                   const struct GNUNET_PeerIdentity *id,
4365                   const struct GNUNET_CONFIGURATION_Handle *cfg,
4366                   struct GNUNET_TESTING_Daemon *d, const char *emsg)
4367 {
4368   struct RestartContext *restart_context = cls;
4369
4370   if (emsg == NULL)
4371     {
4372       restart_context->peers_restarted++;
4373     }
4374   else
4375     {
4376       restart_context->peers_restart_failed++;
4377     }
4378
4379   if (restart_context->peers_restarted == restart_context->peer_group->total)
4380     {
4381       restart_context->callback (restart_context->callback_cls, NULL);
4382       GNUNET_free (restart_context);
4383     }
4384   else if (restart_context->peers_restart_failed +
4385            restart_context->peers_restarted ==
4386            restart_context->peer_group->total)
4387     {
4388       restart_context->callback (restart_context->callback_cls,
4389                                  "Failed to restart peers!");
4390       GNUNET_free (restart_context);
4391     }
4392
4393 }
4394
4395 /**
4396  * Callback for informing us about a successful
4397  * or unsuccessful churn stop call.
4398  *
4399  * @param cls a ChurnContext
4400  * @param emsg NULL on success, non-NULL on failure
4401  *
4402  */
4403 void
4404 churn_stop_callback (void *cls, const char *emsg)
4405 {
4406   struct ShutdownContext *shutdown_ctx = cls;
4407   struct ChurnContext *churn_ctx = shutdown_ctx->cb_cls;
4408   unsigned int total_left;
4409   char *error_message;
4410
4411   error_message = NULL;
4412   shutdown_ctx->outstanding--;
4413
4414   if (emsg != NULL)
4415     {
4416       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4417                   "Churn stop callback failed with error `%s'\n", emsg);
4418       churn_ctx->num_failed_stop++;
4419     }
4420   else
4421     {
4422       churn_ctx->num_to_stop--;
4423     }
4424
4425 #if DEBUG_CHURN
4426   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4427               "Stopped peer, %d left.\n", churn_ctx->num_to_stop);
4428 #endif
4429   total_left =
4430     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
4431     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
4432
4433   if (total_left == 0)
4434     {
4435       if ((churn_ctx->num_failed_stop > 0)
4436           || (churn_ctx->num_failed_start > 0))
4437         {
4438           GNUNET_asprintf (&error_message,
4439                            "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4440                            churn_ctx->num_failed_start,
4441                            churn_ctx->num_failed_stop);
4442         }
4443       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4444       GNUNET_free_non_null (error_message);
4445       GNUNET_free (churn_ctx);
4446       GNUNET_free (shutdown_ctx);
4447     }
4448 }
4449
4450 /**
4451  * Count the number of running peers.
4452  *
4453  * @param pg handle for the peer group
4454  *
4455  * @return the number of currently running peers in the peer group
4456  */
4457 unsigned int
4458 GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
4459 {
4460   unsigned int i;
4461   unsigned int running = 0;
4462   for (i = 0; i < pg->total; i++)
4463     {
4464       if (pg->peers[i].daemon->running == GNUNET_YES)
4465         {
4466           GNUNET_assert (running != -1);
4467           running++;
4468         }
4469     }
4470   return running;
4471 }
4472
4473 /**
4474  * Task to rate limit the number of outstanding peer shutdown
4475  * requests.  This is necessary for making sure we don't do
4476  * too many ssh connections at once, but is generally nicer
4477  * to any system as well (graduated task starts, as opposed
4478  * to calling gnunet-arm N times all at once).
4479  */
4480 static void
4481 schedule_churn_shutdown_task (void *cls,
4482                               const struct GNUNET_SCHEDULER_TaskContext *tc)
4483 {
4484   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4485   struct ShutdownContext *shutdown_ctx;
4486
4487   GNUNET_assert (peer_shutdown_ctx != NULL);
4488   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4489   GNUNET_assert (shutdown_ctx != NULL);
4490
4491   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4492     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4493                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4494                                   &schedule_churn_shutdown_task,
4495                                   peer_shutdown_ctx);
4496   else
4497     {
4498       shutdown_ctx->outstanding++;
4499       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4500                                   shutdown_ctx->timeout, shutdown_ctx->cb,
4501                                   shutdown_ctx, GNUNET_NO, GNUNET_YES);
4502       GNUNET_free (peer_shutdown_ctx);
4503     }
4504 }
4505
4506 /**
4507  * Simulate churn by stopping some peers (and possibly
4508  * re-starting others if churn is called multiple times).  This
4509  * function can only be used to create leave-join churn (peers "never"
4510  * leave for good).  First "voff" random peers that are currently
4511  * online will be taken offline; then "von" random peers that are then
4512  * offline will be put back online.  No notifications will be
4513  * generated for any of these operations except for the callback upon
4514  * completion.
4515  *
4516  * @param pg handle for the peer group
4517  * @param voff number of peers that should go offline
4518  * @param von number of peers that should come back online;
4519  *            must be zero on first call (since "testbed_start"
4520  *            always starts all of the peers)
4521  * @param timeout how long to wait for operations to finish before
4522  *        giving up
4523  * @param cb function to call at the end
4524  * @param cb_cls closure for cb
4525  */
4526 void
4527 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
4528                               unsigned int voff,
4529                               unsigned int von,
4530                               struct GNUNET_TIME_Relative timeout,
4531                               GNUNET_TESTING_NotifyCompletion cb,
4532                               void *cb_cls)
4533 {
4534   struct ChurnContext *churn_ctx;
4535   struct ShutdownContext *shutdown_ctx;
4536   struct PeerShutdownContext *peer_shutdown_ctx;
4537   struct PeerRestartContext *peer_restart_ctx;
4538   struct ChurnRestartContext *churn_startup_ctx;
4539
4540   unsigned int running;
4541   unsigned int stopped;
4542   unsigned int total_running;
4543   unsigned int total_stopped;
4544   unsigned int i;
4545   unsigned int *running_arr;
4546   unsigned int *stopped_arr;
4547   unsigned int *running_permute;
4548   unsigned int *stopped_permute;
4549
4550   running = 0;
4551   stopped = 0;
4552
4553   if ((von == 0) && (voff == 0))        /* No peers at all? */
4554     {
4555       cb (cb_cls, NULL);
4556       return;
4557     }
4558
4559   for (i = 0; i < pg->total; i++)
4560     {
4561       if (pg->peers[i].daemon->running == GNUNET_YES)
4562         {
4563           GNUNET_assert (running != -1);
4564           running++;
4565         }
4566       else
4567         {
4568           GNUNET_assert (stopped != -1);
4569           stopped++;
4570         }
4571     }
4572
4573   if (voff > running)
4574     {
4575       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4576                   "Trying to stop more peers than are currently running!\n");
4577       cb (cb_cls, "Trying to stop more peers than are currently running!");
4578       return;
4579     }
4580
4581   if (von > stopped)
4582     {
4583       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4584                   "Trying to start more peers than are currently stopped!\n");
4585       cb (cb_cls, "Trying to start more peers than are currently stopped!");
4586       return;
4587     }
4588
4589   churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4590
4591   running_arr = NULL;
4592   if (running > 0)
4593     running_arr = GNUNET_malloc (running * sizeof (unsigned int));
4594
4595   stopped_arr = NULL;
4596   if (stopped > 0)
4597     stopped_arr = GNUNET_malloc (stopped * sizeof (unsigned int));
4598
4599   running_permute = NULL;
4600   stopped_permute = NULL;
4601
4602   if (running > 0)
4603     running_permute =
4604       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, running);
4605   if (stopped > 0)
4606     stopped_permute =
4607       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, stopped);
4608
4609   total_running = running;
4610   total_stopped = stopped;
4611   running = 0;
4612   stopped = 0;
4613
4614   churn_ctx->num_to_start = von;
4615   churn_ctx->num_to_stop = voff;
4616   churn_ctx->cb = cb;
4617   churn_ctx->cb_cls = cb_cls;
4618
4619   for (i = 0; i < pg->total; i++)
4620     {
4621       if (pg->peers[i].daemon->running == GNUNET_YES)
4622         {
4623           GNUNET_assert ((running_arr != NULL) && (total_running > running));
4624           running_arr[running] = i;
4625           running++;
4626         }
4627       else
4628         {
4629           GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped));
4630           stopped_arr[stopped] = i;
4631           stopped++;
4632         }
4633     }
4634
4635   GNUNET_assert (running >= voff);
4636   if (voff > 0)
4637     {
4638       shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4639       shutdown_ctx->cb = &churn_stop_callback;
4640       shutdown_ctx->cb_cls = churn_ctx;
4641       shutdown_ctx->total_peers = voff;
4642       shutdown_ctx->timeout = timeout;
4643     }
4644
4645   for (i = 0; i < voff; i++)
4646     {
4647 #if DEBUG_CHURN
4648       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n",
4649                   running_permute[i]);
4650 #endif
4651       GNUNET_assert (running_arr != NULL);
4652       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4653       peer_shutdown_ctx->daemon =
4654         pg->peers[running_arr[running_permute[i]]].daemon;
4655       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4656       GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task,
4657                                 peer_shutdown_ctx);
4658
4659       /*
4660          GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
4661          timeout,
4662          &churn_stop_callback, churn_ctx,
4663          GNUNET_NO, GNUNET_YES); */
4664     }
4665
4666   GNUNET_assert (stopped >= von);
4667   if (von > 0)
4668     {
4669       churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4670       churn_startup_ctx->churn_ctx = churn_ctx;
4671       churn_startup_ctx->timeout = timeout;
4672     }
4673   for (i = 0; i < von; i++)
4674     {
4675 #if DEBUG_CHURN
4676       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n",
4677                   stopped_permute[i]);
4678 #endif
4679       GNUNET_assert (stopped_arr != NULL);
4680       peer_restart_ctx = GNUNET_malloc (sizeof (struct PeerRestartContext));
4681       peer_restart_ctx->churn_restart_ctx = churn_startup_ctx;
4682       peer_restart_ctx->daemon =
4683         pg->peers[stopped_arr[stopped_permute[i]]].daemon;
4684       GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx);
4685       /*
4686          GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
4687          timeout, &churn_start_callback, churn_ctx); */
4688     }
4689
4690   GNUNET_free_non_null (running_arr);
4691   GNUNET_free_non_null (stopped_arr);
4692   GNUNET_free_non_null (running_permute);
4693   GNUNET_free_non_null (stopped_permute);
4694 }
4695
4696
4697 /**
4698  * Restart all peers in the given group.
4699  *
4700  * @param pg the handle to the peer group
4701  * @param callback function to call on completion (or failure)
4702  * @param callback_cls closure for the callback function
4703  */
4704 void
4705 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
4706                                 GNUNET_TESTING_NotifyCompletion callback,
4707                                 void *callback_cls)
4708 {
4709   struct RestartContext *restart_context;
4710   unsigned int off;
4711
4712   if (pg->total > 0)
4713     {
4714       restart_context = GNUNET_malloc (sizeof (struct RestartContext));
4715       restart_context->peer_group = pg;
4716       restart_context->peers_restarted = 0;
4717       restart_context->callback = callback;
4718       restart_context->callback_cls = callback_cls;
4719
4720       for (off = 0; off < pg->total; off++)
4721         {
4722           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon,
4723                                          &restart_callback, restart_context);
4724         }
4725     }
4726 }
4727
4728 /**
4729  * Start or stop an individual peer from the given group.
4730  *
4731  * @param pg handle to the peer group
4732  * @param offset which peer to start or stop
4733  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
4734  * @param timeout how long to wait for shutdown
4735  * @param cb function to call at the end
4736  * @param cb_cls closure for cb
4737  */
4738 void
4739 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
4740                              unsigned int offset,
4741                              int desired_status,
4742                              struct GNUNET_TIME_Relative timeout,
4743                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4744 {
4745   struct ShutdownContext *shutdown_ctx;
4746   struct ChurnRestartContext *startup_ctx;
4747   struct ChurnContext *churn_ctx;
4748
4749   if (GNUNET_NO == desired_status)
4750     {
4751       if (NULL != pg->peers[offset].daemon)
4752         {
4753           shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4754           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4755           churn_ctx->num_to_start = 0;
4756           churn_ctx->num_to_stop = 1;
4757           churn_ctx->cb = cb;
4758           churn_ctx->cb_cls = cb_cls;
4759           shutdown_ctx->cb_cls = churn_ctx;
4760           GNUNET_TESTING_daemon_stop (pg->peers[offset].daemon,
4761                                       timeout, &churn_stop_callback,
4762                                       shutdown_ctx, GNUNET_NO, GNUNET_YES);
4763         }
4764     }
4765   else if (GNUNET_YES == desired_status)
4766     {
4767       if (NULL == pg->peers[offset].daemon)
4768         {
4769           startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4770           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4771           churn_ctx->num_to_start = 1;
4772           churn_ctx->num_to_stop = 0;
4773           churn_ctx->cb = cb;
4774           churn_ctx->cb_cls = cb_cls;
4775           startup_ctx->churn_ctx = churn_ctx;
4776           GNUNET_TESTING_daemon_start_stopped (pg->peers[offset].daemon,
4777                                                timeout, &churn_start_callback,
4778                                                startup_ctx);
4779         }
4780     }
4781   else
4782     GNUNET_break (0);
4783 }
4784
4785
4786 /**
4787  * Callback for shutting down peers in a peer group.
4788  *
4789  * @param cls closure (struct ShutdownContext)
4790  * @param emsg NULL on success
4791  */
4792 void
4793 internal_shutdown_callback (void *cls, const char *emsg)
4794 {
4795   struct ShutdownContext *shutdown_ctx = cls;
4796
4797   shutdown_ctx->outstanding--;
4798   if (emsg == NULL)
4799     {
4800       shutdown_ctx->peers_down++;
4801     }
4802   else
4803     {
4804       shutdown_ctx->peers_failed++;
4805     }
4806
4807   if ((shutdown_ctx->cb != NULL)
4808       && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed ==
4809           shutdown_ctx->total_peers))
4810     {
4811       if (shutdown_ctx->peers_failed > 0)
4812         shutdown_ctx->cb (shutdown_ctx->cb_cls,
4813                           "Not all peers successfully shut down!");
4814       else
4815         shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
4816       GNUNET_free (shutdown_ctx);
4817     }
4818 }
4819
4820
4821 /**
4822  * Task to rate limit the number of outstanding peer shutdown
4823  * requests.  This is necessary for making sure we don't do
4824  * too many ssh connections at once, but is generally nicer
4825  * to any system as well (graduated task starts, as opposed
4826  * to calling gnunet-arm N times all at once).
4827  */
4828 static void
4829 schedule_shutdown_task (void *cls,
4830                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4831 {
4832   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4833   struct ShutdownContext *shutdown_ctx;
4834
4835   GNUNET_assert (peer_shutdown_ctx != NULL);
4836   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4837   GNUNET_assert (shutdown_ctx != NULL);
4838
4839   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4840     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4841                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4842                                   &schedule_shutdown_task, peer_shutdown_ctx);
4843   else
4844     {
4845       shutdown_ctx->outstanding++;
4846       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4847                                   shutdown_ctx->timeout,
4848                                   &internal_shutdown_callback, shutdown_ctx,
4849                                   GNUNET_YES, GNUNET_NO);
4850       GNUNET_free (peer_shutdown_ctx);
4851     }
4852 }
4853
4854 /**
4855  * Shutdown all peers started in the given group.
4856  *
4857  * @param pg handle to the peer group
4858  * @param timeout how long to wait for shutdown
4859  * @param cb callback to notify upon success or failure
4860  * @param cb_cls closure for cb
4861  */
4862 void
4863 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4864                              struct GNUNET_TIME_Relative timeout,
4865                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4866 {
4867   unsigned int off;
4868   struct ShutdownContext *shutdown_ctx;
4869   struct PeerShutdownContext *peer_shutdown_ctx;
4870
4871   GNUNET_assert (pg->total > 0);
4872
4873   shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4874   shutdown_ctx->cb = cb;
4875   shutdown_ctx->cb_cls = cb_cls;
4876   shutdown_ctx->total_peers = pg->total;
4877   shutdown_ctx->timeout = timeout;
4878   /* shtudown_ctx->outstanding = 0; */
4879
4880   for (off = 0; off < pg->total; off++)
4881     {
4882       GNUNET_assert (NULL != pg->peers[off].daemon);
4883       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4884       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4885       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4886       GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
4887       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4888       if (NULL != pg->peers[off].cfg)
4889         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4890       if (pg->peers[off].allowed_peers != NULL)
4891         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
4892       if (pg->peers[off].connect_peers != NULL)
4893         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
4894       if (pg->peers[off].blacklisted_peers != NULL)
4895         GNUNET_CONTAINER_multihashmap_destroy (pg->
4896                                                peers[off].blacklisted_peers);
4897     }
4898   GNUNET_free (pg->peers);
4899   for (off = 0; off < pg->num_hosts; off++)
4900     {
4901       GNUNET_free (pg->hosts[off].hostname);
4902       GNUNET_free_non_null (pg->hosts[off].username);
4903     }
4904   GNUNET_free_non_null (pg->hosts);
4905   GNUNET_free (pg);
4906 }
4907
4908
4909 /* end of testing_group.c */