src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define VERBOSE_TOPOLOGY GNUNET_YES
  36
  37 #define DEBUG_CHURN GNUNET_NO
  38
  39 /**
  40  * Lowest port used for GNUnet testing.  Should be high enough to not
  41  * conflict with other applications running on the hosts but be low
  42  * enough to not conflict with client-ports (typically starting around
  43  * 32k).
  44  */
  45 #define LOW_PORT 10000
  46
  47 /**
  48  * Highest port used for GNUnet testing.  Should be low enough to not
  49  * conflict with the port range for "local" ports (client apps; see
  50  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  51  */
  52 #define HIGH_PORT 56000
  53
  54 #define MAX_OUTSTANDING_CONNECTIONS 40
  55
  56 #define MAX_CONCURRENT_HOSTKEYS 10
  57
  58 #define MAX_CONCURRENT_STARTING 10
  59
  60 #define MAX_CONCURRENT_SHUTDOWN 10
  61
  62 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 200)
  63
  64 #define CONNECT_ATTEMPTS 21
  65
  66 /**
  67  * Prototype of a function called whenever two peers would be connected
  68  * in a certain topology.
  69  */
  70 typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
  71                                                             GNUNET_TESTING_PeerGroup
  72                                                             * pg,
  73                                                             unsigned int
  74                                                             first,
  75                                                             unsigned int
  76                                                             second);
  77
  78
  79 /**
  80  * Context for handling churning a peer group
  81  */
  82 struct ChurnContext
  83 {
  84   /**
  85    * Callback used to notify of churning finished
  86    */
  87   GNUNET_TESTING_NotifyCompletion cb;
  88
  89   /**
  90    * Closure for callback
  91    */
  92   void *cb_cls;
  93
  94   /**
  95    * Number of peers that still need to be started
  96    */
  97   unsigned int num_to_start;
  98
  99   /**
 100    * Number of peers that still need to be stopped
 101    */
 102   unsigned int num_to_stop;
 103
 104   /**
 105    * Number of peers that failed to start
 106    */
 107   unsigned int num_failed_start;
 108
 109   /**
 110    * Number of peers that failed to stop
 111    */
 112   unsigned int num_failed_stop;
 113 };
 114
 115 struct RestartContext
 116 {
 117   /**
 118    * The group of peers being restarted
 119    */
 120   struct GNUNET_TESTING_PeerGroup *peer_group;
 121
 122   /**
 123    * How many peers have been restarted thus far
 124    */
 125   unsigned int peers_restarted;
 126
 127   /**
 128    * How many peers got an error when restarting
 129    */
 130   unsigned int peers_restart_failed;
 131
 132   /**
 133    * The function to call once all peers have been restarted
 134    */
 135   GNUNET_TESTING_NotifyCompletion callback;
 136
 137   /**
 138    * Closure for callback function
 139    */
 140   void *callback_cls;
 141
 142 };
 143
 144
 145 struct ShutdownContext
 146 {
 147   /**
 148    * Total peers to wait for
 149    */
 150   unsigned int total_peers;
 151
 152   /**
 153    * Number of peers successfully shut down
 154    */
 155   unsigned int peers_down;
 156
 157   /**
 158    * Number of peers failed to shut down
 159    */
 160   unsigned int peers_failed;
 161
 162   /**
 163    * Number of peers we have started shutting
 164    * down.  If too many, wait on them.
 165    */
 166   unsigned int outstanding;
 167
 168   /**
 169    * Timeout for shutdown.
 170    */
 171   struct GNUNET_TIME_Relative timeout;
 172
 173   /**
 174    * Callback to call when all peers either
 175    * shutdown or failed to shutdown
 176    */
 177   GNUNET_TESTING_NotifyCompletion cb;
 178
 179   /**
 180    * Closure for cb
 181    */
 182   void *cb_cls;
 183 };
 184
 185 /**
 186  * Individual shutdown context for a particular peer.
 187  */
 188 struct PeerShutdownContext
 189 {
 190   /**
 191    * Pointer to the high level shutdown context.
 192    */
 193   struct ShutdownContext *shutdown_ctx;
 194
 195   /**
 196    * The daemon handle for the peer to shut down.
 197    */
 198   struct GNUNET_TESTING_Daemon *daemon;
 199 };
 200
 201 /**
 202  * Individual shutdown context for a particular peer.
 203  */
 204 struct PeerRestartContext
 205 {
 206   /**
 207    * Pointer to the high level restart context.
 208    */
 209   struct ChurnRestartContext *churn_restart_ctx;
 210
 211   /**
 212    * The daemon handle for the peer to shut down.
 213    */
 214   struct GNUNET_TESTING_Daemon *daemon;
 215 };
 216
 217
 218 struct CreateTopologyContext
 219 {
 220
 221   /**
 222    * Function to call with number of connections
 223    */
 224   GNUNET_TESTING_NotifyConnections cont;
 225
 226   /**
 227    * Closure for connection notification
 228    */
 229   void *cls;
 230 };
 231
 232 #if OLD
 233 struct PeerConnection
 234 {
 235   /*
 236    * Linked list
 237    */
 238   struct PeerConnection *next;
 239
 240   /*
 241    * Pointer to daemon handle
 242    */
 243   struct GNUNET_TESTING_Daemon *daemon;
 244
 245 };
 246 #endif
 247
 248 struct InternalStartContext
 249 {
 250   /**
 251    * Pointer to peerdata
 252    */
 253   struct PeerData *peer;
 254
 255   /**
 256    * Timeout for peer startup
 257    */
 258   struct GNUNET_TIME_Relative timeout;
 259
 260   /**
 261    * Client callback for hostkey notification
 262    */
 263   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 264
 265   /**
 266    * Closure for hostkey_callback
 267    */
 268   void *hostkey_cls;
 269
 270   /**
 271    * Client callback for peer start notification
 272    */
 273   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 274
 275   /**
 276    * Closure for cb
 277    */
 278   void *start_cb_cls;
 279
 280   /**
 281    * Hostname, where to start the peer
 282    */
 283   const char *hostname;
 284
 285   /**
 286    * Username to use when connecting to the
 287    * host via ssh.
 288    */
 289   const char *username;
 290
 291   /**
 292    * Port to use for ssh.
 293    */
 294   uint16_t sshport;
 295
 296 };
 297
 298 struct ChurnRestartContext
 299 {
 300   /**
 301    * Number of restarts currently in flight.
 302    */
 303   unsigned int outstanding;
 304
 305   /**
 306    * Handle to the underlying churn context.
 307    */
 308   struct ChurnContext *churn_ctx;
 309
 310   /**
 311    * How long to allow the operation to take.
 312    */
 313   struct GNUNET_TIME_Relative timeout;
 314 };
 315
 316 /**
 317  * Data we keep per peer.
 318  */
 319 struct PeerData
 320 {
 321   /**
 322    * (Initial) configuration of the host.
 323    * (initial because clients could change
 324    *  it and we would not know about those
 325    *  updates).
 326    */
 327   struct GNUNET_CONFIGURATION_Handle *cfg;
 328
 329   /**
 330    * Handle for controlling the daemon.
 331    */
 332   struct GNUNET_TESTING_Daemon *daemon;
 333
 334   /**
 335    * The peergroup this peer belongs to.
 336    */
 337   struct GNUNET_TESTING_PeerGroup *pg;
 338
 339   /**
 340    * Hash map of allowed peer connections (F2F created topology)
 341    */
 342   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 343
 344   /**
 345    * Hash map of blacklisted peers
 346    */
 347   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 348
 349   /**
 350    * Hash map of peer connections
 351    */
 352   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 353
 354   /**
 355    * Temporary hash map of peer connections
 356    */
 357   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 358
 359   /**
 360    * Temporary variable for topology creation, should be reset before
 361    * creating any topology so the count is valid once finished.
 362    */
 363   int num_connections;
 364
 365   /**
 366    * Context to keep track of peers being started, to
 367    * stagger hostkey generation and peer startup.
 368    */
 369   struct InternalStartContext internal_context;
 370 };
 371
 372
 373 /**
 374  * Linked list of per-host data.
 375  */
 376 struct HostData
 377 {
 378   /**
 379    * Name of the host.
 380    */
 381   char *hostname;
 382
 383   /**
 384    * SSH username to use when connecting to this host.
 385    */
 386   char *username;
 387
 388   /**
 389    * SSH port to use when connecting to this host.
 390    */
 391   uint16_t sshport;
 392
 393   /**
 394    * Lowest port that we have not yet used
 395    * for GNUnet.
 396    */
 397   uint16_t minport;
 398 };
 399
 400 struct TopologyIterateContext
 401 {
 402   /**
 403    * Callback for notifying of two connected peers.
 404    */
 405   GNUNET_TESTING_NotifyTopology topology_cb;
 406
 407   /**
 408    * Closure for topology_cb
 409    */
 410   void *cls;
 411
 412   /**
 413    * Number of peers currently connected to.
 414    */
 415   unsigned int connected;
 416
 417   /**
 418    * Number of peers we have finished iterating.
 419    */
 420   unsigned int completed;
 421
 422   /**
 423    * Number of peers total.
 424    */
 425   unsigned int total;
 426 };
 427
 428 struct StatsIterateContext
 429 {
 430   /**
 431    * Continuation to call once all stats information has been retrieved.
 432    */
 433   GNUNET_STATISTICS_Callback cont;
 434
 435   /**
 436    * Proc function to call on each value received.
 437    */
 438   GNUNET_TESTING_STATISTICS_Iterator proc;
 439
 440   /**
 441    * Closure for topology_cb
 442    */
 443   void *cls;
 444
 445   /**
 446    * Number of peers currently connected to.
 447    */
 448   unsigned int connected;
 449
 450   /**
 451    * Number of peers we have finished iterating.
 452    */
 453   unsigned int completed;
 454
 455   /**
 456    * Number of peers total.
 457    */
 458   unsigned int total;
 459 };
 460
 461 struct CoreContext
 462 {
 463   void *iter_context;
 464   struct GNUNET_TESTING_Daemon *daemon;
 465 };
 466
 467 struct StatsCoreContext
 468 {
 469   void *iter_context;
 470   struct GNUNET_TESTING_Daemon *daemon;
 471   /**
 472    * Handle to the statistics service.
 473    */
 474   struct GNUNET_STATISTICS_Handle *stats_handle;
 475
 476   /**
 477    * Handle for getting statistics.
 478    */
 479   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 480 };
 481
 482 /**
 483  * Handle to a group of GNUnet peers.
 484  */
 485 struct GNUNET_TESTING_PeerGroup
 486 {
 487   /**
 488    * Configuration template.
 489    */
 490   const struct GNUNET_CONFIGURATION_Handle *cfg;
 491
 492   /**
 493    * Function to call on each started daemon.
 494    */
 495   //GNUNET_TESTING_NotifyDaemonRunning cb;
 496
 497   /**
 498    * Closure for cb.
 499    */
 500   //void *cb_cls;
 501
 502   /*
 503    * Function to call on each topology connection created
 504    */
 505   GNUNET_TESTING_NotifyConnection notify_connection;
 506
 507   /*
 508    * Callback for notify_connection
 509    */
 510   void *notify_connection_cls;
 511
 512   /**
 513    * Array of information about hosts.
 514    */
 515   struct HostData *hosts;
 516
 517   /**
 518    * Number of hosts (size of HostData)
 519    */
 520   unsigned int num_hosts;
 521
 522   /**
 523    * Array of "total" peers.
 524    */
 525   struct PeerData *peers;
 526
 527   /**
 528    * Number of peers in this group.
 529    */
 530   unsigned int total;
 531
 532   /**
 533    * At what time should we fail the peer startup process?
 534    */
 535   struct GNUNET_TIME_Absolute max_timeout;
 536
 537   /**
 538    * How many peers are being started right now?
 539    */
 540   unsigned int starting;
 541
 542   /**
 543    * How many peers have already been started?
 544    */
 545   unsigned int started;
 546 };
 547
 548 struct UpdateContext
 549 {
 550   struct GNUNET_CONFIGURATION_Handle *ret;
 551   const struct GNUNET_CONFIGURATION_Handle *orig;
 552   const char *hostname;
 553   unsigned int nport;
 554   unsigned int upnum;
 555   unsigned int fdnum;
 556 };
 557
 558 struct ConnectTopologyContext
 559 {
 560   /**
 561    * How many connections are left to create.
 562    */
 563   unsigned int remaining_connections;
 564
 565   /**
 566    * Handle to group of peers.
 567    */
 568   struct GNUNET_TESTING_PeerGroup *pg;
 569
 570   /**
 571    * Temp value set for each iteration.
 572    */
 573   struct PeerData *first;
 574
 575   /**
 576    * Notification that all peers are connected.
 577    */
 578   GNUNET_TESTING_NotifyCompletion notify_connections_done;
 579
 580   /**
 581    * Closure for notify.
 582    */
 583   void *notify_cls;
 584 };
 585
 586 struct ConnectContext
 587 {
 588   /**
 589    * Peer to connect second to.
 590    */
 591   struct GNUNET_TESTING_Daemon *first;
 592
 593   /**
 594    * Peer to connect first to.
 595    */
 596   struct GNUNET_TESTING_Daemon *second;
 597
 598   /**
 599    * Higher level topology connection context.
 600    */
 601   struct ConnectTopologyContext *ct_ctx;
 602 };
 603
 604 /**
 605  * Convert unique ID to hash code.
 606  *
 607  * @param uid unique ID to convert
 608  * @param hash set to uid (extended with zeros)
 609  */
 610 static void
 611 hash_from_uid (uint32_t uid, GNUNET_HashCode * hash)
 612 {
 613   memset (hash, 0, sizeof (GNUNET_HashCode));
 614   *((uint32_t *) hash) = uid;
 615 }
 616
 617 /**
 618  * Convert hash code to unique ID.
 619  *
 620  * @param uid unique ID to convert
 621  * @param hash set to uid (extended with zeros)
 622  */
 623 static void
 624 uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 625 {
 626   memcpy (uid, hash, sizeof (uint32_t));
 627 }
 628
 629 /**
 630  * Number of connects we are waiting on, allows us to rate limit
 631  * connect attempts.
 632  */
 633 static int outstanding_connects;
 634
 635 /**
 636  * Get a topology from a string input.
 637  *
 638  * @param topology where to write the retrieved topology
 639  * @param topology_string The string to attempt to
 640  *        get a configuration value from
 641  * @return GNUNET_YES if topology string matched a
 642  *         known topology, GNUNET_NO if not
 643  */
 644 int
 645 GNUNET_TESTING_topology_get (enum GNUNET_TESTING_Topology *topology,
 646                              const char *topology_string)
 647 {
 648   /**
 649    * Strings representing topologies in enum
 650    */
 651   static const char *topology_strings[] = {
 652       /**
 653        * A clique (everyone connected to everyone else).
 654        */
 655     "CLIQUE",
 656
 657       /**
 658        * Small-world network (2d torus plus random links).
 659        */
 660     "SMALL_WORLD",
 661
 662       /**
 663        * Small-world network (ring plus random links).
 664        */
 665     "SMALL_WORLD_RING",
 666
 667       /**
 668        * Ring topology.
 669        */
 670     "RING",
 671
 672       /**
 673        * 2-d torus.
 674        */
 675     "2D_TORUS",
 676
 677       /**
 678        * Random graph.
 679        */
 680     "ERDOS_RENYI",
 681
 682       /**
 683        * Certain percentage of peers are unable to communicate directly
 684        * replicating NAT conditions
 685        */
 686     "INTERNAT",
 687
 688       /**
 689        * Scale free topology.
 690        */
 691     "SCALE_FREE",
 692
 693       /**
 694        * Straight line topology.
 695        */
 696     "LINE",
 697
 698       /**
 699        * All peers are disconnected.
 700        */
 701     "NONE",
 702
 703     NULL
 704   };
 705
 706   int curr = 0;
 707   if (topology_string == NULL)
 708     return GNUNET_NO;
 709   while (topology_strings[curr] != NULL)
 710     {
 711       if (strcasecmp (topology_strings[curr], topology_string) == 0)
 712         {
 713           *topology = curr;
 714           return GNUNET_YES;
 715         }
 716       curr++;
 717     }
 718   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 719   return GNUNET_NO;
 720 }
 721
 722
 723 /**
 724  * Get connect topology option from string input.
 725  *
 726  * @param topology_option where to write the retrieved topology
 727  * @param topology_string The string to attempt to
 728  *        get a configuration value from
 729  * @return GNUNET_YES if string matched a known
 730  *         topology option, GNUNET_NO if not
 731  */
 732 int
 733 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption
 734                                     *topology_option,
 735                                     const char *topology_string)
 736 {
 737   /**
 738    * Options for connecting a topology as strings.
 739    */
 740   static const char *topology_option_strings[] = {
 741       /**
 742        * Try to connect all peers specified in the topology.
 743        */
 744     "CONNECT_ALL",
 745
 746       /**
 747        * Choose a random subset of connections to create.
 748        */
 749     "CONNECT_RANDOM_SUBSET",
 750
 751       /**
 752        * Create at least X connections for each peer.
 753        */
 754     "CONNECT_MINIMUM",
 755
 756       /**
 757        * Using a depth first search, create one connection
 758        * per peer.  If any are missed (graph disconnected)
 759        * start over at those peers until all have at least one
 760        * connection.
 761        */
 762     "CONNECT_DFS",
 763
 764       /**
 765        * Find the N closest peers to each allowed peer in the
 766        * topology and make sure a connection to those peers
 767        * exists in the connect topology.
 768        */
 769     "CONNECT_CLOSEST",
 770
 771       /**
 772        * No options specified.
 773        */
 774     "CONNECT_NONE",
 775
 776     NULL
 777   };
 778   int curr = 0;
 779
 780   if (topology_string == NULL)
 781     return GNUNET_NO;
 782   while (NULL != topology_option_strings[curr])
 783     {
 784       if (strcasecmp (topology_option_strings[curr], topology_string) == 0)
 785         {
 786           *topology_option = curr;
 787           return GNUNET_YES;
 788         }
 789       curr++;
 790     }
 791   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 792   return GNUNET_NO;
 793 }
 794
 795 /**
 796  * Function to iterate over options.  Copies
 797  * the options to the target configuration,
 798  * updating PORT values as needed.
 799  *
 800  * @param cls closure
 801  * @param section name of the section
 802  * @param option name of the option
 803  * @param value value of the option
 804  */
 805 static void
 806 update_config (void *cls,
 807                const char *section, const char *option, const char *value)
 808 {
 809   struct UpdateContext *ctx = cls;
 810   unsigned int ival;
 811   char cval[12];
 812   char uval[128];
 813   char *single_variable;
 814   char *per_host_variable;
 815   unsigned long long num_per_host;
 816
 817   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 818     {
 819       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 820       if ((ival != 0)
 821           && (GNUNET_YES !=
 822               GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 823                                                     single_variable)))
 824         {
 825           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 826           value = cval;
 827         }
 828
 829       GNUNET_free (single_variable);
 830     }
 831
 832   if (0 == strcmp (option, "UNIXPATH"))
 833     {
 834       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 835       GNUNET_asprintf (&per_host_variable, "num_%s_per_host", section);
 836       if (GNUNET_YES !=
 837           GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 838                                                 single_variable))
 839         {
 840           GNUNET_snprintf (uval,
 841                            sizeof (uval),
 842                            "/tmp/test-service-%s-%u", section, ctx->upnum++);
 843           value = uval;
 844         }
 845       else if ((GNUNET_YES ==
 846                GNUNET_CONFIGURATION_get_value_number (ctx->orig, "testing",
 847                                                       per_host_variable,
 848                                                       &num_per_host)) && (num_per_host > 0))
 849
 850         {
 851           GNUNET_snprintf (uval,
 852                            sizeof (uval),
 853                            "/tmp/test-service-%s-%u",
 854                            section, ctx->fdnum % num_per_host);
 855           value = uval;
 856         }
 857       GNUNET_free (single_variable);
 858       GNUNET_free (per_host_variable);
 859
 860     }
 861
 862   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 863     {
 864       value = ctx->hostname;
 865     }
 866
 867   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 868 }
 869
 870
 871 /**
 872  * Create a new configuration using the given configuration
 873  * as a template; however, each PORT in the existing cfg
 874  * must be renumbered by incrementing "*port".  If we run
 875  * out of "*port" numbers, return NULL.
 876  *
 877  * @param cfg template configuration
 878  * @param port port numbers to use, update to reflect
 879  *             port numbers that were used
 880  * @param upnum number to make unix domain socket names unique
 881  * @param hostname hostname of the controlling host, to allow control connections from
 882  * @param fdnum number used to offset the unix domain socket for grouped processes
 883  *              (such as statistics or peerinfo, which can be shared among others)
 884  *
 885  * @return new configuration, NULL on error
 886  */
 887 static struct GNUNET_CONFIGURATION_Handle *
 888 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 889              uint16_t * port,
 890              uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 891 {
 892   struct UpdateContext uc;
 893   uint16_t orig;
 894   char *control_host;
 895   char *allowed_hosts;
 896
 897   orig = *port;
 898   uc.nport = *port;
 899   uc.upnum = *upnum;
 900   uc.fdnum = *fdnum;
 901   uc.ret = GNUNET_CONFIGURATION_create ();
 902   uc.hostname = hostname;
 903   uc.orig = cfg;
 904
 905   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 906   if (uc.nport >= HIGH_PORT)
 907     {
 908       *port = orig;
 909       GNUNET_CONFIGURATION_destroy (uc.ret);
 910       return NULL;
 911     }
 912
 913   if (GNUNET_CONFIGURATION_get_value_string
 914       (cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 915     {
 916       if (hostname != NULL)
 917         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1; %s;", control_host,
 918                          hostname);
 919       else
 920         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", control_host);
 921
 922       GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
 923                                              allowed_hosts);
 924       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
 925                                              "ACCEPT_FROM", allowed_hosts);
 926       GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
 927                                              allowed_hosts);
 928       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
 929                                              "ACCEPT_FROM", allowed_hosts);
 930       GNUNET_free_non_null (control_host);
 931       GNUNET_free (allowed_hosts);
 932     }
 933
 934
 935   /* arm needs to know to allow connections from the host on which it is running,
 936    * otherwise gnunet-arm is unable to connect to it in some instances */
 937   if (hostname != NULL)
 938     {
 939       GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", hostname);
 940       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 941                                              "BINDTO", hostname);
 942       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 943                                              "BINDTO", hostname);
 944       GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
 945                                              allowed_hosts);
 946       GNUNET_free (allowed_hosts);
 947     }
 948   else
 949     {
 950       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 951                                              "BINDTO", "127.0.0.1");
 952       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 953                                              "BINDTO", "127.0.0.1");
 954     }
 955
 956   *port = (uint16_t) uc.nport;
 957   *upnum = uc.upnum;
 958   uc.fdnum++;
 959   *fdnum = uc.fdnum;
 960   return uc.ret;
 961 }
 962
 963
 964 /*
 965  * Add entries to the peers connect list
 966  *
 967  * @param pg the peer group we are working with
 968  * @param first index of the first peer
 969  * @param second index of the second peer
 970  *
 971  * @return the number of connections added
 972  *         technically should only be 0 or 2
 973  *
 974  */
 975 static unsigned int
 976 add_actual_connections (struct GNUNET_TESTING_PeerGroup *pg,
 977                         unsigned int first, unsigned int second)
 978 {
 979   int added;
 980   int add_first;
 981   int add_second;
 982
 983   GNUNET_HashCode hash_first;
 984   GNUNET_HashCode hash_second;
 985
 986   hash_from_uid (first, &hash_first);
 987   hash_from_uid (second, &hash_second);
 988
 989   add_first = GNUNET_NO;
 990   if (GNUNET_NO ==
 991       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].connect_peers,
 992                                               &hash_second))
 993     {
 994       add_first = GNUNET_YES;
 995     }
 996
 997   add_second = GNUNET_NO;
 998   if (GNUNET_NO ==
 999       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].connect_peers,
1000                                               &hash_first))
1001     {
1002       add_second = GNUNET_YES;
1003     }
1004
1005   added = 0;
1006   if (add_first)
1007     {
1008       GNUNET_assert (GNUNET_OK ==
1009                      GNUNET_CONTAINER_multihashmap_put (pg->
1010                                                         peers
1011                                                         [first].connect_peers,
1012                                                         &hash_second,
1013                                                         pg->
1014                                                         peers[second].daemon,
1015                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1016       pg->peers[first].num_connections++;
1017       added++;
1018     }
1019
1020   if (add_second)
1021     {
1022       GNUNET_assert (GNUNET_OK ==
1023                      GNUNET_CONTAINER_multihashmap_put (pg->
1024                                                         peers
1025                                                         [second].connect_peers,
1026                                                         &hash_first,
1027                                                         pg->
1028                                                         peers[first].daemon,
1029                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1030       pg->peers[second].num_connections++;
1031       added++;
1032     }
1033
1034   return added;
1035 }
1036
1037
1038 /*
1039  * Add entries to the peers allowed connections list
1040  *
1041  * @param pg the peer group we are working with
1042  * @param first index of the first peer
1043  * @param second index of the second peer
1044  *
1045  * @return the number of connections added (can be 0, 1 or 2)
1046  *         technically should only be 0 or 2, but the small price
1047  *         of iterating over the lists (hashmaps in the future)
1048  *         for being sure doesn't bother me!
1049  *
1050  */
1051 static unsigned int
1052 add_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg,
1053                          unsigned int first, unsigned int second)
1054 {
1055   int added;
1056 #if OLD
1057   struct PeerConnection *first_iter;
1058   struct PeerConnection *second_iter;
1059   struct PeerConnection *new_first;
1060   struct PeerConnection *new_second;
1061 #endif
1062   int add_first;
1063   int add_second;
1064
1065   GNUNET_HashCode hash_first;
1066   GNUNET_HashCode hash_second;
1067
1068   hash_from_uid (first, &hash_first);
1069   hash_from_uid (second, &hash_second);
1070
1071   add_first = GNUNET_NO;
1072   if (GNUNET_NO ==
1073       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].allowed_peers,
1074                                               &hash_second))
1075     {
1076       add_first = GNUNET_YES;
1077     }
1078
1079   add_second = GNUNET_NO;
1080   if (GNUNET_NO ==
1081       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].allowed_peers,
1082                                               &hash_first))
1083     {
1084       add_second = GNUNET_YES;
1085     }
1086 #if OLD
1087   first_iter = pg->peers[first].connected_peers;
1088   while (first_iter != NULL)
1089     {
1090       if (first_iter->daemon == pg->peers[second].daemon)
1091         add_first = GNUNET_NO;
1092       first_iter = first_iter->next;
1093     }
1094
1095   second_iter = pg->peers[second].connected_peers;
1096   add_second = GNUNET_YES;
1097   while (second_iter != NULL)
1098     {
1099       if (second_iter->daemon == pg->peers[first].daemon)
1100         add_second = GNUNET_NO;
1101       second_iter = second_iter->next;
1102     }
1103 #endif
1104
1105   added = 0;
1106   if (add_first)
1107     {
1108       GNUNET_assert (GNUNET_OK ==
1109                      GNUNET_CONTAINER_multihashmap_put (pg->
1110                                                         peers
1111                                                         [first].allowed_peers,
1112                                                         &hash_second,
1113                                                         pg->
1114                                                         peers[second].daemon,
1115                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1116 #if OLD
1117       new_first = GNUNET_malloc (sizeof (struct PeerConnection));
1118       new_first->daemon = pg->peers[second].daemon;
1119       new_first->next = pg->peers[first].connected_peers;
1120       pg->peers[first].connected_peers = new_first;
1121 #endif
1122       pg->peers[first].num_connections++;
1123       added++;
1124     }
1125
1126   if (add_second)
1127     {
1128       GNUNET_assert (GNUNET_OK ==
1129                      GNUNET_CONTAINER_multihashmap_put (pg->
1130                                                         peers
1131                                                         [second].allowed_peers,
1132                                                         &hash_first,
1133                                                         pg->
1134                                                         peers[first].daemon,
1135                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1136 #if OLD
1137       new_second = GNUNET_malloc (sizeof (struct PeerConnection));
1138       new_second->daemon = pg->peers[first].daemon;
1139       new_second->next = pg->peers[second].connected_peers;
1140       pg->peers[second].connected_peers = new_second;
1141       pg->peers[first].num_connections++;
1142 #endif
1143       pg->peers[second].num_connections++;
1144       added++;
1145     }
1146
1147   return added;
1148 }
1149
1150 /*
1151  * Add entries to the peers blacklisted list
1152  *
1153  * @param pg the peer group we are working with
1154  * @param first index of the first peer
1155  * @param second index of the second peer
1156  *
1157  * @return the number of connections added (can be 0, 1 or 2)
1158  *
1159  */
1160 static unsigned int
1161 blacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1162                        unsigned int first, unsigned int second)
1163 {
1164   int added;
1165   int add_first;
1166   int add_second;
1167   GNUNET_HashCode hash_first;
1168   GNUNET_HashCode hash_second;
1169
1170   hash_from_uid (first, &hash_first);
1171   hash_from_uid (second, &hash_second);
1172
1173   add_first = GNUNET_NO;
1174   if (GNUNET_NO ==
1175       GNUNET_CONTAINER_multihashmap_contains (pg->
1176                                               peers[first].blacklisted_peers,
1177                                               &hash_second))
1178     {
1179       add_first = GNUNET_YES;
1180     }
1181
1182   add_second = GNUNET_NO;
1183   if (GNUNET_NO ==
1184       GNUNET_CONTAINER_multihashmap_contains (pg->
1185                                               peers[second].blacklisted_peers,
1186                                               &hash_first))
1187     {
1188       add_second = GNUNET_YES;
1189     }
1190
1191   added = 0;
1192   if (add_first)
1193     {
1194       GNUNET_assert (GNUNET_OK ==
1195                      GNUNET_CONTAINER_multihashmap_put (pg->
1196                                                         peers
1197                                                         [first].blacklisted_peers,
1198                                                         &hash_second,
1199                                                         pg->
1200                                                         peers[second].daemon,
1201                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1202       pg->peers[first].num_connections++;
1203       added++;
1204     }
1205
1206   if (add_second)
1207     {
1208       GNUNET_assert (GNUNET_OK ==
1209                      GNUNET_CONTAINER_multihashmap_put (pg->
1210                                                         peers
1211                                                         [second].blacklisted_peers,
1212                                                         &hash_first,
1213                                                         pg->
1214                                                         peers[first].daemon,
1215                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1216       pg->peers[second].num_connections++;
1217       added++;
1218     }
1219
1220   return added;
1221 }
1222
1223 /*
1224  * Remove entries from the peers blacklisted list
1225  *
1226  * @param pg the peer group we are working with
1227  * @param first index of the first peer
1228  * @param second index of the second peer
1229  *
1230  * @return the number of connections removed (can be 0, 1 or 2)
1231  *
1232  */
1233 static unsigned int
1234 unblacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1235                          unsigned int first, unsigned int second)
1236 {
1237   int removed;
1238   int remove_first;
1239   int remove_second;
1240   GNUNET_HashCode hash_first;
1241   GNUNET_HashCode hash_second;
1242
1243   hash_from_uid (first, &hash_first);
1244   hash_from_uid (second, &hash_second);
1245
1246   remove_first =
1247     GNUNET_CONTAINER_multihashmap_contains (pg->
1248                                             peers[first].blacklisted_peers,
1249                                             &hash_second);
1250   remove_second =
1251     GNUNET_CONTAINER_multihashmap_contains (pg->
1252                                             peers[second].blacklisted_peers,
1253                                             &hash_first);
1254
1255   removed = 0;
1256   if (remove_first)
1257     {
1258       GNUNET_assert (GNUNET_YES ==
1259                      GNUNET_CONTAINER_multihashmap_remove (pg->
1260                                                            peers
1261                                                            [first].blacklisted_peers,
1262                                                            &hash_second,
1263                                                            pg->
1264                                                            peers
1265                                                            [second].daemon));
1266       removed++;
1267     }
1268
1269   if (remove_second)
1270     {
1271       GNUNET_assert (GNUNET_YES ==
1272                      GNUNET_CONTAINER_multihashmap_remove (pg->
1273                                                            peers
1274                                                            [second].blacklisted_peers,
1275                                                            &hash_first,
1276                                                            pg->
1277                                                            peers
1278                                                            [first].daemon));
1279       removed++;
1280     }
1281
1282   return removed;
1283 }
1284
1285 /**
1286  * Scale free network construction as described in:
1287  *
1288  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1289  *
1290  * Start with a network of "one" peer, then progressively add
1291  * peers up to the total number.  At each step, iterate over
1292  * all possible peers and connect new peer based on number of
1293  * existing connections of the target peer.
1294  *
1295  * @param pg the peer group we are dealing with
1296  * @param proc the connection processor to use
1297  *
1298  * @return the number of connections created
1299  */
1300 static unsigned int
1301 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg,
1302                    GNUNET_TESTING_ConnectionProcessor proc)
1303 {
1304
1305   unsigned int total_connections;
1306   unsigned int outer_count;
1307   unsigned int i;
1308   unsigned int previous_total_connections;
1309   double random;
1310   double probability;
1311
1312   GNUNET_assert (pg->total > 1);
1313
1314   /* Add a connection between the first two nodes */
1315   total_connections = proc (pg, 0, 1);
1316
1317   for (outer_count = 1; outer_count < pg->total; outer_count++)
1318     {
1319       previous_total_connections = total_connections;
1320       for (i = 0; i < outer_count; i++)
1321         {
1322           probability =
1323             pg->peers[i].num_connections /
1324             (double) previous_total_connections;
1325           random =
1326             ((double)
1327              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1328                                        UINT64_MAX)) / ((double) UINT64_MAX);
1329 #if VERBOSE_TESTING
1330           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1331                       "Considering connecting peer %d to peer %d\n",
1332                       outer_count, i);
1333 #endif
1334           if (random < probability)
1335             {
1336 #if VERBOSE_TESTING
1337               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1338                           "Connecting peer %d to peer %d\n", outer_count, i);
1339 #endif
1340               total_connections += proc (pg, outer_count, i);
1341             }
1342         }
1343     }
1344
1345   return total_connections;
1346 }
1347
1348 /**
1349  * Create a topology given a peer group (set of running peers)
1350  * and a connection processor.
1351  *
1352  * @param pg the peergroup to create the topology on
1353  * @param proc the connection processor to call to actually set
1354  *        up connections between two peers
1355  *
1356  * @return the number of connections that were set up
1357  *
1358  */
1359 static unsigned int
1360 create_small_world_ring (struct GNUNET_TESTING_PeerGroup *pg,
1361                          GNUNET_TESTING_ConnectionProcessor proc)
1362 {
1363   unsigned int i, j;
1364   int nodeToConnect;
1365   unsigned int natLog;
1366   unsigned int randomPeer;
1367   double random, logNModifier, percentage;
1368   unsigned int smallWorldConnections;
1369   int connsPerPeer;
1370   char *p_string;
1371   int max;
1372   int min;
1373   unsigned int useAnd;
1374   int connect_attempts;
1375
1376   logNModifier = 0.5;           /* FIXME: default value? */
1377   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1378                                                           "TESTING",
1379                                                           "LOGNMODIFIER",
1380                                                           &p_string))
1381     {
1382       if (sscanf (p_string, "%lf", &logNModifier) != 1)
1383         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1384                     _
1385                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1386                     p_string, "LOGNMODIFIER", "TESTING");
1387       GNUNET_free (p_string);
1388     }
1389   percentage = 0.5;             /* FIXME: default percentage? */
1390   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1391                                                           "TESTING",
1392                                                           "PERCENTAGE",
1393                                                           &p_string))
1394     {
1395       if (sscanf (p_string, "%lf", &percentage) != 1)
1396         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1397                     _
1398                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1399                     p_string, "PERCENTAGE", "TESTING");
1400       GNUNET_free (p_string);
1401     }
1402   natLog = log (pg->total);
1403   connsPerPeer = ceil (natLog * logNModifier);
1404
1405   if (connsPerPeer % 2 == 1)
1406     connsPerPeer += 1;
1407
1408   smallWorldConnections = 0;
1409   connect_attempts = 0;
1410   for (i = 0; i < pg->total; i++)
1411     {
1412       useAnd = 0;
1413       max = i + connsPerPeer / 2;
1414       min = i - connsPerPeer / 2;
1415
1416       if (max > pg->total - 1)
1417         {
1418           max = max - pg->total;
1419           useAnd = 1;
1420         }
1421
1422       if (min < 0)
1423         {
1424           min = pg->total - 1 + min;
1425           useAnd = 1;
1426         }
1427
1428       for (j = 0; j < connsPerPeer / 2; j++)
1429         {
1430           random =
1431             ((double)
1432              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1433                                        UINT64_MAX) / ((double) UINT64_MAX));
1434           if (random < percentage)
1435             {
1436               /* Connect to uniformly selected random peer */
1437               randomPeer =
1438                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1439                                           pg->total);
1440               while ((((randomPeer < max) && (randomPeer > min))
1441                       && (useAnd == 0)) || (((randomPeer > min)
1442                                              || (randomPeer < max))
1443                                             && (useAnd == 1)))
1444                 {
1445                   randomPeer =
1446                     GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1447                                               pg->total);
1448                 }
1449               smallWorldConnections += proc (pg, i, randomPeer);
1450             }
1451           else
1452             {
1453               nodeToConnect = i + j + 1;
1454               if (nodeToConnect > pg->total - 1)
1455                 {
1456                   nodeToConnect = nodeToConnect - pg->total;
1457                 }
1458               connect_attempts += proc (pg, i, nodeToConnect);
1459             }
1460         }
1461
1462     }
1463
1464   connect_attempts += smallWorldConnections;
1465
1466   return connect_attempts;
1467 }
1468
1469 /**
1470  * Create a topology given a peer group (set of running peers)
1471  * and a connection processor.
1472  *
1473  * @param pg the peergroup to create the topology on
1474  * @param proc the connection processor to call to actually set
1475  *        up connections between two peers
1476  *
1477  * @return the number of connections that were set up
1478  *
1479  */
1480 static unsigned int
1481 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg,
1482                        GNUNET_TESTING_ConnectionProcessor proc)
1483 {
1484   unsigned int outer_count, inner_count;
1485   unsigned int cutoff;
1486   int connect_attempts;
1487   double nat_percentage;
1488   char *p_string;
1489
1490   nat_percentage = 0.6;         /* FIXME: default percentage? */
1491   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1492                                                           "TESTING",
1493                                                           "PERCENTAGE",
1494                                                           &p_string))
1495     {
1496       if (sscanf (p_string, "%lf", &nat_percentage) != 1)
1497         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1498                     _
1499                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1500                     p_string, "PERCENTAGE", "TESTING");
1501       GNUNET_free (p_string);
1502     }
1503
1504
1505
1506   cutoff = (unsigned int) (nat_percentage * pg->total);
1507
1508   connect_attempts = 0;
1509
1510   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1511     {
1512       for (inner_count = outer_count + 1; inner_count < pg->total;
1513            inner_count++)
1514         {
1515           if ((outer_count > cutoff) || (inner_count > cutoff))
1516             {
1517 #if VERBOSE_TESTING
1518               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1519                           "Connecting peer %d to peer %d\n",
1520                           outer_count, inner_count);
1521 #endif
1522               connect_attempts += proc (pg, outer_count, inner_count);
1523             }
1524         }
1525     }
1526
1527   return connect_attempts;
1528
1529 }
1530
1531 /**
1532  * Create a topology given a peer group (set of running peers)
1533  * and a connection processor.
1534  *
1535  * @param pg the peergroup to create the topology on
1536  * @param proc the connection processor to call to actually set
1537  *        up connections between two peers
1538  *
1539  * @return the number of connections that were set up
1540  *
1541  */
1542 static unsigned int
1543 create_small_world (struct GNUNET_TESTING_PeerGroup *pg,
1544                     GNUNET_TESTING_ConnectionProcessor proc)
1545 {
1546   unsigned int i, j, k;
1547   unsigned int square;
1548   unsigned int rows;
1549   unsigned int cols;
1550   unsigned int toggle = 1;
1551   unsigned int nodeToConnect;
1552   unsigned int natLog;
1553   unsigned int node1Row;
1554   unsigned int node1Col;
1555   unsigned int node2Row;
1556   unsigned int node2Col;
1557   unsigned int distance;
1558   double probability, random, percentage;
1559   unsigned int smallWorldConnections;
1560   unsigned int small_world_it;
1561   char *p_string;
1562   int connect_attempts;
1563   square = floor (sqrt (pg->total));
1564   rows = square;
1565   cols = square;
1566
1567   percentage = 0.5;             /* FIXME: default percentage? */
1568   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1569                                                           "TESTING",
1570                                                           "PERCENTAGE",
1571                                                           &p_string))
1572     {
1573       if (sscanf (p_string, "%lf", &percentage) != 1)
1574         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1575                     _
1576                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1577                     p_string, "PERCENTAGE", "TESTING");
1578       GNUNET_free (p_string);
1579     }
1580   if (percentage < 0.0)
1581     {
1582       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1583                   _
1584                   ("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1585                   "PERCENTAGE", "TESTING", percentage);
1586       percentage = 0.5;
1587     }
1588   probability = 0.5;            /* FIXME: default percentage? */
1589   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1590                                                           "TESTING",
1591                                                           "PROBABILITY",
1592                                                           &p_string))
1593     {
1594       if (sscanf (p_string, "%lf", &probability) != 1)
1595         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1596                     _
1597                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1598                     p_string, "PROBABILITY", "TESTING");
1599       GNUNET_free (p_string);
1600     }
1601   if (square * square != pg->total)
1602     {
1603       while (rows * cols < pg->total)
1604         {
1605           if (toggle % 2 == 0)
1606             rows++;
1607           else
1608             cols++;
1609
1610           toggle++;
1611         }
1612     }
1613 #if VERBOSE_TESTING
1614   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1615               _
1616               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1617               rows, cols);
1618 #endif
1619
1620   connect_attempts = 0;
1621   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1622    * to the node to its right and above.  Once this is over, we'll have our torus!
1623    * Special case for the last node (if the rows and columns are not equal), connect
1624    * to the first in the row to maintain topology.
1625    */
1626   for (i = 0; i < pg->total; i++)
1627     {
1628       /* First connect to the node to the right */
1629       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1630         nodeToConnect = i + 1;
1631       else if (i + 1 == pg->total)
1632         nodeToConnect = rows * cols - cols;
1633       else
1634         nodeToConnect = i - cols + 1;
1635
1636       connect_attempts += proc (pg, i, nodeToConnect);
1637
1638       if (i < cols)
1639         nodeToConnect = (rows * cols) - cols + i;
1640       else
1641         nodeToConnect = i - cols;
1642
1643       if (nodeToConnect < pg->total)
1644         connect_attempts += proc (pg, i, nodeToConnect);
1645     }
1646   natLog = log (pg->total);
1647 #if VERBOSE_TESTING > 2
1648   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1649               _("natural log of %d is %d, will run %d iterations\n"),
1650               pg->total, natLog, (int) (natLog * percentage));
1651   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1652               _("Total connections added thus far: %u!\n"), connect_attempts);
1653 #endif
1654   smallWorldConnections = 0;
1655   small_world_it = (unsigned int) (natLog * percentage);
1656   if (small_world_it < 1)
1657     small_world_it = 1;
1658   GNUNET_assert (small_world_it > 0 && small_world_it < (unsigned int) -1);
1659   for (i = 0; i < small_world_it; i++)
1660     {
1661       for (j = 0; j < pg->total; j++)
1662         {
1663           /* Determine the row and column of node at position j on the 2d torus */
1664           node1Row = j / cols;
1665           node1Col = j - (node1Row * cols);
1666           for (k = 0; k < pg->total; k++)
1667             {
1668               /* Determine the row and column of node at position k on the 2d torus */
1669               node2Row = k / cols;
1670               node2Col = k - (node2Row * cols);
1671               /* Simple Cartesian distance */
1672               distance =
1673                 abs (node1Row - node2Row) + abs (node1Col - node2Col);
1674               if (distance > 1)
1675                 {
1676                   /* Calculate probability as 1 over the square of the distance */
1677                   probability = 1.0 / (distance * distance);
1678                   /* Choose a random value between 0 and 1 */
1679                   random =
1680                     ((double)
1681                      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1682                                                UINT64_MAX)) /
1683                     ((double) UINT64_MAX);
1684                   /* If random < probability, then connect the two nodes */
1685                   if (random < probability)
1686                     smallWorldConnections += proc (pg, j, k);
1687
1688                 }
1689             }
1690         }
1691     }
1692   connect_attempts += smallWorldConnections;
1693 #if VERBOSE_TESTING > 2
1694   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1695               _("Total connections added for small world: %d!\n"),
1696               smallWorldConnections);
1697 #endif
1698   return connect_attempts;
1699 }
1700
1701 /**
1702  * Create a topology given a peer group (set of running peers)
1703  * and a connection processor.
1704  *
1705  * @param pg the peergroup to create the topology on
1706  * @param proc the connection processor to call to actually set
1707  *        up connections between two peers
1708  *
1709  * @return the number of connections that were set up
1710  *
1711  */
1712 static unsigned int
1713 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg,
1714                     GNUNET_TESTING_ConnectionProcessor proc)
1715 {
1716   double temp_rand;
1717   unsigned int outer_count;
1718   unsigned int inner_count;
1719   int connect_attempts;
1720   double probability;
1721   char *p_string;
1722
1723   probability = 0.5;            /* FIXME: default percentage? */
1724   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1725                                                           "TESTING",
1726                                                           "PROBABILITY",
1727                                                           &p_string))
1728     {
1729       if (sscanf (p_string, "%lf", &probability) != 1)
1730         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1731                     _
1732                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1733                     p_string, "PROBABILITY", "TESTING");
1734       GNUNET_free (p_string);
1735     }
1736   connect_attempts = 0;
1737   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1738     {
1739       for (inner_count = outer_count + 1; inner_count < pg->total;
1740            inner_count++)
1741         {
1742           temp_rand =
1743             ((double)
1744              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1745                                        UINT64_MAX)) / ((double) UINT64_MAX);
1746 #if VERBOSE_TESTING
1747           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1748                       _("rand is %f probability is %f\n"), temp_rand,
1749                       probability);
1750 #endif
1751           if (temp_rand < probability)
1752             {
1753               connect_attempts += proc (pg, outer_count, inner_count);
1754             }
1755         }
1756     }
1757
1758   return connect_attempts;
1759 }
1760
1761 /**
1762  * Create a topology given a peer group (set of running peers)
1763  * and a connection processor.  This particular function creates
1764  * the connections for a 2d-torus, plus additional "closest"
1765  * connections per peer.
1766  *
1767  * @param pg the peergroup to create the topology on
1768  * @param proc the connection processor to call to actually set
1769  *        up connections between two peers
1770  *
1771  * @return the number of connections that were set up
1772  *
1773  */
1774 static unsigned int
1775 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg,
1776                  GNUNET_TESTING_ConnectionProcessor proc)
1777 {
1778   unsigned int i;
1779   unsigned int square;
1780   unsigned int rows;
1781   unsigned int cols;
1782   unsigned int toggle = 1;
1783   unsigned int nodeToConnect;
1784   int connect_attempts;
1785
1786   connect_attempts = 0;
1787
1788   square = floor (sqrt (pg->total));
1789   rows = square;
1790   cols = square;
1791
1792   if (square * square != pg->total)
1793     {
1794       while (rows * cols < pg->total)
1795         {
1796           if (toggle % 2 == 0)
1797             rows++;
1798           else
1799             cols++;
1800
1801           toggle++;
1802         }
1803     }
1804 #if VERBOSE_TESTING
1805   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1806               _
1807               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1808               rows, cols);
1809 #endif
1810   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1811    * to the node to its right and above.  Once this is over, we'll have our torus!
1812    * Special case for the last node (if the rows and columns are not equal), connect
1813    * to the first in the row to maintain topology.
1814    */
1815   for (i = 0; i < pg->total; i++)
1816     {
1817       /* First connect to the node to the right */
1818       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1819         nodeToConnect = i + 1;
1820       else if (i + 1 == pg->total)
1821         nodeToConnect = rows * cols - cols;
1822       else
1823         nodeToConnect = i - cols + 1;
1824 #if VERBOSE_TESTING
1825       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1826                   "Connecting peer %d to peer %d\n", i, nodeToConnect);
1827 #endif
1828       connect_attempts += proc (pg, i, nodeToConnect);
1829
1830       /* Second connect to the node immediately above */
1831       if (i < cols)
1832         nodeToConnect = (rows * cols) - cols + i;
1833       else
1834         nodeToConnect = i - cols;
1835
1836       if (nodeToConnect < pg->total)
1837         {
1838 #if VERBOSE_TESTING
1839           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1840                       "Connecting peer %d to peer %d\n", i, nodeToConnect);
1841 #endif
1842           connect_attempts += proc (pg, i, nodeToConnect);
1843         }
1844
1845     }
1846
1847   return connect_attempts;
1848 }
1849
1850
1851 /**
1852  * Create a topology given a peer group (set of running peers)
1853  * and a connection processor.
1854  *
1855  * @param pg the peergroup to create the topology on
1856  * @param proc the connection processor to call to actually set
1857  *        up connections between two peers
1858  *
1859  * @return the number of connections that were set up
1860  *
1861  */
1862 static unsigned int
1863 create_clique (struct GNUNET_TESTING_PeerGroup *pg,
1864                GNUNET_TESTING_ConnectionProcessor proc)
1865 {
1866   unsigned int outer_count;
1867   unsigned int inner_count;
1868   int connect_attempts;
1869
1870   connect_attempts = 0;
1871
1872   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1873     {
1874       for (inner_count = outer_count + 1; inner_count < pg->total;
1875            inner_count++)
1876         {
1877 #if VERBOSE_TESTING
1878           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1879                       "Connecting peer %d to peer %d\n",
1880                       outer_count, inner_count);
1881 #endif
1882           connect_attempts += proc (pg, outer_count, inner_count);
1883         }
1884     }
1885
1886   return connect_attempts;
1887 }
1888
1889 /**
1890  * Create a topology given a peer group (set of running peers)
1891  * and a connection processor.
1892  *
1893  * @param pg the peergroup to create the topology on
1894  * @param proc the connection processor to call to actually set
1895  *        up connections between two peers
1896  *
1897  * @return the number of connections that were set up
1898  *
1899  */
1900 static unsigned int
1901 create_line (struct GNUNET_TESTING_PeerGroup *pg,
1902              GNUNET_TESTING_ConnectionProcessor proc)
1903 {
1904   unsigned int count;
1905   int connect_attempts;
1906
1907   connect_attempts = 0;
1908
1909   /* Connect each peer to the next highest numbered peer */
1910   for (count = 0; count < pg->total - 1; count++)
1911     {
1912 #if VERBOSE_TESTING
1913       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1914                   "Connecting peer %d to peer %d\n", count, count + 1);
1915 #endif
1916       connect_attempts += proc (pg, count, count + 1);
1917     }
1918
1919   return connect_attempts;
1920 }
1921
1922 /**
1923  * Create a topology given a peer group (set of running peers)
1924  * and a connection processor.
1925  *
1926  * @param pg the peergroup to create the topology on
1927  * @param proc the connection processor to call to actually set
1928  *        up connections between two peers
1929  *
1930  * @return the number of connections that were set up
1931  *
1932  */
1933 static unsigned int
1934 create_ring (struct GNUNET_TESTING_PeerGroup *pg,
1935              GNUNET_TESTING_ConnectionProcessor proc)
1936 {
1937   unsigned int count;
1938   int connect_attempts;
1939
1940   connect_attempts = 0;
1941
1942   /* Connect each peer to the next highest numbered peer */
1943   for (count = 0; count < pg->total - 1; count++)
1944     {
1945 #if VERBOSE_TESTING
1946       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1947                   "Connecting peer %d to peer %d\n", count, count + 1);
1948 #endif
1949       connect_attempts += proc (pg, count, count + 1);
1950     }
1951
1952   /* Connect the last peer to the first peer */
1953   connect_attempts += proc (pg, pg->total - 1, 0);
1954
1955   return connect_attempts;
1956 }
1957
1958
1959 /**
1960  * Iterator for writing friends of a peer to a file.
1961  *
1962  * @param cls closure, an open writable file handle
1963  * @param key the key the daemon was stored under
1964  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1965  *
1966  * @return GNUNET_YES to continue iteration
1967  *
1968  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1969  *       with a single file_iterator that takes a closure which contains
1970  *       the prefix to write before the peer.  Then this could be used
1971  *       for blacklisting multiple transports and writing the friend
1972  *       file.  I'm sure *someone* will complain loudly about other
1973  *       things that negate these functions even existing so no point in
1974  *       "fixing" now.
1975  */
1976 static int
1977 friend_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
1978 {
1979   FILE *temp_friend_handle = cls;
1980   struct GNUNET_TESTING_Daemon *peer = value;
1981   struct GNUNET_PeerIdentity *temppeer;
1982   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1983
1984   temppeer = &peer->id;
1985   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
1986   fprintf (temp_friend_handle, "%s\n", (char *) &peer_enc);
1987
1988   return GNUNET_YES;
1989 }
1990
1991 struct BlacklistContext
1992 {
1993   /*
1994    * The (open) file handle to write to
1995    */
1996   FILE *temp_file_handle;
1997
1998   /*
1999    * The transport that this peer will be blacklisted on.
2000    */
2001   char *transport;
2002 };
2003
2004 /**
2005  * Iterator for writing blacklist data to appropriate files.
2006  *
2007  * @param cls closure, an open writable file handle
2008  * @param key the key the daemon was stored under
2009  * @param value the GNUNET_TESTING_Daemon that needs to be written.
2010  *
2011  * @return GNUNET_YES to continue iteration
2012  */
2013 static int
2014 blacklist_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2015 {
2016   struct BlacklistContext *blacklist_ctx = cls;
2017   //FILE *temp_blacklist_handle = cls;
2018   struct GNUNET_TESTING_Daemon *peer = value;
2019   struct GNUNET_PeerIdentity *temppeer;
2020   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
2021
2022   temppeer = &peer->id;
2023   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
2024   fprintf (blacklist_ctx->temp_file_handle, "%s:%s\n",
2025            blacklist_ctx->transport, (char *) &peer_enc);
2026
2027   return GNUNET_YES;
2028 }
2029
2030 /*
2031  * Create the friend files based on the PeerConnection's
2032  * of each peer in the peer group, and copy the files
2033  * to the appropriate place
2034  *
2035  * @param pg the peer group we are dealing with
2036  */
2037 static int
2038 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
2039 {
2040   FILE *temp_friend_handle;
2041   unsigned int pg_iter;
2042   char *temp_service_path;
2043   struct GNUNET_OS_Process **procarr;
2044   char *arg;
2045   char *mytemp;
2046   enum GNUNET_OS_ProcessStatusType type;
2047   unsigned long return_code;
2048   int count;
2049   int ret;
2050   int max_wait = 10;
2051
2052   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2053   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2054     {
2055       mytemp = GNUNET_DISK_mktemp ("friends");
2056       GNUNET_assert (mytemp != NULL);
2057       temp_friend_handle = fopen (mytemp, "wt");
2058       GNUNET_assert (temp_friend_handle != NULL);
2059       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].allowed_peers,
2060                                              &friend_file_iterator,
2061                                              temp_friend_handle);
2062       fclose (temp_friend_handle);
2063
2064       if (GNUNET_OK !=
2065           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2066                                                  daemon->cfg, "PATHS",
2067                                                  "SERVICEHOME",
2068                                                  &temp_service_path))
2069         {
2070           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2071                       _
2072                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2073                       "SERVICEHOME", "PATHS");
2074           if (UNLINK (mytemp) != 0)
2075             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2076                                       mytemp);
2077           GNUNET_free (mytemp);
2078           break;
2079         }
2080
2081       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2082         {
2083           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
2084           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2085                                                       "mv", mytemp, arg,
2086                                                       NULL);
2087 #if VERBOSE_TESTING
2088           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2089                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2090 #endif
2091
2092           GNUNET_free (arg);
2093         }
2094       else                      /* Remote, scp the file to the correct place */
2095         {
2096           if (NULL != pg->peers[pg_iter].daemon->username)
2097             GNUNET_asprintf (&arg, "%s@%s:%s/friends",
2098                              pg->peers[pg_iter].daemon->username,
2099                              pg->peers[pg_iter].daemon->hostname,
2100                              temp_service_path);
2101           else
2102             GNUNET_asprintf (&arg, "%s:%s/friends",
2103                              pg->peers[pg_iter].daemon->hostname,
2104                              temp_service_path);
2105           procarr[pg_iter] =
2106             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2107                                      NULL);
2108
2109 #if VERBOSE_TESTING
2110           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2111                       _("Copying file with command scp %s %s\n"), mytemp,
2112                       arg);
2113 #endif
2114           GNUNET_free (arg);
2115         }
2116       GNUNET_free (temp_service_path);
2117       GNUNET_free (mytemp);
2118     }
2119
2120   count = 0;
2121   ret = GNUNET_SYSERR;
2122   while ((count < max_wait) && (ret != GNUNET_OK))
2123     {
2124       ret = GNUNET_OK;
2125       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2126         {
2127 #if VERBOSE_TESTING
2128           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2129                       _("Checking copy status of file %d\n"), pg_iter);
2130 #endif
2131           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2132             {
2133               if (GNUNET_OS_process_status
2134                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2135                 {
2136                   ret = GNUNET_SYSERR;
2137                 }
2138               else if ((type != GNUNET_OS_PROCESS_EXITED)
2139                        || (return_code != 0))
2140                 {
2141                   ret = GNUNET_SYSERR;
2142                 }
2143               else
2144                 {
2145                   GNUNET_OS_process_close (procarr[pg_iter]);
2146                   procarr[pg_iter] = NULL;
2147 #if VERBOSE_TESTING
2148                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2149                               _("File %d copied\n"), pg_iter);
2150 #endif
2151                 }
2152             }
2153         }
2154       count++;
2155       if (ret == GNUNET_SYSERR)
2156         {
2157           /* FIXME: why sleep here? -CG */
2158           sleep (1);
2159         }
2160     }
2161
2162 #if VERBOSE_TESTING
2163   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2164               _("Finished copying all friend files!\n"));
2165 #endif
2166   GNUNET_free (procarr);
2167   return ret;
2168 }
2169
2170
2171 /*
2172  * Create the blacklist files based on the PeerConnection's
2173  * of each peer in the peer group, and copy the files
2174  * to the appropriate place.
2175  *
2176  * @param pg the peer group we are dealing with
2177  * @param transports space delimited list of transports to blacklist
2178  */
2179 static int
2180 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
2181                                  const char *transports)
2182 {
2183   FILE *temp_file_handle;
2184   static struct BlacklistContext blacklist_ctx;
2185   unsigned int pg_iter;
2186   char *temp_service_path;
2187   struct GNUNET_OS_Process **procarr;
2188   char *arg;
2189   char *mytemp;
2190   enum GNUNET_OS_ProcessStatusType type;
2191   unsigned long return_code;
2192   int count;
2193   int ret;
2194   int max_wait = 10;
2195   int transport_len;
2196   unsigned int i;
2197   char *pos;
2198   char *temp_transports;
2199
2200   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2201   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2202     {
2203       mytemp = GNUNET_DISK_mktemp ("blacklist");
2204       GNUNET_assert (mytemp != NULL);
2205       temp_file_handle = fopen (mytemp, "wt");
2206       GNUNET_assert (temp_file_handle != NULL);
2207       temp_transports = GNUNET_strdup (transports);
2208       blacklist_ctx.temp_file_handle = temp_file_handle;
2209       transport_len = strlen (temp_transports) + 1;
2210       pos = NULL;
2211
2212       for (i = 0; i < transport_len; i++)
2213         {
2214           if ((temp_transports[i] == ' ') && (pos == NULL))
2215             continue;           /* At start of string (whitespace) */
2216           else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
2217             {
2218               temp_transports[i] = '\0';
2219               blacklist_ctx.transport = pos;
2220               GNUNET_CONTAINER_multihashmap_iterate (pg->
2221                                                      peers
2222                                                      [pg_iter].blacklisted_peers,
2223                                                      &blacklist_file_iterator,
2224                                                      &blacklist_ctx);
2225               pos = NULL;
2226             }                   /* At beginning of actual string */
2227           else if (pos == NULL)
2228             {
2229               pos = &temp_transports[i];
2230             }
2231         }
2232
2233       GNUNET_free (temp_transports);
2234       fclose (temp_file_handle);
2235
2236       if (GNUNET_OK !=
2237           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2238                                                  daemon->cfg, "PATHS",
2239                                                  "SERVICEHOME",
2240                                                  &temp_service_path))
2241         {
2242           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2243                       _
2244                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2245                       "SERVICEHOME", "PATHS");
2246           if (UNLINK (mytemp) != 0)
2247             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2248                                       mytemp);
2249           GNUNET_free (mytemp);
2250           break;
2251         }
2252
2253       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2254         {
2255           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
2256           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2257                                                       "mv", mytemp, arg,
2258                                                       NULL);
2259 #if VERBOSE_TESTING
2260           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2261                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2262 #endif
2263
2264           GNUNET_free (arg);
2265         }
2266       else                      /* Remote, scp the file to the correct place */
2267         {
2268           if (NULL != pg->peers[pg_iter].daemon->username)
2269             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist",
2270                              pg->peers[pg_iter].daemon->username,
2271                              pg->peers[pg_iter].daemon->hostname,
2272                              temp_service_path);
2273           else
2274             GNUNET_asprintf (&arg, "%s:%s/blacklist",
2275                              pg->peers[pg_iter].daemon->hostname,
2276                              temp_service_path);
2277           procarr[pg_iter] =
2278             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2279                                      NULL);
2280
2281 #if VERBOSE_TESTING
2282           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2283                       _("Copying file with command scp %s %s\n"), mytemp,
2284                       arg);
2285 #endif
2286           GNUNET_free (arg);
2287         }
2288       GNUNET_free (temp_service_path);
2289       GNUNET_free (mytemp);
2290     }
2291
2292   count = 0;
2293   ret = GNUNET_SYSERR;
2294   while ((count < max_wait) && (ret != GNUNET_OK))
2295     {
2296       ret = GNUNET_OK;
2297       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2298         {
2299 #if VERBOSE_TESTING
2300           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2301                       _("Checking copy status of file %d\n"), pg_iter);
2302 #endif
2303           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2304             {
2305               if (GNUNET_OS_process_status
2306                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2307                 {
2308                   ret = GNUNET_SYSERR;
2309                 }
2310               else if ((type != GNUNET_OS_PROCESS_EXITED)
2311                        || (return_code != 0))
2312                 {
2313                   ret = GNUNET_SYSERR;
2314                 }
2315               else
2316                 {
2317                   GNUNET_OS_process_close (procarr[pg_iter]);
2318                   procarr[pg_iter] = NULL;
2319 #if VERBOSE_TESTING
2320                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2321                               _("File %d copied\n"), pg_iter);
2322 #endif
2323                 }
2324             }
2325         }
2326       count++;
2327       if (ret == GNUNET_SYSERR)
2328         {
2329           /* FIXME: why sleep here? -CG */
2330           sleep (1);
2331         }
2332     }
2333
2334 #if VERBOSE_TESTING
2335   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2336               _("Finished copying all blacklist files!\n"));
2337 #endif
2338   GNUNET_free (procarr);
2339   return ret;
2340 }
2341
2342
2343 /**
2344  * Internal notification of a connection, kept so that we can ensure some connections
2345  * happen instead of flooding all testing daemons with requests to connect.
2346  */
2347 static void
2348 internal_connect_notify (void *cls,
2349                          const struct GNUNET_PeerIdentity *first,
2350                          const struct GNUNET_PeerIdentity *second,
2351                          uint32_t distance,
2352                          const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2353                          const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2354                          struct GNUNET_TESTING_Daemon *first_daemon,
2355                          struct GNUNET_TESTING_Daemon *second_daemon,
2356                          const char *emsg)
2357 {
2358   struct ConnectTopologyContext *ct_ctx = cls;
2359   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
2360   outstanding_connects--;
2361   ct_ctx->remaining_connections--;
2362   if (ct_ctx->remaining_connections == 0)
2363     {
2364       if (ct_ctx->notify_connections_done != NULL)
2365         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
2366       GNUNET_free (ct_ctx);
2367     }
2368
2369   if (pg->notify_connection != NULL)
2370     pg->notify_connection (pg->notify_connection_cls, first, second, distance,
2371                            first_cfg, second_cfg, first_daemon, second_daemon,
2372                            emsg);
2373 }
2374
2375
2376 /**
2377  * Either delay a connection (because there are too many outstanding)
2378  * or schedule it for right now.
2379  *
2380  * @param cls a connection context
2381  * @param tc the task runtime context
2382  */
2383 static void
2384 schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2385 {
2386   struct ConnectContext *connect_context = cls;
2387
2388   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2389     return;
2390
2391   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2392     {
2393 #if VERBOSE_TESTING > 2
2394       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2395                   _
2396                   ("Delaying connect, we have too many outstanding connections!\n"));
2397 #endif
2398       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
2399                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
2400                                     &schedule_connect, connect_context);
2401     }
2402   else
2403     {
2404 #if VERBOSE_TESTING > 2
2405       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2406                   _("Creating connection, outstanding_connections is %d\n"),
2407                   outstanding_connects);
2408 #endif
2409       outstanding_connects++;
2410       GNUNET_TESTING_daemons_connect (connect_context->first,
2411                                       connect_context->second,
2412                                       CONNECT_TIMEOUT,
2413                                       CONNECT_ATTEMPTS,
2414                                       &internal_connect_notify,
2415                                       connect_context->ct_ctx);
2416       GNUNET_free (connect_context);
2417     }
2418 }
2419
2420
2421 /**
2422  * Iterator for actually scheduling connections to be created
2423  * between two peers.
2424  *
2425  * @param cls closure, a GNUNET_TESTING_Daemon
2426  * @param key the key the second Daemon was stored under
2427  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2428  *
2429  * @return GNUNET_YES to continue iteration
2430  */
2431 static int
2432 connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2433 {
2434   struct ConnectTopologyContext *ct_ctx = cls;
2435   struct PeerData *first = ct_ctx->first;
2436   struct GNUNET_TESTING_Daemon *second = value;
2437   struct ConnectContext *connect_context;
2438
2439   connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2440   connect_context->first = first->daemon;
2441   connect_context->second = second;
2442   connect_context->ct_ctx = ct_ctx;
2443   GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2444
2445   return GNUNET_YES;
2446 }
2447
2448
2449 /**
2450  * Iterator for copying all entries in the allowed hashmap to the
2451  * connect hashmap.
2452  *
2453  * @param cls closure, a GNUNET_TESTING_Daemon
2454  * @param key the key the second Daemon was stored under
2455  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2456  *
2457  * @return GNUNET_YES to continue iteration
2458  */
2459 static int
2460 copy_topology_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2461 {
2462   struct PeerData *first = cls;
2463
2464   GNUNET_assert (GNUNET_OK ==
2465                  GNUNET_CONTAINER_multihashmap_put (first->connect_peers, key,
2466                                                     value,
2467                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2468
2469   return GNUNET_YES;
2470 }
2471
2472 /**
2473  * Make the peers to connect the same as those that are allowed to be
2474  * connected.
2475  *
2476  * @param pg the peer group
2477  */
2478 static int
2479 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2480 {
2481   unsigned int pg_iter;
2482   int ret;
2483   int total;
2484
2485   total = 0;
2486   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2487     {
2488       ret =
2489         GNUNET_CONTAINER_multihashmap_iterate (pg->
2490                                                peers[pg_iter].allowed_peers,
2491                                                &copy_topology_iterator,
2492                                                &pg->peers[pg_iter]);
2493       if (GNUNET_SYSERR == ret)
2494         return GNUNET_SYSERR;
2495
2496       total = total + ret;
2497     }
2498
2499   return total;
2500 }
2501
2502
2503 /**
2504  * Connect the topology as specified by the PeerConnection's
2505  * of each peer in the peer group
2506  *
2507  * @param pg the peer group we are dealing with
2508  * @param notify_callback callback to notify when finished
2509  * @param notify_cls closure for notify callback
2510  *
2511  * @return the number of connections that will be attempted
2512  */
2513 static int
2514 connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
2515                   GNUNET_TESTING_NotifyCompletion notify_callback,
2516                   void *notify_cls)
2517 {
2518   unsigned int pg_iter;
2519   int ret;
2520   unsigned int total;
2521   struct ConnectTopologyContext *ct_ctx;
2522 #if OLD
2523   struct PeerConnection *connection_iter;
2524   struct ConnectContext *connect_context;
2525 #endif
2526
2527   total = 0;
2528   ct_ctx = GNUNET_malloc (sizeof (struct ConnectTopologyContext));
2529   ct_ctx->notify_connections_done = notify_callback;
2530   ct_ctx->notify_cls = notify_cls;
2531   ct_ctx->pg = pg;
2532
2533   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2534     {
2535       total +=
2536         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
2537     }
2538
2539   if (total == 0)
2540     {
2541       GNUNET_free (ct_ctx);
2542       return total;
2543     }
2544   ct_ctx->remaining_connections = total;
2545   total = 0;
2546
2547   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2548     {
2549       ct_ctx->first = &pg->peers[pg_iter];
2550       ret =
2551         GNUNET_CONTAINER_multihashmap_iterate (pg->
2552                                                peers[pg_iter].connect_peers,
2553                                                &connect_iterator, ct_ctx);
2554       GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
2555       total = total + ret;
2556
2557 #if OLD
2558       connection_iter = FIXME;
2559       while (connection_iter != NULL)
2560         {
2561           connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2562           connect_context->pg = pg;
2563           connect_context->first = FIXME;
2564           connect_context->second = connection_iter->daemon;
2565           GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2566           connection_iter = connection_iter->next;
2567         }
2568 #endif
2569     }
2570   return total;
2571 }
2572
2573
2574 /**
2575  * Takes a peer group and creates a topology based on the
2576  * one specified.  Creates a topology means generates friend
2577  * files for the peers so they can only connect to those allowed
2578  * by the topology.  This will only have an effect once peers
2579  * are started if the FRIENDS_ONLY option is set in the base
2580  * config.  Also takes an optional restrict topology which
2581  * disallows connections based on particular transports
2582  * UNLESS they are specified in the restricted topology.
2583  *
2584  * @param pg the peer group struct representing the running peers
2585  * @param topology which topology to connect the peers in
2586  * @param restrict_topology disallow restrict_transports transport
2587  *                          connections to peers NOT in this topology
2588  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2589  * @param restrict_transports space delimited list of transports to blacklist
2590  *                            to create restricted topology
2591  *
2592  * @return the maximum number of connections were all allowed peers
2593  *         connected to each other
2594  */
2595 unsigned int
2596 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2597                                 enum GNUNET_TESTING_Topology topology,
2598                                 enum GNUNET_TESTING_Topology
2599                                 restrict_topology,
2600                                 const char *restrict_transports)
2601 {
2602   int ret;
2603   unsigned int num_connections;
2604   int unblacklisted_connections;
2605
2606   switch (topology)
2607     {
2608     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2609 #if VERBOSE_TESTING
2610       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating clique topology\n"));
2611 #endif
2612       num_connections = create_clique (pg, &add_allowed_connections);
2613       break;
2614     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2615 #if VERBOSE_TESTING
2616       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2617                   _("Creating small world (ring) topology\n"));
2618 #endif
2619       num_connections =
2620         create_small_world_ring (pg, &add_allowed_connections);
2621       break;
2622     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2623 #if VERBOSE_TESTING
2624       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2625                   _("Creating small world (2d-torus) topology\n"));
2626 #endif
2627       num_connections = create_small_world (pg, &add_allowed_connections);
2628       break;
2629     case GNUNET_TESTING_TOPOLOGY_RING:
2630 #if VERBOSE_TESTING
2631       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating ring topology\n"));
2632 #endif
2633       num_connections = create_ring (pg, &add_allowed_connections);
2634       break;
2635     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2636 #if VERBOSE_TESTING
2637       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating 2d torus topology\n"));
2638 #endif
2639       num_connections = create_2d_torus (pg, &add_allowed_connections);
2640       break;
2641     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2642 #if VERBOSE_TESTING
2643       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2644                   _("Creating Erdos-Renyi topology\n"));
2645 #endif
2646       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2647       break;
2648     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2649 #if VERBOSE_TESTING
2650       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating InterNAT topology\n"));
2651 #endif
2652       num_connections = create_nated_internet (pg, &add_allowed_connections);
2653       break;
2654     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2655 #if VERBOSE_TESTING
2656       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2657                   _("Creating Scale Free topology\n"));
2658 #endif
2659       num_connections = create_scale_free (pg, &add_allowed_connections);
2660       break;
2661     case GNUNET_TESTING_TOPOLOGY_LINE:
2662 #if VERBOSE_TESTING
2663       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2664                   _("Creating straight line topology\n"));
2665 #endif
2666       num_connections = create_line (pg, &add_allowed_connections);
2667       break;
2668     case GNUNET_TESTING_TOPOLOGY_NONE:
2669 #if VERBOSE_TESTING
2670       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2671                   _
2672                   ("Creating no allowed topology (all peers can connect at core level)\n"));
2673 #endif
2674       num_connections = 0;
2675       break;
2676     default:
2677       num_connections = 0;
2678       break;
2679     }
2680
2681   if (GNUNET_YES ==
2682       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2683     {
2684       ret = create_and_copy_friend_files (pg);
2685       if (ret != GNUNET_OK)
2686         {
2687 #if VERBOSE_TESTING
2688           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2689                       _("Failed during friend file copying!\n"));
2690 #endif
2691           return GNUNET_SYSERR;
2692         }
2693       else
2694         {
2695 #if VERBOSE_TESTING
2696           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2697                       _("Friend files created/copied successfully!\n"));
2698 #endif
2699         }
2700     }
2701
2702   /* Use the create clique method to initially set all connections as blacklisted. */
2703   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2704     create_clique (pg, &blacklist_connections);
2705
2706   unblacklisted_connections = 0;
2707   /* Un-blacklist connections as per the topology specified */
2708   switch (restrict_topology)
2709     {
2710     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2711 #if VERBOSE_TESTING
2712       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2713                   _("Blacklisting all but clique topology\n"));
2714 #endif
2715       unblacklisted_connections =
2716         create_clique (pg, &unblacklist_connections);
2717       break;
2718     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2719 #if VERBOSE_TESTING
2720       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2721                   _("Blacklisting all but small world (ring) topology\n"));
2722 #endif
2723       unblacklisted_connections =
2724         create_small_world_ring (pg, &unblacklist_connections);
2725       break;
2726     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2727 #if VERBOSE_TESTING
2728       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2729                   _
2730                   ("Blacklisting all but small world (2d-torus) topology\n"));
2731 #endif
2732       unblacklisted_connections =
2733         create_small_world (pg, &unblacklist_connections);
2734       break;
2735     case GNUNET_TESTING_TOPOLOGY_RING:
2736 #if VERBOSE_TESTING
2737       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2738                   _("Blacklisting all but ring topology\n"));
2739 #endif
2740       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2741       break;
2742     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2743 #if VERBOSE_TESTING
2744       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2745                   _("Blacklisting all but 2d torus topology\n"));
2746 #endif
2747       unblacklisted_connections =
2748         create_2d_torus (pg, &unblacklist_connections);
2749       break;
2750     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2751 #if VERBOSE_TESTING
2752       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2753                   _("Blacklisting all but Erdos-Renyi topology\n"));
2754 #endif
2755       unblacklisted_connections =
2756         create_erdos_renyi (pg, &unblacklist_connections);
2757       break;
2758     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2759 #if VERBOSE_TESTING
2760       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2761                   _("Blacklisting all but InterNAT topology\n"));
2762 #endif
2763       unblacklisted_connections =
2764         create_nated_internet (pg, &unblacklist_connections);
2765       break;
2766     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2767 #if VERBOSE_TESTING
2768       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2769                   _("Blacklisting all but Scale Free topology\n"));
2770 #endif
2771       unblacklisted_connections =
2772         create_scale_free (pg, &unblacklist_connections);
2773       break;
2774     case GNUNET_TESTING_TOPOLOGY_LINE:
2775 #if VERBOSE_TESTING
2776       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2777                   _("Blacklisting all but straight line topology\n"));
2778 #endif
2779       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2780       break;
2781     case GNUNET_TESTING_TOPOLOGY_NONE:
2782 #if VERBOSE_TESTING
2783       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2784                   _
2785                   ("Creating no blacklist topology (all peers can connect at transport level)\n"));
2786 #endif
2787     default:
2788       break;
2789     }
2790
2791   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2792     {
2793       ret = create_and_copy_blacklist_files (pg, restrict_transports);
2794       if (ret != GNUNET_OK)
2795         {
2796 #if VERBOSE_TESTING
2797           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2798                       _("Failed during blacklist file copying!\n"));
2799 #endif
2800           return 0;
2801         }
2802       else
2803         {
2804 #if VERBOSE_TESTING
2805           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2806                       _("Blacklist files created/copied successfully!\n"));
2807 #endif
2808         }
2809     }
2810   return num_connections;
2811 }
2812
2813 struct RandomContext
2814 {
2815   /**
2816    * The peergroup
2817    */
2818   struct GNUNET_TESTING_PeerGroup *pg;
2819
2820   /**
2821    * uid of the first peer
2822    */
2823   uint32_t first_uid;
2824
2825   /**
2826    * Peer data for first peer.
2827    */
2828   struct PeerData *first;
2829
2830   /**
2831    * Random percentage to use
2832    */
2833   double percentage;
2834 };
2835
2836 struct MinimumContext
2837 {
2838   /**
2839    * The peergroup
2840    */
2841   struct GNUNET_TESTING_PeerGroup *pg;
2842
2843   /**
2844    * uid of the first peer
2845    */
2846   uint32_t first_uid;
2847
2848   /**
2849    * Peer data for first peer.
2850    */
2851   struct PeerData *first;
2852
2853   /**
2854    * Number of conns per peer
2855    */
2856   unsigned int num_to_add;
2857
2858   /**
2859    * Permuted array of all possible connections.  Only add the Nth
2860    * peer if it's in the Nth position.
2861    */
2862   unsigned int *pg_array;
2863
2864   /**
2865    * What number is the current element we are iterating over?
2866    */
2867   unsigned int current;
2868 };
2869
2870 struct DFSContext
2871 {
2872   /**
2873    * The peergroup
2874    */
2875   struct GNUNET_TESTING_PeerGroup *pg;
2876
2877   /**
2878    * uid of the first peer
2879    */
2880   uint32_t first_uid;
2881
2882   /**
2883    * uid of the second peer
2884    */
2885   uint32_t second_uid;
2886
2887   /**
2888    * Peer data for first peer.
2889    */
2890   struct PeerData *first;
2891
2892   /**
2893    * Which peer has been chosen as the one to add?
2894    */
2895   unsigned int chosen;
2896
2897   /**
2898    * What number is the current element we are iterating over?
2899    */
2900   unsigned int current;
2901 };
2902
2903 /**
2904  * Iterator for choosing random peers to connect.
2905  *
2906  * @param cls closure, a RandomContext
2907  * @param key the key the second Daemon was stored under
2908  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2909  *
2910  * @return GNUNET_YES to continue iteration
2911  */
2912 static int
2913 random_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2914 {
2915   struct RandomContext *random_ctx = cls;
2916   double random_number;
2917   uint32_t second_pos;
2918   GNUNET_HashCode first_hash;
2919   random_number =
2920     ((double)
2921      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
2922                                UINT64_MAX)) / ((double) UINT64_MAX);
2923   if (random_number < random_ctx->percentage)
2924     {
2925       GNUNET_assert (GNUNET_OK ==
2926                      GNUNET_CONTAINER_multihashmap_put (random_ctx->
2927                                                         first->connect_peers_working_set,
2928                                                         key, value,
2929                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2930     }
2931   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2932   uid_from_hash (key, &second_pos);
2933   hash_from_uid (random_ctx->first_uid, &first_hash);
2934   GNUNET_assert (random_ctx->pg->total > second_pos);
2935   GNUNET_assert (GNUNET_YES ==
2936                  GNUNET_CONTAINER_multihashmap_remove (random_ctx->
2937                                                        pg->peers
2938                                                        [second_pos].connect_peers,
2939                                                        &first_hash,
2940                                                        random_ctx->
2941                                                        first->daemon));
2942
2943   return GNUNET_YES;
2944 }
2945
2946 /**
2947  * Iterator for adding at least X peers to a peers connection set.
2948  *
2949  * @param cls closure, MinimumContext
2950  * @param key the key the second Daemon was stored under
2951  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2952  *
2953  * @return GNUNET_YES to continue iteration
2954  */
2955 static int
2956 minimum_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2957 {
2958   struct MinimumContext *min_ctx = cls;
2959   uint32_t second_pos;
2960   GNUNET_HashCode first_hash;
2961   unsigned int i;
2962
2963   if (GNUNET_CONTAINER_multihashmap_size
2964       (min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2965     {
2966       for (i = 0; i < min_ctx->num_to_add; i++)
2967         {
2968           if (min_ctx->pg_array[i] == min_ctx->current)
2969             {
2970               GNUNET_assert (GNUNET_OK ==
2971                              GNUNET_CONTAINER_multihashmap_put
2972                              (min_ctx->first->connect_peers_working_set, key,
2973                               value,
2974                               GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2975               uid_from_hash (key, &second_pos);
2976               hash_from_uid (min_ctx->first_uid, &first_hash);
2977               GNUNET_assert (min_ctx->pg->total > second_pos);
2978               GNUNET_assert (GNUNET_OK ==
2979                              GNUNET_CONTAINER_multihashmap_put (min_ctx->
2980                                                                 pg->peers
2981                                                                 [second_pos].connect_peers_working_set,
2982                                                                 &first_hash,
2983                                                                 min_ctx->first->
2984                                                                 daemon,
2985                                                                 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2986               /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2987               GNUNET_assert (GNUNET_YES ==
2988                              GNUNET_CONTAINER_multihashmap_remove
2989                              (min_ctx->pg->peers[second_pos].connect_peers,
2990                               &first_hash, min_ctx->first->daemon));
2991             }
2992         }
2993       min_ctx->current++;
2994       return GNUNET_YES;
2995     }
2996   else
2997     return GNUNET_NO;           /* We can stop iterating, we have enough peers! */
2998
2999 }
3000
3001
3002 /**
3003  * Iterator for adding peers to a connection set based on a depth first search.
3004  *
3005  * @param cls closure, MinimumContext
3006  * @param key the key the second daemon was stored under
3007  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
3008  *
3009  * @return GNUNET_YES to continue iteration
3010  */
3011 static int
3012 dfs_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
3013 {
3014   struct DFSContext *dfs_ctx = cls;
3015   GNUNET_HashCode first_hash;
3016
3017   if (dfs_ctx->current == dfs_ctx->chosen)
3018     {
3019       GNUNET_assert (GNUNET_OK ==
3020                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3021                                                         first->connect_peers_working_set,
3022                                                         key, value,
3023                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3024       uid_from_hash (key, &dfs_ctx->second_uid);
3025       hash_from_uid (dfs_ctx->first_uid, &first_hash);
3026       GNUNET_assert (GNUNET_OK ==
3027                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3028                                                         pg->peers
3029                                                         [dfs_ctx->second_uid].connect_peers_working_set,
3030                                                         &first_hash,
3031                                                         dfs_ctx->
3032                                                         first->daemon,
3033                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3034       GNUNET_assert (GNUNET_YES ==
3035                      GNUNET_CONTAINER_multihashmap_remove (dfs_ctx->
3036                                                            pg->peers
3037                                                            [dfs_ctx->second_uid].connect_peers,
3038                                                            &first_hash,
3039                                                            dfs_ctx->
3040                                                            first->daemon));
3041       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
3042       return GNUNET_NO;         /* We have found our peer, don't iterate more */
3043     }
3044
3045   dfs_ctx->current++;
3046   return GNUNET_YES;
3047 }
3048
3049
3050 /**
3051  * From the set of connections possible, choose percentage percent of connections
3052  * to actually connect.
3053  *
3054  * @param pg the peergroup we are dealing with
3055  * @param percentage what percent of total connections to make
3056  */
3057 void
3058 choose_random_connections (struct GNUNET_TESTING_PeerGroup *pg,
3059                            double percentage)
3060 {
3061   struct RandomContext random_ctx;
3062   uint32_t pg_iter;
3063
3064   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3065     {
3066       random_ctx.first_uid = pg_iter;
3067       random_ctx.first = &pg->peers[pg_iter];
3068       random_ctx.percentage = percentage;
3069       random_ctx.pg = pg;
3070       pg->peers[pg_iter].connect_peers_working_set =
3071         GNUNET_CONTAINER_multihashmap_create (pg->total);
3072       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3073                                              &random_connect_iterator,
3074                                              &random_ctx);
3075       /* Now remove the old connections */
3076       GNUNET_CONTAINER_multihashmap_destroy (pg->
3077                                              peers[pg_iter].connect_peers);
3078       /* And replace with the random set */
3079       pg->peers[pg_iter].connect_peers =
3080         pg->peers[pg_iter].connect_peers_working_set;
3081     }
3082 }
3083
3084 /**
3085  * From the set of connections possible, choose at least num connections per
3086  * peer.
3087  *
3088  * @param pg the peergroup we are dealing with
3089  * @param num how many connections at least should each peer have (if possible)?
3090  */
3091 static void
3092 choose_minimum (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3093 {
3094   struct MinimumContext minimum_ctx;
3095   uint32_t pg_iter;
3096
3097   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3098     {
3099       pg->peers[pg_iter].connect_peers_working_set =
3100         GNUNET_CONTAINER_multihashmap_create (num);
3101     }
3102
3103   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3104     {
3105       minimum_ctx.first_uid = pg_iter;
3106       minimum_ctx.pg_array =
3107         GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK,
3108                                       GNUNET_CONTAINER_multihashmap_size
3109                                       (pg->peers[pg_iter].connect_peers));
3110       minimum_ctx.first = &pg->peers[pg_iter];
3111       minimum_ctx.pg = pg;
3112       minimum_ctx.num_to_add = num;
3113       minimum_ctx.current = 0;
3114       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3115                                              &minimum_connect_iterator,
3116                                              &minimum_ctx);
3117     }
3118
3119   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3120     {
3121       /* Remove the "old" connections */
3122       GNUNET_CONTAINER_multihashmap_destroy (pg->
3123                                              peers[pg_iter].connect_peers);
3124       /* And replace with the working set */
3125       pg->peers[pg_iter].connect_peers =
3126         pg->peers[pg_iter].connect_peers_working_set;
3127     }
3128
3129 }
3130
3131
3132 static unsigned int
3133 count_workingset_connections (struct GNUNET_TESTING_PeerGroup *pg)
3134 {
3135   unsigned int count;
3136   unsigned int pg_iter;
3137
3138   count = 0;
3139
3140   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3141     {
3142       count +=
3143         GNUNET_CONTAINER_multihashmap_size (pg->
3144                                             peers
3145                                             [pg_iter].connect_peers_working_set);
3146     }
3147
3148   return count;
3149 }
3150
3151
3152 static unsigned int
3153 count_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg)
3154 {
3155   unsigned int count;
3156   unsigned int pg_iter;
3157
3158   count = 0;
3159
3160   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3161     {
3162       count +=
3163         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
3164     }
3165
3166   return count;
3167 }
3168
3169
3170 struct FindClosestContext
3171 {
3172   /**
3173    * The currently known closest peer.
3174    */
3175   struct GNUNET_TESTING_Daemon *closest;
3176
3177   /**
3178    * The info for the peer we are adding connections for.
3179    */
3180   struct PeerData *curr_peer;
3181
3182   /**
3183    * The distance (bits) between the current
3184    * peer and the currently known closest.
3185    */
3186   unsigned int closest_dist;
3187
3188   /**
3189    * The offset of the closest known peer in
3190    * the peer group.
3191    */
3192   unsigned int closest_num;
3193 };
3194
3195 /**
3196  * Iterator over hash map entries of the allowed
3197  * peer connections.  Find the closest, not already
3198  * connected peer and return it.
3199  *
3200  * @param cls closure (struct FindClosestContext)
3201  * @param key current key code (hash of offset in pg)
3202  * @param value value in the hash map - a GNUNET_TESTING_Daemon
3203  * @return GNUNET_YES if we should continue to
3204  *         iterate,
3205  *         GNUNET_NO if not.
3206  */
3207 static int
3208 find_closest_peers (void *cls, const GNUNET_HashCode * key, void *value)
3209 {
3210   struct FindClosestContext *closest_ctx = cls;
3211   struct GNUNET_TESTING_Daemon *daemon = value;
3212
3213   if (((closest_ctx->closest == NULL) ||
3214        (GNUNET_CRYPTO_hash_matching_bits
3215         (&daemon->id.hashPubKey,
3216          &closest_ctx->curr_peer->daemon->id.hashPubKey) >
3217         closest_ctx->closest_dist))
3218       && (GNUNET_YES !=
3219           GNUNET_CONTAINER_multihashmap_contains (closest_ctx->
3220                                                   curr_peer->connect_peers,
3221                                                   key)))
3222     {
3223       closest_ctx->closest_dist =
3224         GNUNET_CRYPTO_hash_matching_bits (&daemon->id.hashPubKey,
3225                                           &closest_ctx->curr_peer->daemon->
3226                                           id.hashPubKey);
3227       closest_ctx->closest = daemon;
3228       uid_from_hash (key, &closest_ctx->closest_num);
3229     }
3230   return GNUNET_YES;
3231 }
3232
3233 /**
3234  * From the set of connections possible, choose at num connections per
3235  * peer based on depth which are closest out of those allowed.  Guaranteed
3236  * to add num peers to connect to, provided there are that many peers
3237  * in the underlay topology to connect to.
3238  *
3239  * @param pg the peergroup we are dealing with
3240  * @param num how many connections at least should each peer have (if possible)?
3241  * @param proc processor to actually add the connections
3242  */
3243 void
3244 add_closest (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num,
3245              GNUNET_TESTING_ConnectionProcessor proc)
3246 {
3247   struct FindClosestContext closest_ctx;
3248   uint32_t pg_iter;
3249   uint32_t i;
3250
3251   for (i = 0; i < num; i++)     /* Each time find a closest peer (from those available) */
3252     {
3253       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3254         {
3255           closest_ctx.curr_peer = &pg->peers[pg_iter];
3256           closest_ctx.closest = NULL;
3257           closest_ctx.closest_dist = 0;
3258           closest_ctx.closest_num = 0;
3259           GNUNET_CONTAINER_multihashmap_iterate (pg->
3260                                                  peers[pg_iter].allowed_peers,
3261                                                  &find_closest_peers,
3262                                                  &closest_ctx);
3263           if (closest_ctx.closest != NULL)
3264             {
3265               GNUNET_assert (closest_ctx.closest_num < pg->total);
3266               proc (pg, pg_iter, closest_ctx.closest_num);
3267             }
3268         }
3269     }
3270 }
3271
3272 /**
3273  * From the set of connections possible, choose at least num connections per
3274  * peer based on depth first traversal of peer connections.  If DFS leaves
3275  * peers unconnected, ensure those peers get connections.
3276  *
3277  * @param pg the peergroup we are dealing with
3278  * @param num how many connections at least should each peer have (if possible)?
3279  */
3280 void
3281 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3282 {
3283   struct DFSContext dfs_ctx;
3284   uint32_t pg_iter;
3285   uint32_t dfs_count;
3286   uint32_t starting_peer;
3287   uint32_t least_connections;
3288   GNUNET_HashCode second_hash;
3289
3290   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3291     {
3292       pg->peers[pg_iter].connect_peers_working_set =
3293         GNUNET_CONTAINER_multihashmap_create (num);
3294     }
3295
3296   starting_peer = 0;
3297   dfs_count = 0;
3298   while ((count_workingset_connections (pg) < num * pg->total)
3299          && (count_allowed_connections (pg) > 0))
3300     {
3301       if (dfs_count % pg->total == 0)   /* Restart the DFS at some weakly connected peer */
3302         {
3303           least_connections = -1;       /* Set to very high number */
3304           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3305             {
3306               if (GNUNET_CONTAINER_multihashmap_size
3307                   (pg->peers[pg_iter].connect_peers_working_set) <
3308                   least_connections)
3309                 {
3310                   starting_peer = pg_iter;
3311                   least_connections =
3312                     GNUNET_CONTAINER_multihashmap_size (pg->
3313                                                         peers
3314                                                         [pg_iter].connect_peers_working_set);
3315                 }
3316             }
3317         }
3318
3319       if (GNUNET_CONTAINER_multihashmap_size (pg->peers[starting_peer].connect_peers) == 0)     /* Ensure there is at least one peer left to connect! */
3320         {
3321           dfs_count = 0;
3322           continue;
3323         }
3324
3325       /* Choose a random peer from the chosen peers set of connections to add */
3326       dfs_ctx.chosen =
3327         GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
3328                                   GNUNET_CONTAINER_multihashmap_size
3329                                   (pg->peers[starting_peer].connect_peers));
3330       dfs_ctx.first_uid = starting_peer;
3331       dfs_ctx.first = &pg->peers[starting_peer];
3332       dfs_ctx.pg = pg;
3333       dfs_ctx.current = 0;
3334
3335       GNUNET_CONTAINER_multihashmap_iterate (pg->
3336                                              peers
3337                                              [starting_peer].connect_peers,
3338                                              &dfs_connect_iterator, &dfs_ctx);
3339       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
3340       hash_from_uid (dfs_ctx.second_uid, &second_hash);
3341       GNUNET_assert (GNUNET_YES ==
3342                      GNUNET_CONTAINER_multihashmap_remove (pg->peers
3343                                                            [starting_peer].connect_peers,
3344                                                            &second_hash,
3345                                                            pg->
3346                                                            peers
3347                                                            [dfs_ctx.second_uid].daemon));
3348       starting_peer = dfs_ctx.second_uid;
3349     }
3350
3351   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3352     {
3353       /* Remove the "old" connections */
3354       GNUNET_CONTAINER_multihashmap_destroy (pg->
3355                                              peers[pg_iter].connect_peers);
3356       /* And replace with the working set */
3357       pg->peers[pg_iter].connect_peers =
3358         pg->peers[pg_iter].connect_peers_working_set;
3359     }
3360 }
3361
3362 /**
3363  * Internal callback for topology information for a particular peer.
3364  */
3365 static void
3366 internal_topology_callback (void *cls,
3367                             const struct GNUNET_PeerIdentity *peer,
3368                             const struct GNUNET_TRANSPORT_ATS_Information
3369                             *atsi)
3370 {
3371   struct CoreContext *core_ctx = cls;
3372   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
3373
3374   if (peer == NULL)             /* Either finished, or something went wrong */
3375     {
3376       iter_ctx->completed++;
3377       iter_ctx->connected--;
3378       /* One core context allocated per iteration, must free! */
3379       GNUNET_free (core_ctx);
3380     }
3381   else
3382     {
3383       iter_ctx->topology_cb (iter_ctx->cls, &core_ctx->daemon->id,
3384                              peer, NULL);
3385     }
3386
3387   if (iter_ctx->completed == iter_ctx->total)
3388     {
3389       iter_ctx->topology_cb (iter_ctx->cls, NULL, NULL, NULL);
3390       /* Once all are done, free the iteration context */
3391       GNUNET_free (iter_ctx);
3392     }
3393 }
3394
3395
3396 /**
3397  * Check running topology iteration tasks, if below max start a new one, otherwise
3398  * schedule for some time in the future.
3399  */
3400 static void
3401 schedule_get_topology (void *cls,
3402                        const struct GNUNET_SCHEDULER_TaskContext *tc)
3403 {
3404   struct CoreContext *core_context = cls;
3405   struct TopologyIterateContext *topology_context =
3406     (struct TopologyIterateContext *) core_context->iter_context;
3407   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3408     return;
3409
3410   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3411     {
3412 #if VERBOSE_TESTING > 2
3413       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3414                   _
3415                   ("Delaying connect, we have too many outstanding connections!\n"));
3416 #endif
3417       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3418                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3419                                     &schedule_get_topology, core_context);
3420     }
3421   else
3422     {
3423 #if VERBOSE_TESTING > 2
3424       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3425                   _("Creating connection, outstanding_connections is %d\n"),
3426                   outstanding_connects);
3427 #endif
3428       topology_context->connected++;
3429
3430       if (GNUNET_OK !=
3431           GNUNET_CORE_iterate_peers (core_context->daemon->cfg,
3432                                      &internal_topology_callback,
3433                                      core_context))
3434         {
3435           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Topology iteration failed.\n");
3436           internal_topology_callback (core_context, NULL, NULL);
3437         }
3438     }
3439 }
3440
3441 /**
3442  * Iterate over all (running) peers in the peer group, retrieve
3443  * all connections that each currently has.
3444  */
3445 void
3446 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
3447                              GNUNET_TESTING_NotifyTopology cb, void *cls)
3448 {
3449   struct TopologyIterateContext *topology_context;
3450   struct CoreContext *core_ctx;
3451   unsigned int i;
3452   unsigned int total_count;
3453
3454   /* Allocate a single topology iteration context */
3455   topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
3456   topology_context->topology_cb = cb;
3457   topology_context->cls = cls;
3458   total_count = 0;
3459   for (i = 0; i < pg->total; i++)
3460     {
3461       if (pg->peers[i].daemon->running == GNUNET_YES)
3462         {
3463           /* Allocate one core context per core we need to connect to */
3464           core_ctx = GNUNET_malloc (sizeof (struct CoreContext));
3465           core_ctx->daemon = pg->peers[i].daemon;
3466           /* Set back pointer to topology iteration context */
3467           core_ctx->iter_context = topology_context;
3468           GNUNET_SCHEDULER_add_now (&schedule_get_topology, core_ctx);
3469           total_count++;
3470         }
3471     }
3472   if (total_count == 0)
3473     {
3474       cb (cls, NULL, NULL, "Cannot iterate over topology, no running peers!");
3475       GNUNET_free (topology_context);
3476     }
3477   else
3478     topology_context->total = total_count;
3479   return;
3480 }
3481
3482 /**
3483  * Callback function to process statistic values.
3484  * This handler is here only really to insert a peer
3485  * identity (or daemon) so the statistics can be uniquely
3486  * tied to a single running peer.
3487  *
3488  * @param cls closure
3489  * @param subsystem name of subsystem that created the statistic
3490  * @param name the name of the datum
3491  * @param value the current value
3492  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
3493  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
3494  */
3495 static int
3496 internal_stats_callback (void *cls,
3497                          const char *subsystem,
3498                          const char *name, uint64_t value, int is_persistent)
3499 {
3500   struct StatsCoreContext *core_context = cls;
3501   struct StatsIterateContext *stats_context =
3502     (struct StatsIterateContext *) core_context->iter_context;
3503
3504   return stats_context->proc (stats_context->cls, &core_context->daemon->id,
3505                               subsystem, name, value, is_persistent);
3506 }
3507
3508 /**
3509  * Internal continuation call for statistics iteration.
3510  *
3511  * @param cls closure, the CoreContext for this iteration
3512  * @param success whether or not the statistics iterations
3513  *        was canceled or not (we don't care)
3514  */
3515 static void
3516 internal_stats_cont (void *cls, int success)
3517 {
3518   struct StatsCoreContext *core_context = cls;
3519   struct StatsIterateContext *stats_context =
3520     (struct StatsIterateContext *) core_context->iter_context;
3521
3522   stats_context->connected--;
3523   stats_context->completed++;
3524
3525   if (stats_context->completed == stats_context->total)
3526     {
3527       stats_context->cont (stats_context->cls, GNUNET_YES);
3528       GNUNET_free (stats_context);
3529     }
3530
3531   if (core_context->stats_handle != NULL)
3532     GNUNET_STATISTICS_destroy (core_context->stats_handle, GNUNET_NO);
3533
3534   GNUNET_free (core_context);
3535 }
3536
3537 /**
3538  * Check running topology iteration tasks, if below max start a new one, otherwise
3539  * schedule for some time in the future.
3540  */
3541 static void
3542 schedule_get_statistics (void *cls,
3543                          const struct GNUNET_SCHEDULER_TaskContext *tc)
3544 {
3545   struct StatsCoreContext *core_context = cls;
3546   struct StatsIterateContext *stats_context =
3547     (struct StatsIterateContext *) core_context->iter_context;
3548
3549   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3550     return;
3551
3552   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3553     {
3554 #if VERBOSE_TESTING > 2
3555       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3556                   _
3557                   ("Delaying connect, we have too many outstanding connections!\n"));
3558 #endif
3559       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3560                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3561                                     &schedule_get_statistics, core_context);
3562     }
3563   else
3564     {
3565 #if VERBOSE_TESTING > 2
3566       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3567                   _("Creating connection, outstanding_connections is %d\n"),
3568                   outstanding_connects);
3569 #endif
3570
3571       stats_context->connected++;
3572       core_context->stats_handle =
3573         GNUNET_STATISTICS_create ("testing", core_context->daemon->cfg);
3574       if (core_context->stats_handle == NULL)
3575         {
3576           internal_stats_cont (core_context, GNUNET_NO);
3577           return;
3578         }
3579
3580       core_context->stats_get_handle =
3581         GNUNET_STATISTICS_get (core_context->stats_handle, NULL, NULL,
3582                                GNUNET_TIME_relative_get_forever (),
3583                                &internal_stats_cont, &internal_stats_callback,
3584                                core_context);
3585       if (core_context->stats_get_handle == NULL)
3586         internal_stats_cont (core_context, GNUNET_NO);
3587
3588     }
3589 }
3590
3591 struct DuplicateStats
3592 {
3593   /**
3594    * Next item in the list
3595    */
3596   struct DuplicateStats *next;
3597
3598   /**
3599    * Nasty string, concatenation of relevant information.
3600    */
3601   char *unique_string;
3602 };
3603
3604 /**
3605  * Check whether the combination of port/host/unix domain socket
3606  * already exists in the list of peers being checked for statistics.
3607  *
3608  * @param pg the peergroup in question
3609  * @param specific_peer the peer we're concerned with
3610  * @param stats_list the list to return to the caller
3611  *
3612  * @return GNUNET_YES if the statistics instance has been seen already,
3613  *         GNUNET_NO if not (and we may have added it to the list)
3614  */
3615 static int
3616 stats_check_existing (struct GNUNET_TESTING_PeerGroup *pg,
3617                       struct PeerData *specific_peer,
3618                       struct DuplicateStats **stats_list)
3619 {
3620   struct DuplicateStats *pos;
3621   char *unix_domain_socket;
3622   unsigned long long port;
3623   char *to_match;
3624   if (GNUNET_YES !=
3625       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "testing",
3626                                             "single_statistics_per_host"))
3627     return GNUNET_NO;           /* Each peer has its own statistics instance, do nothing! */
3628
3629   pos = *stats_list;
3630   if (GNUNET_OK !=
3631       GNUNET_CONFIGURATION_get_value_string (specific_peer->cfg, "statistics",
3632                                              "unixpath", &unix_domain_socket))
3633     return GNUNET_NO;
3634
3635   if (GNUNET_OK !=
3636       GNUNET_CONFIGURATION_get_value_number (specific_peer->cfg, "statistics",
3637                                              "port", &port))
3638     {
3639       GNUNET_free(unix_domain_socket);
3640       return GNUNET_NO;
3641     }
3642
3643   if (specific_peer->daemon->hostname != NULL)
3644     GNUNET_asprintf (&to_match, "%s%s%llu", specific_peer->daemon->hostname,
3645                      unix_domain_socket, port);
3646   else
3647     GNUNET_asprintf (&to_match, "%s%llu", unix_domain_socket, port);
3648
3649   while (pos != NULL)
3650     {
3651       if (0 == strcmp (to_match, pos->unique_string))
3652         {
3653           GNUNET_free (unix_domain_socket);
3654           GNUNET_free (to_match);
3655           return GNUNET_YES;
3656         }
3657       pos = pos->next;
3658     }
3659   pos = GNUNET_malloc (sizeof (struct DuplicateStats));
3660   pos->unique_string = to_match;
3661   pos->next = *stats_list;
3662   *stats_list = pos;
3663   GNUNET_free (unix_domain_socket);
3664   return GNUNET_NO;
3665 }
3666
3667 /**
3668  * Iterate over all (running) peers in the peer group, retrieve
3669  * all statistics from each.
3670  */
3671 void
3672 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3673                                GNUNET_STATISTICS_Callback cont,
3674                                GNUNET_TESTING_STATISTICS_Iterator proc,
3675                                void *cls)
3676 {
3677   struct StatsIterateContext *stats_context;
3678   struct StatsCoreContext *core_ctx;
3679   unsigned int i;
3680   unsigned int total_count;
3681   struct DuplicateStats *stats_list;
3682   struct DuplicateStats *pos;
3683   stats_list = NULL;
3684
3685   /* Allocate a single stats iteration context */
3686   stats_context = GNUNET_malloc (sizeof (struct StatsIterateContext));
3687   stats_context->cont = cont;
3688   stats_context->proc = proc;
3689   stats_context->cls = cls;
3690   total_count = 0;
3691
3692   for (i = 0; i < pg->total; i++)
3693     {
3694       if ((pg->peers[i].daemon->running == GNUNET_YES)
3695           && (GNUNET_NO ==
3696               stats_check_existing (pg, &pg->peers[i], &stats_list)))
3697         {
3698           /* Allocate one core context per core we need to connect to */
3699           core_ctx = GNUNET_malloc (sizeof (struct StatsCoreContext));
3700           core_ctx->daemon = pg->peers[i].daemon;
3701           /* Set back pointer to topology iteration context */
3702           core_ctx->iter_context = stats_context;
3703           GNUNET_SCHEDULER_add_now (&schedule_get_statistics, core_ctx);
3704           total_count++;
3705         }
3706     }
3707
3708   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3709               "Retrieving stats from %u total instances.\n", total_count);
3710   stats_context->total = total_count;
3711   if (stats_list != NULL)
3712     {
3713       pos = stats_list;
3714       while (pos != NULL)
3715         {
3716           GNUNET_free (pos->unique_string);
3717           stats_list = pos->next;
3718           GNUNET_free (pos);
3719           pos = stats_list->next;
3720         }
3721     }
3722   return;
3723 }
3724
3725 /**
3726  * There are many ways to connect peers that are supported by this function.
3727  * To connect peers in the same topology that was created via the
3728  * GNUNET_TESTING_create_topology, the topology variable must be set to
3729  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3730  * a new instance of that topology will be generated and attempted to be
3731  * connected.  This could result in some connections being impossible,
3732  * because some topologies are non-deterministic.
3733  *
3734  * @param pg the peer group struct representing the running peers
3735  * @param topology which topology to connect the peers in
3736  * @param options options for connecting the topology
3737  * @param option_modifier modifier for options that take a parameter
3738  * @param notify_callback notification to be called once all connections completed
3739  * @param notify_cls closure for notification callback
3740  *
3741  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3742  */
3743 int
3744 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3745                                  enum GNUNET_TESTING_Topology topology,
3746                                  enum GNUNET_TESTING_TopologyOption options,
3747                                  double option_modifier,
3748                                  GNUNET_TESTING_NotifyCompletion
3749                                  notify_callback, void *notify_cls)
3750 {
3751   switch (topology)
3752     {
3753     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3754 #if VERBOSE_TOPOLOGY
3755       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3756                   _("Creating clique CONNECT topology\n"));
3757 #endif
3758       create_clique (pg, &add_actual_connections);
3759       break;
3760     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3761 #if VERBOSE_TOPOLOGY
3762       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3763                   _("Creating small world (ring) CONNECT topology\n"));
3764 #endif
3765       create_small_world_ring (pg, &add_actual_connections);
3766       break;
3767     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3768 #if VERBOSE_TOPOLOGY
3769       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3770                   _("Creating small world (2d-torus) CONNECT topology\n"));
3771 #endif
3772       create_small_world (pg, &add_actual_connections);
3773       break;
3774     case GNUNET_TESTING_TOPOLOGY_RING:
3775 #if VERBOSE_TOPOLOGY
3776       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3777                   _("Creating ring CONNECT topology\n"));
3778 #endif
3779       create_ring (pg, &add_actual_connections);
3780       break;
3781     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3782 #if VERBOSE_TOPOLOGY
3783       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3784                   _("Creating 2d torus CONNECT topology\n"));
3785 #endif
3786       create_2d_torus (pg, &add_actual_connections);
3787       break;
3788     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3789 #if VERBOSE_TOPOLOGY
3790       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3791                   _("Creating Erdos-Renyi CONNECT topology\n"));
3792 #endif
3793       create_erdos_renyi (pg, &add_actual_connections);
3794       break;
3795     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3796 #if VERBOSE_TOPOLOGY
3797       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3798                   _("Creating InterNAT CONNECT topology\n"));
3799 #endif
3800       create_nated_internet (pg, &add_actual_connections);
3801       break;
3802     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3803 #if VERBOSE_TOPOLOGY
3804       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3805                   _("Creating Scale Free CONNECT topology\n"));
3806 #endif
3807       create_scale_free (pg, &add_actual_connections);
3808       break;
3809     case GNUNET_TESTING_TOPOLOGY_LINE:
3810 #if VERBOSE_TOPOLOGY
3811       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3812                   _("Creating straight line CONNECT topology\n"));
3813 #endif
3814       create_line (pg, &add_actual_connections);
3815       break;
3816     case GNUNET_TESTING_TOPOLOGY_NONE:
3817 #if VERBOSE_TOPOLOGY
3818       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3819                   _("Creating no CONNECT topology\n"));
3820 #endif
3821       copy_allowed_topology (pg);
3822       break;
3823     default:
3824       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3825                   _
3826                   ("Unknown topology specification, can't connect peers!\n"));
3827       return GNUNET_SYSERR;
3828     }
3829
3830   switch (options)
3831     {
3832     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3833 #if VERBOSE_TOPOLOGY
3834       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3835                   _
3836                   ("Connecting random subset (%'.2f percent) of possible peers\n"),
3837                   100 * option_modifier);
3838 #endif
3839       choose_random_connections (pg, option_modifier);
3840       break;
3841     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3842 #if VERBOSE_TOPOLOGY
3843       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3844                   _("Connecting a minimum of %u peers each (if possible)\n"),
3845                   (unsigned int) option_modifier);
3846 #endif
3847       choose_minimum (pg, (unsigned int) option_modifier);
3848       break;
3849     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3850 #if VERBOSE_TOPOLOGY
3851       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3852                   _
3853                   ("Using DFS to connect a minimum of %u peers each (if possible)\n"),
3854                   (unsigned int) option_modifier);
3855 #endif
3856       perform_dfs (pg, (int) option_modifier);
3857       break;
3858     case GNUNET_TESTING_TOPOLOGY_OPTION_ADD_CLOSEST:
3859 #if VERBOSE_TOPOLOGY
3860       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3861                   _
3862                   ("Finding additional %u closest peers each (if possible)\n"),
3863                   (unsigned int) option_modifier);
3864 #endif
3865       add_closest (pg, (unsigned int) option_modifier,
3866                    &add_actual_connections);
3867       break;
3868     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3869       break;
3870     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3871       break;
3872     default:
3873       break;
3874     }
3875
3876   return connect_topology (pg, notify_callback, notify_cls);
3877 }
3878
3879 /**
3880  * Callback that is called whenever a hostkey is generated
3881  * for a peer.  Call the real callback and decrement the
3882  * starting counter for the peergroup.
3883  *
3884  * @param cls closure
3885  * @param id identifier for the daemon, NULL on error
3886  * @param d handle for the daemon
3887  * @param emsg error message (NULL on success)
3888  */
3889 static void
3890 internal_hostkey_callback (void *cls,
3891                            const struct GNUNET_PeerIdentity *id,
3892                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3893 {
3894   struct InternalStartContext *internal_context = cls;
3895   internal_context->peer->pg->starting--;
3896   internal_context->peer->pg->started++;
3897   if (internal_context->hostkey_callback != NULL)
3898     internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
3899                                         emsg);
3900   else if (internal_context->peer->pg->started ==
3901            internal_context->peer->pg->total)
3902     {
3903       internal_context->peer->pg->started = 0;  /* Internal startup may use this counter! */
3904       GNUNET_TESTING_daemons_continue_startup (internal_context->peer->pg);
3905     }
3906 }
3907
3908 /**
3909  * Callback that is called whenever a peer has finished starting.
3910  * Call the real callback and decrement the starting counter
3911  * for the peergroup.
3912  *
3913  * @param cls closure
3914  * @param id identifier for the daemon, NULL on error
3915  * @param cfg config
3916  * @param d handle for the daemon
3917  * @param emsg error message (NULL on success)
3918  */
3919 static void
3920 internal_startup_callback (void *cls,
3921                            const struct GNUNET_PeerIdentity *id,
3922                            const struct GNUNET_CONFIGURATION_Handle *cfg,
3923                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3924 {
3925   struct InternalStartContext *internal_context = cls;
3926   internal_context->peer->pg->starting--;
3927   if (internal_context->start_cb != NULL)
3928     internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
3929                                 emsg);
3930 }
3931
3932 static void
3933 internal_continue_startup (void *cls,
3934                            const struct GNUNET_SCHEDULER_TaskContext *tc)
3935 {
3936   struct InternalStartContext *internal_context = cls;
3937
3938   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3939     {
3940       return;
3941     }
3942
3943   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3944     {
3945       internal_context->peer->pg->starting++;
3946       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3947     }
3948   else
3949     {
3950       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3951                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3952                                     &internal_continue_startup,
3953                                     internal_context);
3954     }
3955 }
3956
3957
3958 /**
3959  * Callback for informing us about a successful
3960  * or unsuccessful churn start call.
3961  *
3962  * @param cls a ChurnContext
3963  * @param id the peer identity of the started peer
3964  * @param cfg the handle to the configuration of the peer
3965  * @param d handle to the daemon for the peer
3966  * @param emsg NULL on success, non-NULL on failure
3967  *
3968  */
3969 void
3970 churn_start_callback (void *cls,
3971                       const struct GNUNET_PeerIdentity *id,
3972                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3973                       struct GNUNET_TESTING_Daemon *d, const char *emsg)
3974 {
3975   struct ChurnRestartContext *startup_ctx = cls;
3976   struct ChurnContext *churn_ctx = startup_ctx->churn_ctx;
3977
3978   unsigned int total_left;
3979   char *error_message;
3980
3981   error_message = NULL;
3982   if (emsg != NULL)
3983     {
3984       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3985                   "Churn stop callback failed with error `%s'\n", emsg);
3986       churn_ctx->num_failed_start++;
3987     }
3988   else
3989     {
3990       churn_ctx->num_to_start--;
3991     }
3992
3993 #if DEBUG_CHURN
3994   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3995               "Started peer, %d left.\n", churn_ctx->num_to_start);
3996 #endif
3997
3998   total_left =
3999     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
4000     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
4001
4002   if (total_left == 0)
4003     {
4004       if ((churn_ctx->num_failed_stop > 0)
4005           || (churn_ctx->num_failed_start > 0))
4006         GNUNET_asprintf (&error_message,
4007                          "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4008                          churn_ctx->num_failed_start,
4009                          churn_ctx->num_failed_stop);
4010       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4011       GNUNET_free_non_null (error_message);
4012       GNUNET_free (churn_ctx);
4013       GNUNET_free (startup_ctx);
4014     }
4015 }
4016
4017
4018 static void
4019 schedule_churn_restart (void *cls,
4020                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4021 {
4022   struct PeerRestartContext *peer_restart_ctx = cls;
4023   struct ChurnRestartContext *startup_ctx =
4024     peer_restart_ctx->churn_restart_ctx;
4025
4026   if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
4027     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4028                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4029                                   &schedule_churn_restart, peer_restart_ctx);
4030   else
4031     {
4032       GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon,
4033                                            startup_ctx->timeout,
4034                                            &churn_start_callback,
4035                                            startup_ctx);
4036       GNUNET_free (peer_restart_ctx);
4037     }
4038 }
4039
4040 static void
4041 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
4042 {
4043   struct InternalStartContext *internal_context = cls;
4044
4045   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
4046     {
4047       return;
4048     }
4049
4050   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
4051     {
4052       internal_context->peer->pg->starting++;
4053       internal_context->peer->daemon =
4054         GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
4055                                      internal_context->timeout,
4056                                      internal_context->hostname,
4057                                      internal_context->username,
4058                                      internal_context->sshport,
4059                                      &internal_hostkey_callback,
4060                                      internal_context,
4061                                      &internal_startup_callback,
4062                                      internal_context);
4063     }
4064   else
4065     {
4066       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4067                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4068                                     &internal_start, internal_context);
4069     }
4070 }
4071
4072 /**
4073  * Function which continues a peer group starting up
4074  * after successfully generating hostkeys for each peer.
4075  *
4076  * @param pg the peer group to continue starting
4077  *
4078  */
4079 void
4080 GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
4081 {
4082   unsigned int i;
4083
4084   pg->starting = 0;
4085   for (i = 0; i < pg->total; i++)
4086     {
4087       GNUNET_SCHEDULER_add_now (&internal_continue_startup,
4088                                 &pg->peers[i].internal_context);
4089       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
4090     }
4091 }
4092
4093 /**
4094  * Start count gnunet instances with the same set of transports and
4095  * applications.  The port numbers (any option called "PORT") will be
4096  * adjusted to ensure that no two peers running on the same system
4097  * have the same port(s) in their respective configurations.
4098  *
4099  * @param cfg configuration template to use
4100  * @param total number of daemons to start
4101  * @param timeout total time allowed for peers to start
4102  * @param hostkey_callback function to call on each peers hostkey generation
4103  *        if NULL, peers will be started by this call, if non-null,
4104  *        GNUNET_TESTING_daemons_continue_startup must be called after
4105  *        successful hostkey generation
4106  * @param hostkey_cls closure for hostkey callback
4107  * @param cb function to call on each daemon that was started
4108  * @param cb_cls closure for cb
4109  * @param connect_callback function to call each time two hosts are connected
4110  * @param connect_callback_cls closure for connect_callback
4111  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
4112  *
4113  * @return NULL on error, otherwise handle to control peer group
4114  */
4115 struct GNUNET_TESTING_PeerGroup *
4116 GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
4117                               unsigned int total,
4118                               struct GNUNET_TIME_Relative timeout,
4119                               GNUNET_TESTING_NotifyHostkeyCreated
4120                               hostkey_callback, void *hostkey_cls,
4121                               GNUNET_TESTING_NotifyDaemonRunning cb,
4122                               void *cb_cls,
4123                               GNUNET_TESTING_NotifyConnection
4124                               connect_callback, void *connect_callback_cls,
4125                               const struct GNUNET_TESTING_Host *hostnames)
4126 {
4127   struct GNUNET_TESTING_PeerGroup *pg;
4128   const struct GNUNET_TESTING_Host *hostpos;
4129 #if 0
4130   char *pos;
4131   const char *rpos;
4132   char *start;
4133 #endif
4134   const char *hostname;
4135   const char *username;
4136   char *baseservicehome;
4137   char *newservicehome;
4138   char *tmpdir;
4139   struct GNUNET_CONFIGURATION_Handle *pcfg;
4140   unsigned int off;
4141   unsigned int hostcnt;
4142   uint16_t minport;
4143   uint16_t sshport;
4144   uint32_t upnum;
4145   uint32_t fdnum;
4146
4147   if (0 == total)
4148     {
4149       GNUNET_break (0);
4150       return NULL;
4151     }
4152   upnum = 0;
4153   fdnum = 0;
4154   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
4155   pg->cfg = cfg;
4156   pg->notify_connection = connect_callback;
4157   pg->notify_connection_cls = connect_callback_cls;
4158   pg->total = total;
4159   pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
4160   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
4161   if (NULL != hostnames)
4162     {
4163       off = 0;
4164       hostpos = hostnames;
4165       while (hostpos != NULL)
4166         {
4167           hostpos = hostpos->next;
4168           off++;
4169         }
4170       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4171       off = 0;
4172
4173       hostpos = hostnames;
4174       while (hostpos != NULL)
4175         {
4176           pg->hosts[off].minport = LOW_PORT;
4177           pg->hosts[off].hostname = GNUNET_strdup (hostpos->hostname);
4178           if (hostpos->username != NULL)
4179             pg->hosts[off].username = GNUNET_strdup (hostpos->username);
4180           pg->hosts[off].sshport = hostpos->port;
4181           hostpos = hostpos->next;
4182           off++;
4183         }
4184
4185       if (off == 0)
4186         {
4187           pg->hosts = NULL;
4188         }
4189       hostcnt = off;
4190       minport = 0;
4191       pg->num_hosts = off;
4192
4193 #if NO_LL
4194       off = 2;
4195       /* skip leading spaces */
4196       while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
4197         hostnames++;
4198       rpos = hostnames;
4199       while ('\0' != *rpos)
4200         {
4201           if (isspace ((unsigned char) *rpos))
4202             off++;
4203           rpos++;
4204         }
4205       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4206       off = 0;
4207       start = GNUNET_strdup (hostnames);
4208       pos = start;
4209       while ('\0' != *pos)
4210         {
4211           if (isspace ((unsigned char) *pos))
4212             {
4213               *pos = '\0';
4214               if (strlen (start) > 0)
4215                 {
4216                   pg->hosts[off].minport = LOW_PORT;
4217                   pg->hosts[off++].hostname = start;
4218                 }
4219               start = pos + 1;
4220             }
4221           pos++;
4222         }
4223       if (strlen (start) > 0)
4224         {
4225           pg->hosts[off].minport = LOW_PORT;
4226           pg->hosts[off++].hostname = start;
4227         }
4228       if (off == 0)
4229         {
4230           GNUNET_free (start);
4231           GNUNET_free (pg->hosts);
4232           pg->hosts = NULL;
4233         }
4234       hostcnt = off;
4235       minport = 0;              /* make gcc happy */
4236 #endif
4237     }
4238   else
4239     {
4240       hostcnt = 0;
4241       minport = LOW_PORT;
4242     }
4243   for (off = 0; off < total; off++)
4244     {
4245       if (hostcnt > 0)
4246         {
4247           hostname = pg->hosts[off % hostcnt].hostname;
4248           username = pg->hosts[off % hostcnt].username;
4249           sshport = pg->hosts[off % hostcnt].sshport;
4250           pcfg = make_config (cfg,
4251                               &pg->hosts[off % hostcnt].minport,
4252                               &upnum, hostname, &fdnum);
4253         }
4254       else
4255         {
4256           hostname = NULL;
4257           username = NULL;
4258           sshport = 0;
4259           pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
4260         }
4261
4262       if (NULL == pcfg)
4263         {
4264           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4265                       _
4266                       ("Could not create configuration for peer number %u on `%s'!\n"),
4267                       off, hostname == NULL ? "localhost" : hostname);
4268           continue;
4269         }
4270
4271       if (GNUNET_YES ==
4272           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
4273                                                  &baseservicehome))
4274         {
4275           GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
4276           GNUNET_free (baseservicehome);
4277         }
4278       else
4279         {
4280           tmpdir = getenv ("TMPDIR");
4281           tmpdir = tmpdir ? tmpdir : "/tmp";
4282           GNUNET_asprintf (&newservicehome,
4283                            "%s/%s/%d/",
4284                            tmpdir, "gnunet-testing-test-test", off);
4285         }
4286       GNUNET_CONFIGURATION_set_value_string (pcfg,
4287                                              "PATHS",
4288                                              "SERVICEHOME", newservicehome);
4289       GNUNET_free (newservicehome);
4290       pg->peers[off].cfg = pcfg;
4291       pg->peers[off].allowed_peers =
4292         GNUNET_CONTAINER_multihashmap_create (total);
4293       pg->peers[off].connect_peers =
4294         GNUNET_CONTAINER_multihashmap_create (total);
4295       pg->peers[off].blacklisted_peers =
4296         GNUNET_CONTAINER_multihashmap_create (total);
4297       pg->peers[off].pg = pg;
4298
4299       pg->peers[off].internal_context.peer = &pg->peers[off];
4300       pg->peers[off].internal_context.timeout = timeout;
4301       pg->peers[off].internal_context.hostname = hostname;
4302       pg->peers[off].internal_context.username = username;
4303       pg->peers[off].internal_context.sshport = sshport;
4304       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
4305       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
4306       pg->peers[off].internal_context.start_cb = cb;
4307       pg->peers[off].internal_context.start_cb_cls = cb_cls;
4308
4309       GNUNET_SCHEDULER_add_now (&internal_start,
4310                                 &pg->peers[off].internal_context);
4311
4312     }
4313   return pg;
4314 }
4315
4316 /*
4317  * Get a daemon by number, so callers don't have to do nasty
4318  * offsetting operation.
4319  */
4320 struct GNUNET_TESTING_Daemon *
4321 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg,
4322                            unsigned int position)
4323 {
4324   if (position < pg->total)
4325     return pg->peers[position].daemon;
4326   else
4327     return NULL;
4328 }
4329
4330 /*
4331  * Get a daemon by peer identity, so callers can
4332  * retrieve the daemon without knowing it's offset.
4333  *
4334  * @param pg the peer group to retrieve the daemon from
4335  * @param peer_id the peer identity of the daemon to retrieve
4336  *
4337  * @return the daemon on success, or NULL if no such peer identity is found
4338  */
4339 struct GNUNET_TESTING_Daemon *
4340 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg,
4341                                  struct GNUNET_PeerIdentity *peer_id)
4342 {
4343   unsigned int i;
4344
4345   for (i = 0; i < pg->total; i++)
4346     {
4347       if (0 ==
4348           memcmp (&pg->peers[i].daemon->id, peer_id,
4349                   sizeof (struct GNUNET_PeerIdentity)))
4350         return pg->peers[i].daemon;
4351     }
4352
4353   return NULL;
4354 }
4355
4356 /**
4357  * Prototype of a function that will be called when a
4358  * particular operation was completed the testing library.
4359  *
4360  * @param cls closure (a struct RestartContext)
4361  * @param id id of the peer that was restarted
4362  * @param cfg handle to the configuration of the peer
4363  * @param d handle to the daemon that was restarted
4364  * @param emsg NULL on success
4365  */
4366 void
4367 restart_callback (void *cls,
4368                   const struct GNUNET_PeerIdentity *id,
4369                   const struct GNUNET_CONFIGURATION_Handle *cfg,
4370                   struct GNUNET_TESTING_Daemon *d, const char *emsg)
4371 {
4372   struct RestartContext *restart_context = cls;
4373
4374   if (emsg == NULL)
4375     {
4376       restart_context->peers_restarted++;
4377     }
4378   else
4379     {
4380       restart_context->peers_restart_failed++;
4381     }
4382
4383   if (restart_context->peers_restarted == restart_context->peer_group->total)
4384     {
4385       restart_context->callback (restart_context->callback_cls, NULL);
4386       GNUNET_free (restart_context);
4387     }
4388   else if (restart_context->peers_restart_failed +
4389            restart_context->peers_restarted ==
4390            restart_context->peer_group->total)
4391     {
4392       restart_context->callback (restart_context->callback_cls,
4393                                  "Failed to restart peers!");
4394       GNUNET_free (restart_context);
4395     }
4396
4397 }
4398
4399 /**
4400  * Callback for informing us about a successful
4401  * or unsuccessful churn stop call.
4402  *
4403  * @param cls a ChurnContext
4404  * @param emsg NULL on success, non-NULL on failure
4405  *
4406  */
4407 void
4408 churn_stop_callback (void *cls, const char *emsg)
4409 {
4410   struct ShutdownContext *shutdown_ctx = cls;
4411   struct ChurnContext *churn_ctx = shutdown_ctx->cb_cls;
4412   unsigned int total_left;
4413   char *error_message;
4414
4415   error_message = NULL;
4416   shutdown_ctx->outstanding--;
4417
4418   if (emsg != NULL)
4419     {
4420       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4421                   "Churn stop callback failed with error `%s'\n", emsg);
4422       churn_ctx->num_failed_stop++;
4423     }
4424   else
4425     {
4426       churn_ctx->num_to_stop--;
4427     }
4428
4429 #if DEBUG_CHURN
4430   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4431               "Stopped peer, %d left.\n", churn_ctx->num_to_stop);
4432 #endif
4433   total_left =
4434     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
4435     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
4436
4437   if (total_left == 0)
4438     {
4439       if ((churn_ctx->num_failed_stop > 0)
4440           || (churn_ctx->num_failed_start > 0))
4441         {
4442           GNUNET_asprintf (&error_message,
4443                            "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4444                            churn_ctx->num_failed_start,
4445                            churn_ctx->num_failed_stop);
4446         }
4447       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4448       GNUNET_free_non_null (error_message);
4449       GNUNET_free (churn_ctx);
4450       GNUNET_free (shutdown_ctx);
4451     }
4452 }
4453
4454 /**
4455  * Count the number of running peers.
4456  *
4457  * @param pg handle for the peer group
4458  *
4459  * @return the number of currently running peers in the peer group
4460  */
4461 unsigned int
4462 GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
4463 {
4464   unsigned int i;
4465   unsigned int running = 0;
4466   for (i = 0; i < pg->total; i++)
4467     {
4468       if (pg->peers[i].daemon->running == GNUNET_YES)
4469         {
4470           GNUNET_assert (running != -1);
4471           running++;
4472         }
4473     }
4474   return running;
4475 }
4476
4477 /**
4478  * Task to rate limit the number of outstanding peer shutdown
4479  * requests.  This is necessary for making sure we don't do
4480  * too many ssh connections at once, but is generally nicer
4481  * to any system as well (graduated task starts, as opposed
4482  * to calling gnunet-arm N times all at once).
4483  */
4484 static void
4485 schedule_churn_shutdown_task (void *cls,
4486                               const struct GNUNET_SCHEDULER_TaskContext *tc)
4487 {
4488   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4489   struct ShutdownContext *shutdown_ctx;
4490
4491   GNUNET_assert (peer_shutdown_ctx != NULL);
4492   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4493   GNUNET_assert (shutdown_ctx != NULL);
4494
4495   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4496     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4497                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4498                                   &schedule_churn_shutdown_task,
4499                                   peer_shutdown_ctx);
4500   else
4501     {
4502       shutdown_ctx->outstanding++;
4503       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4504                                   shutdown_ctx->timeout, shutdown_ctx->cb,
4505                                   shutdown_ctx, GNUNET_NO, GNUNET_YES);
4506       GNUNET_free (peer_shutdown_ctx);
4507     }
4508 }
4509
4510 /**
4511  * Simulate churn by stopping some peers (and possibly
4512  * re-starting others if churn is called multiple times).  This
4513  * function can only be used to create leave-join churn (peers "never"
4514  * leave for good).  First "voff" random peers that are currently
4515  * online will be taken offline; then "von" random peers that are then
4516  * offline will be put back online.  No notifications will be
4517  * generated for any of these operations except for the callback upon
4518  * completion.
4519  *
4520  * @param pg handle for the peer group
4521  * @param voff number of peers that should go offline
4522  * @param von number of peers that should come back online;
4523  *            must be zero on first call (since "testbed_start"
4524  *            always starts all of the peers)
4525  * @param timeout how long to wait for operations to finish before
4526  *        giving up
4527  * @param cb function to call at the end
4528  * @param cb_cls closure for cb
4529  */
4530 void
4531 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
4532                               unsigned int voff,
4533                               unsigned int von,
4534                               struct GNUNET_TIME_Relative timeout,
4535                               GNUNET_TESTING_NotifyCompletion cb,
4536                               void *cb_cls)
4537 {
4538   struct ChurnContext *churn_ctx;
4539   struct ShutdownContext *shutdown_ctx;
4540   struct PeerShutdownContext *peer_shutdown_ctx;
4541   struct PeerRestartContext *peer_restart_ctx;
4542   struct ChurnRestartContext *churn_startup_ctx;
4543
4544   unsigned int running;
4545   unsigned int stopped;
4546   unsigned int total_running;
4547   unsigned int total_stopped;
4548   unsigned int i;
4549   unsigned int *running_arr;
4550   unsigned int *stopped_arr;
4551   unsigned int *running_permute;
4552   unsigned int *stopped_permute;
4553
4554   running = 0;
4555   stopped = 0;
4556
4557   if ((von == 0) && (voff == 0))        /* No peers at all? */
4558     {
4559       cb (cb_cls, NULL);
4560       return;
4561     }
4562
4563   for (i = 0; i < pg->total; i++)
4564     {
4565       if (pg->peers[i].daemon->running == GNUNET_YES)
4566         {
4567           GNUNET_assert (running != -1);
4568           running++;
4569         }
4570       else
4571         {
4572           GNUNET_assert (stopped != -1);
4573           stopped++;
4574         }
4575     }
4576
4577   if (voff > running)
4578     {
4579       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4580                   "Trying to stop more peers than are currently running!\n");
4581       cb (cb_cls, "Trying to stop more peers than are currently running!");
4582       return;
4583     }
4584
4585   if (von > stopped)
4586     {
4587       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4588                   "Trying to start more peers than are currently stopped!\n");
4589       cb (cb_cls, "Trying to start more peers than are currently stopped!");
4590       return;
4591     }
4592
4593   churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4594
4595   running_arr = NULL;
4596   if (running > 0)
4597     running_arr = GNUNET_malloc (running * sizeof (unsigned int));
4598
4599   stopped_arr = NULL;
4600   if (stopped > 0)
4601     stopped_arr = GNUNET_malloc (stopped * sizeof (unsigned int));
4602
4603   running_permute = NULL;
4604   stopped_permute = NULL;
4605
4606   if (running > 0)
4607     running_permute =
4608       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, running);
4609   if (stopped > 0)
4610     stopped_permute =
4611       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, stopped);
4612
4613   total_running = running;
4614   total_stopped = stopped;
4615   running = 0;
4616   stopped = 0;
4617
4618   churn_ctx->num_to_start = von;
4619   churn_ctx->num_to_stop = voff;
4620   churn_ctx->cb = cb;
4621   churn_ctx->cb_cls = cb_cls;
4622
4623   for (i = 0; i < pg->total; i++)
4624     {
4625       if (pg->peers[i].daemon->running == GNUNET_YES)
4626         {
4627           GNUNET_assert ((running_arr != NULL) && (total_running > running));
4628           running_arr[running] = i;
4629           running++;
4630         }
4631       else
4632         {
4633           GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped));
4634           stopped_arr[stopped] = i;
4635           stopped++;
4636         }
4637     }
4638
4639   GNUNET_assert (running >= voff);
4640   if (voff > 0)
4641     {
4642       shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4643       shutdown_ctx->cb = &churn_stop_callback;
4644       shutdown_ctx->cb_cls = churn_ctx;
4645       shutdown_ctx->total_peers = voff;
4646       shutdown_ctx->timeout = timeout;
4647     }
4648
4649   for (i = 0; i < voff; i++)
4650     {
4651 #if DEBUG_CHURN
4652       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n",
4653                   running_permute[i]);
4654 #endif
4655       GNUNET_assert (running_arr != NULL);
4656       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4657       peer_shutdown_ctx->daemon =
4658         pg->peers[running_arr[running_permute[i]]].daemon;
4659       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4660       GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task,
4661                                 peer_shutdown_ctx);
4662
4663       /*
4664          GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
4665          timeout,
4666          &churn_stop_callback, churn_ctx,
4667          GNUNET_NO, GNUNET_YES); */
4668     }
4669
4670   GNUNET_assert (stopped >= von);
4671   if (von > 0)
4672     {
4673       churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4674       churn_startup_ctx->churn_ctx = churn_ctx;
4675       churn_startup_ctx->timeout = timeout;
4676     }
4677   for (i = 0; i < von; i++)
4678     {
4679 #if DEBUG_CHURN
4680       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n",
4681                   stopped_permute[i]);
4682 #endif
4683       GNUNET_assert (stopped_arr != NULL);
4684       peer_restart_ctx = GNUNET_malloc (sizeof (struct PeerRestartContext));
4685       peer_restart_ctx->churn_restart_ctx = churn_startup_ctx;
4686       peer_restart_ctx->daemon =
4687         pg->peers[stopped_arr[stopped_permute[i]]].daemon;
4688       GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx);
4689       /*
4690          GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
4691          timeout, &churn_start_callback, churn_ctx); */
4692     }
4693
4694   GNUNET_free_non_null (running_arr);
4695   GNUNET_free_non_null (stopped_arr);
4696   GNUNET_free_non_null (running_permute);
4697   GNUNET_free_non_null (stopped_permute);
4698 }
4699
4700
4701 /**
4702  * Restart all peers in the given group.
4703  *
4704  * @param pg the handle to the peer group
4705  * @param callback function to call on completion (or failure)
4706  * @param callback_cls closure for the callback function
4707  */
4708 void
4709 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
4710                                 GNUNET_TESTING_NotifyCompletion callback,
4711                                 void *callback_cls)
4712 {
4713   struct RestartContext *restart_context;
4714   unsigned int off;
4715
4716   if (pg->total > 0)
4717     {
4718       restart_context = GNUNET_malloc (sizeof (struct RestartContext));
4719       restart_context->peer_group = pg;
4720       restart_context->peers_restarted = 0;
4721       restart_context->callback = callback;
4722       restart_context->callback_cls = callback_cls;
4723
4724       for (off = 0; off < pg->total; off++)
4725         {
4726           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon,
4727                                          &restart_callback, restart_context);
4728         }
4729     }
4730 }
4731
4732 /**
4733  * Start or stop an individual peer from the given group.
4734  *
4735  * @param pg handle to the peer group
4736  * @param offset which peer to start or stop
4737  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
4738  * @param timeout how long to wait for shutdown
4739  * @param cb function to call at the end
4740  * @param cb_cls closure for cb
4741  */
4742 void
4743 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
4744                              unsigned int offset,
4745                              int desired_status,
4746                              struct GNUNET_TIME_Relative timeout,
4747                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4748 {
4749   struct ShutdownContext *shutdown_ctx;
4750   struct ChurnRestartContext *startup_ctx;
4751   struct ChurnContext *churn_ctx;
4752
4753   if (GNUNET_NO == desired_status)
4754     {
4755       if (NULL != pg->peers[offset].daemon)
4756         {
4757           shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4758           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4759           churn_ctx->num_to_start = 0;
4760           churn_ctx->num_to_stop = 1;
4761           churn_ctx->cb = cb;
4762           churn_ctx->cb_cls = cb_cls;
4763           shutdown_ctx->cb_cls = churn_ctx;
4764           GNUNET_TESTING_daemon_stop (pg->peers[offset].daemon,
4765                                       timeout, &churn_stop_callback,
4766                                       shutdown_ctx, GNUNET_NO, GNUNET_YES);
4767         }
4768     }
4769   else if (GNUNET_YES == desired_status)
4770     {
4771       if (NULL == pg->peers[offset].daemon)
4772         {
4773           startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4774           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4775           churn_ctx->num_to_start = 1;
4776           churn_ctx->num_to_stop = 0;
4777           churn_ctx->cb = cb;
4778           churn_ctx->cb_cls = cb_cls;
4779           startup_ctx->churn_ctx = churn_ctx;
4780           GNUNET_TESTING_daemon_start_stopped (pg->peers[offset].daemon,
4781                                                timeout, &churn_start_callback,
4782                                                startup_ctx);
4783         }
4784     }
4785   else
4786     GNUNET_break (0);
4787 }
4788
4789
4790 /**
4791  * Callback for shutting down peers in a peer group.
4792  *
4793  * @param cls closure (struct ShutdownContext)
4794  * @param emsg NULL on success
4795  */
4796 void
4797 internal_shutdown_callback (void *cls, const char *emsg)
4798 {
4799   struct ShutdownContext *shutdown_ctx = cls;
4800
4801   shutdown_ctx->outstanding--;
4802   if (emsg == NULL)
4803     {
4804       shutdown_ctx->peers_down++;
4805     }
4806   else
4807     {
4808       shutdown_ctx->peers_failed++;
4809     }
4810
4811   if ((shutdown_ctx->cb != NULL)
4812       && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed ==
4813           shutdown_ctx->total_peers))
4814     {
4815       if (shutdown_ctx->peers_failed > 0)
4816         shutdown_ctx->cb (shutdown_ctx->cb_cls,
4817                           "Not all peers successfully shut down!");
4818       else
4819         shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
4820       GNUNET_free (shutdown_ctx);
4821     }
4822 }
4823
4824
4825 /**
4826  * Task to rate limit the number of outstanding peer shutdown
4827  * requests.  This is necessary for making sure we don't do
4828  * too many ssh connections at once, but is generally nicer
4829  * to any system as well (graduated task starts, as opposed
4830  * to calling gnunet-arm N times all at once).
4831  */
4832 static void
4833 schedule_shutdown_task (void *cls,
4834                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4835 {
4836   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4837   struct ShutdownContext *shutdown_ctx;
4838
4839   GNUNET_assert (peer_shutdown_ctx != NULL);
4840   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4841   GNUNET_assert (shutdown_ctx != NULL);
4842
4843   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4844     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4845                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4846                                   &schedule_shutdown_task, peer_shutdown_ctx);
4847   else
4848     {
4849       shutdown_ctx->outstanding++;
4850       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4851                                   shutdown_ctx->timeout,
4852                                   &internal_shutdown_callback, shutdown_ctx,
4853                                   GNUNET_YES, GNUNET_NO);
4854       GNUNET_free (peer_shutdown_ctx);
4855     }
4856 }
4857
4858 /**
4859  * Shutdown all peers started in the given group.
4860  *
4861  * @param pg handle to the peer group
4862  * @param timeout how long to wait for shutdown
4863  * @param cb callback to notify upon success or failure
4864  * @param cb_cls closure for cb
4865  */
4866 void
4867 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4868                              struct GNUNET_TIME_Relative timeout,
4869                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4870 {
4871   unsigned int off;
4872   struct ShutdownContext *shutdown_ctx;
4873   struct PeerShutdownContext *peer_shutdown_ctx;
4874
4875   GNUNET_assert (pg->total > 0);
4876
4877   shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4878   shutdown_ctx->cb = cb;
4879   shutdown_ctx->cb_cls = cb_cls;
4880   shutdown_ctx->total_peers = pg->total;
4881   shutdown_ctx->timeout = timeout;
4882   /* shtudown_ctx->outstanding = 0; */
4883
4884   for (off = 0; off < pg->total; off++)
4885     {
4886       GNUNET_assert (NULL != pg->peers[off].daemon);
4887       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4888       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4889       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4890       GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
4891       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4892       if (NULL != pg->peers[off].cfg)
4893         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4894       if (pg->peers[off].allowed_peers != NULL)
4895         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
4896       if (pg->peers[off].connect_peers != NULL)
4897         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
4898       if (pg->peers[off].blacklisted_peers != NULL)
4899         GNUNET_CONTAINER_multihashmap_destroy (pg->
4900                                                peers[off].blacklisted_peers);
4901     }
4902   GNUNET_free (pg->peers);
4903   for (off = 0; off < pg->num_hosts; off++)
4904     {
4905       GNUNET_free (pg->hosts[off].hostname);
4906       GNUNET_free_non_null (pg->hosts[off].username);
4907     }
4908   GNUNET_free_non_null (pg->hosts);
4909   GNUNET_free (pg);
4910 }
4911
4912
4913 /* end of testing_group.c */