src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define VERBOSE_TOPOLOGY GNUNET_YES
  36
  37 #define DEBUG_CHURN GNUNET_NO
  38
  39 /**
  40  * Lowest port used for GNUnet testing.  Should be high enough to not
  41  * conflict with other applications running on the hosts but be low
  42  * enough to not conflict with client-ports (typically starting around
  43  * 32k).
  44  */
  45 #define LOW_PORT 10000
  46
  47 /**
  48  * Highest port used for GNUnet testing.  Should be low enough to not
  49  * conflict with the port range for "local" ports (client apps; see
  50  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  51  */
  52 #define HIGH_PORT 56000
  53
  54 #define MAX_OUTSTANDING_CONNECTIONS 40
  55
  56 #define MAX_CONCURRENT_HOSTKEYS 10
  57
  58 #define MAX_CONCURRENT_STARTING 10
  59
  60 #define MAX_CONCURRENT_SHUTDOWN 10
  61
  62 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 200)
  63
  64 #define CONNECT_ATTEMPTS 21
  65
  66 /**
  67  * Prototype of a function called whenever two peers would be connected
  68  * in a certain topology.
  69  */
  70 typedef unsigned int (*GNUNET_TESTING_ConnectionProcessor) (struct
  71                                                             GNUNET_TESTING_PeerGroup
  72                                                             * pg,
  73                                                             unsigned int
  74                                                             first,
  75                                                             unsigned int
  76                                                             second);
  77
  78
  79 /**
  80  * Context for handling churning a peer group
  81  */
  82 struct ChurnContext
  83 {
  84   /**
  85    * Callback used to notify of churning finished
  86    */
  87   GNUNET_TESTING_NotifyCompletion cb;
  88
  89   /**
  90    * Closure for callback
  91    */
  92   void *cb_cls;
  93
  94   /**
  95    * Number of peers that still need to be started
  96    */
  97   unsigned int num_to_start;
  98
  99   /**
 100    * Number of peers that still need to be stopped
 101    */
 102   unsigned int num_to_stop;
 103
 104   /**
 105    * Number of peers that failed to start
 106    */
 107   unsigned int num_failed_start;
 108
 109   /**
 110    * Number of peers that failed to stop
 111    */
 112   unsigned int num_failed_stop;
 113 };
 114
 115 struct RestartContext
 116 {
 117   /**
 118    * The group of peers being restarted
 119    */
 120   struct GNUNET_TESTING_PeerGroup *peer_group;
 121
 122   /**
 123    * How many peers have been restarted thus far
 124    */
 125   unsigned int peers_restarted;
 126
 127   /**
 128    * How many peers got an error when restarting
 129    */
 130   unsigned int peers_restart_failed;
 131
 132   /**
 133    * The function to call once all peers have been restarted
 134    */
 135   GNUNET_TESTING_NotifyCompletion callback;
 136
 137   /**
 138    * Closure for callback function
 139    */
 140   void *callback_cls;
 141
 142 };
 143
 144
 145 struct ShutdownContext
 146 {
 147   /**
 148    * Total peers to wait for
 149    */
 150   unsigned int total_peers;
 151
 152   /**
 153    * Number of peers successfully shut down
 154    */
 155   unsigned int peers_down;
 156
 157   /**
 158    * Number of peers failed to shut down
 159    */
 160   unsigned int peers_failed;
 161
 162   /**
 163    * Number of peers we have started shutting
 164    * down.  If too many, wait on them.
 165    */
 166   unsigned int outstanding;
 167
 168   /**
 169    * Timeout for shutdown.
 170    */
 171   struct GNUNET_TIME_Relative timeout;
 172
 173   /**
 174    * Callback to call when all peers either
 175    * shutdown or failed to shutdown
 176    */
 177   GNUNET_TESTING_NotifyCompletion cb;
 178
 179   /**
 180    * Closure for cb
 181    */
 182   void *cb_cls;
 183 };
 184
 185 /**
 186  * Individual shutdown context for a particular peer.
 187  */
 188 struct PeerShutdownContext
 189 {
 190   /**
 191    * Pointer to the high level shutdown context.
 192    */
 193   struct ShutdownContext *shutdown_ctx;
 194
 195   /**
 196    * The daemon handle for the peer to shut down.
 197    */
 198   struct GNUNET_TESTING_Daemon *daemon;
 199 };
 200
 201 /**
 202  * Individual shutdown context for a particular peer.
 203  */
 204 struct PeerRestartContext
 205 {
 206   /**
 207    * Pointer to the high level restart context.
 208    */
 209   struct ChurnRestartContext *churn_restart_ctx;
 210
 211   /**
 212    * The daemon handle for the peer to shut down.
 213    */
 214   struct GNUNET_TESTING_Daemon *daemon;
 215 };
 216
 217
 218 struct CreateTopologyContext
 219 {
 220
 221   /**
 222    * Function to call with number of connections
 223    */
 224   GNUNET_TESTING_NotifyConnections cont;
 225
 226   /**
 227    * Closure for connection notification
 228    */
 229   void *cls;
 230 };
 231
 232 #if OLD
 233 struct PeerConnection
 234 {
 235   /*
 236    * Linked list
 237    */
 238   struct PeerConnection *next;
 239
 240   /*
 241    * Pointer to daemon handle
 242    */
 243   struct GNUNET_TESTING_Daemon *daemon;
 244
 245 };
 246 #endif
 247
 248 struct InternalStartContext
 249 {
 250   /**
 251    * Pointer to peerdata
 252    */
 253   struct PeerData *peer;
 254
 255   /**
 256    * Timeout for peer startup
 257    */
 258   struct GNUNET_TIME_Relative timeout;
 259
 260   /**
 261    * Client callback for hostkey notification
 262    */
 263   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 264
 265   /**
 266    * Closure for hostkey_callback
 267    */
 268   void *hostkey_cls;
 269
 270   /**
 271    * Client callback for peer start notification
 272    */
 273   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 274
 275   /**
 276    * Closure for cb
 277    */
 278   void *start_cb_cls;
 279
 280   /**
 281    * Hostname, where to start the peer
 282    */
 283   const char *hostname;
 284
 285   /**
 286    * Username to use when connecting to the
 287    * host via ssh.
 288    */
 289   const char *username;
 290
 291   /**
 292    * Port to use for ssh.
 293    */
 294   uint16_t sshport;
 295
 296 };
 297
 298 struct ChurnRestartContext
 299 {
 300   /**
 301    * Number of restarts currently in flight.
 302    */
 303   unsigned int outstanding;
 304
 305   /**
 306    * Handle to the underlying churn context.
 307    */
 308   struct ChurnContext *churn_ctx;
 309
 310   /**
 311    * How long to allow the operation to take.
 312    */
 313   struct GNUNET_TIME_Relative timeout;
 314 };
 315
 316 /**
 317  * Data we keep per peer.
 318  */
 319 struct PeerData
 320 {
 321   /**
 322    * (Initial) configuration of the host.
 323    * (initial because clients could change
 324    *  it and we would not know about those
 325    *  updates).
 326    */
 327   struct GNUNET_CONFIGURATION_Handle *cfg;
 328
 329   /**
 330    * Handle for controlling the daemon.
 331    */
 332   struct GNUNET_TESTING_Daemon *daemon;
 333
 334   /**
 335    * The peergroup this peer belongs to.
 336    */
 337   struct GNUNET_TESTING_PeerGroup *pg;
 338
 339   /**
 340    * Hash map of allowed peer connections (F2F created topology)
 341    */
 342   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 343
 344   /**
 345    * Hash map of blacklisted peers
 346    */
 347   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 348
 349   /**
 350    * Hash map of peer connections
 351    */
 352   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 353
 354   /**
 355    * Temporary hash map of peer connections
 356    */
 357   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 358
 359   /**
 360    * Temporary variable for topology creation, should be reset before
 361    * creating any topology so the count is valid once finished.
 362    */
 363   int num_connections;
 364
 365   /**
 366    * Context to keep track of peers being started, to
 367    * stagger hostkey generation and peer startup.
 368    */
 369   struct InternalStartContext internal_context;
 370 };
 371
 372
 373 /**
 374  * Linked list of per-host data.
 375  */
 376 struct HostData
 377 {
 378   /**
 379    * Name of the host.
 380    */
 381   char *hostname;
 382
 383   /**
 384    * SSH username to use when connecting to this host.
 385    */
 386   char *username;
 387
 388   /**
 389    * SSH port to use when connecting to this host.
 390    */
 391   uint16_t sshport;
 392
 393   /**
 394    * Lowest port that we have not yet used
 395    * for GNUnet.
 396    */
 397   uint16_t minport;
 398 };
 399
 400 struct TopologyIterateContext
 401 {
 402   /**
 403    * Callback for notifying of two connected peers.
 404    */
 405   GNUNET_TESTING_NotifyTopology topology_cb;
 406
 407   /**
 408    * Closure for topology_cb
 409    */
 410   void *cls;
 411
 412   /**
 413    * Number of peers currently connected to.
 414    */
 415   unsigned int connected;
 416
 417   /**
 418    * Number of peers we have finished iterating.
 419    */
 420   unsigned int completed;
 421
 422   /**
 423    * Number of peers total.
 424    */
 425   unsigned int total;
 426 };
 427
 428 struct StatsIterateContext
 429 {
 430   /**
 431    * Continuation to call once all stats information has been retrieved.
 432    */
 433   GNUNET_STATISTICS_Callback cont;
 434
 435   /**
 436    * Proc function to call on each value received.
 437    */
 438   GNUNET_TESTING_STATISTICS_Iterator proc;
 439
 440   /**
 441    * Closure for topology_cb
 442    */
 443   void *cls;
 444
 445   /**
 446    * Number of peers currently connected to.
 447    */
 448   unsigned int connected;
 449
 450   /**
 451    * Number of peers we have finished iterating.
 452    */
 453   unsigned int completed;
 454
 455   /**
 456    * Number of peers total.
 457    */
 458   unsigned int total;
 459 };
 460
 461 struct CoreContext
 462 {
 463   void *iter_context;
 464   struct GNUNET_TESTING_Daemon *daemon;
 465 };
 466
 467 struct StatsCoreContext
 468 {
 469   void *iter_context;
 470   struct GNUNET_TESTING_Daemon *daemon;
 471   /**
 472    * Handle to the statistics service.
 473    */
 474   struct GNUNET_STATISTICS_Handle *stats_handle;
 475
 476   /**
 477    * Handle for getting statistics.
 478    */
 479   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 480 };
 481
 482 /**
 483  * Handle to a group of GNUnet peers.
 484  */
 485 struct GNUNET_TESTING_PeerGroup
 486 {
 487   /**
 488    * Configuration template.
 489    */
 490   const struct GNUNET_CONFIGURATION_Handle *cfg;
 491
 492   /**
 493    * Function to call on each started daemon.
 494    */
 495   //GNUNET_TESTING_NotifyDaemonRunning cb;
 496
 497   /**
 498    * Closure for cb.
 499    */
 500   //void *cb_cls;
 501
 502   /*
 503    * Function to call on each topology connection created
 504    */
 505   GNUNET_TESTING_NotifyConnection notify_connection;
 506
 507   /*
 508    * Callback for notify_connection
 509    */
 510   void *notify_connection_cls;
 511
 512   /**
 513    * Array of information about hosts.
 514    */
 515   struct HostData *hosts;
 516
 517   /**
 518    * Number of hosts (size of HostData)
 519    */
 520   unsigned int num_hosts;
 521
 522   /**
 523    * Array of "total" peers.
 524    */
 525   struct PeerData *peers;
 526
 527   /**
 528    * Number of peers in this group.
 529    */
 530   unsigned int total;
 531
 532   /**
 533    * At what time should we fail the peer startup process?
 534    */
 535   struct GNUNET_TIME_Absolute max_timeout;
 536
 537   /**
 538    * How many peers are being started right now?
 539    */
 540   unsigned int starting;
 541
 542   /**
 543    * How many peers have already been started?
 544    */
 545   unsigned int started;
 546 };
 547
 548 struct UpdateContext
 549 {
 550   struct GNUNET_CONFIGURATION_Handle *ret;
 551   const struct GNUNET_CONFIGURATION_Handle *orig;
 552   const char *hostname;
 553   unsigned int nport;
 554   unsigned int upnum;
 555   unsigned int fdnum;
 556 };
 557
 558 struct ConnectTopologyContext
 559 {
 560   /**
 561    * How many connections are left to create.
 562    */
 563   unsigned int remaining_connections;
 564
 565   /**
 566    * Handle to group of peers.
 567    */
 568   struct GNUNET_TESTING_PeerGroup *pg;
 569
 570   /**
 571    * Temp value set for each iteration.
 572    */
 573   struct PeerData *first;
 574
 575   /**
 576    * Notification that all peers are connected.
 577    */
 578   GNUNET_TESTING_NotifyCompletion notify_connections_done;
 579
 580   /**
 581    * Closure for notify.
 582    */
 583   void *notify_cls;
 584 };
 585
 586 struct ConnectContext
 587 {
 588   /**
 589    * Peer to connect second to.
 590    */
 591   struct GNUNET_TESTING_Daemon *first;
 592
 593   /**
 594    * Peer to connect first to.
 595    */
 596   struct GNUNET_TESTING_Daemon *second;
 597
 598   /**
 599    * Higher level topology connection context.
 600    */
 601   struct ConnectTopologyContext *ct_ctx;
 602 };
 603
 604 /**
 605  * Convert unique ID to hash code.
 606  *
 607  * @param uid unique ID to convert
 608  * @param hash set to uid (extended with zeros)
 609  */
 610 static void
 611 hash_from_uid (uint32_t uid, GNUNET_HashCode * hash)
 612 {
 613   memset (hash, 0, sizeof (GNUNET_HashCode));
 614   *((uint32_t *) hash) = uid;
 615 }
 616
 617 /**
 618  * Convert hash code to unique ID.
 619  *
 620  * @param uid unique ID to convert
 621  * @param hash set to uid (extended with zeros)
 622  */
 623 static void
 624 uid_from_hash (const GNUNET_HashCode * hash, uint32_t * uid)
 625 {
 626   memcpy (uid, hash, sizeof (uint32_t));
 627 }
 628
 629 /**
 630  * Number of connects we are waiting on, allows us to rate limit
 631  * connect attempts.
 632  */
 633 static int outstanding_connects;
 634
 635 /**
 636  * Get a topology from a string input.
 637  *
 638  * @param topology where to write the retrieved topology
 639  * @param topology_string The string to attempt to
 640  *        get a configuration value from
 641  * @return GNUNET_YES if topology string matched a
 642  *         known topology, GNUNET_NO if not
 643  */
 644 int
 645 GNUNET_TESTING_topology_get (enum GNUNET_TESTING_Topology *topology,
 646                              const char *topology_string)
 647 {
 648   /**
 649    * Strings representing topologies in enum
 650    */
 651   static const char *topology_strings[] = {
 652       /**
 653        * A clique (everyone connected to everyone else).
 654        */
 655     "CLIQUE",
 656
 657       /**
 658        * Small-world network (2d torus plus random links).
 659        */
 660     "SMALL_WORLD",
 661
 662       /**
 663        * Small-world network (ring plus random links).
 664        */
 665     "SMALL_WORLD_RING",
 666
 667       /**
 668        * Ring topology.
 669        */
 670     "RING",
 671
 672       /**
 673        * 2-d torus.
 674        */
 675     "2D_TORUS",
 676
 677       /**
 678        * Random graph.
 679        */
 680     "ERDOS_RENYI",
 681
 682       /**
 683        * Certain percentage of peers are unable to communicate directly
 684        * replicating NAT conditions
 685        */
 686     "INTERNAT",
 687
 688       /**
 689        * Scale free topology.
 690        */
 691     "SCALE_FREE",
 692
 693       /**
 694        * Straight line topology.
 695        */
 696     "LINE",
 697
 698       /**
 699        * All peers are disconnected.
 700        */
 701     "NONE",
 702
 703     NULL
 704   };
 705
 706   int curr = 0;
 707   if (topology_string == NULL)
 708     return GNUNET_NO;
 709   while (topology_strings[curr] != NULL)
 710     {
 711       if (strcasecmp (topology_strings[curr], topology_string) == 0)
 712         {
 713           *topology = curr;
 714           return GNUNET_YES;
 715         }
 716       curr++;
 717     }
 718   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 719   return GNUNET_NO;
 720 }
 721
 722
 723 /**
 724  * Get connect topology option from string input.
 725  *
 726  * @param topology_option where to write the retrieved topology
 727  * @param topology_string The string to attempt to
 728  *        get a configuration value from
 729  * @return GNUNET_YES if string matched a known
 730  *         topology option, GNUNET_NO if not
 731  */
 732 int
 733 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption
 734                                     *topology_option,
 735                                     const char *topology_string)
 736 {
 737   /**
 738    * Options for connecting a topology as strings.
 739    */
 740   static const char *topology_option_strings[] = {
 741       /**
 742        * Try to connect all peers specified in the topology.
 743        */
 744     "CONNECT_ALL",
 745
 746       /**
 747        * Choose a random subset of connections to create.
 748        */
 749     "CONNECT_RANDOM_SUBSET",
 750
 751       /**
 752        * Create at least X connections for each peer.
 753        */
 754     "CONNECT_MINIMUM",
 755
 756       /**
 757        * Using a depth first search, create one connection
 758        * per peer.  If any are missed (graph disconnected)
 759        * start over at those peers until all have at least one
 760        * connection.
 761        */
 762     "CONNECT_DFS",
 763
 764       /**
 765        * Find the N closest peers to each allowed peer in the
 766        * topology and make sure a connection to those peers
 767        * exists in the connect topology.
 768        */
 769     "CONNECT_CLOSEST",
 770
 771       /**
 772        * No options specified.
 773        */
 774     "CONNECT_NONE",
 775
 776     NULL
 777   };
 778   int curr = 0;
 779
 780   if (topology_string == NULL)
 781     return GNUNET_NO;
 782   while (NULL != topology_option_strings[curr])
 783     {
 784       if (strcasecmp (topology_option_strings[curr], topology_string) == 0)
 785         {
 786           *topology_option = curr;
 787           return GNUNET_YES;
 788         }
 789       curr++;
 790     }
 791   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 792   return GNUNET_NO;
 793 }
 794
 795 /**
 796  * Function to iterate over options.  Copies
 797  * the options to the target configuration,
 798  * updating PORT values as needed.
 799  *
 800  * @param cls closure
 801  * @param section name of the section
 802  * @param option name of the option
 803  * @param value value of the option
 804  */
 805 static void
 806 update_config (void *cls,
 807                const char *section, const char *option, const char *value)
 808 {
 809   struct UpdateContext *ctx = cls;
 810   unsigned int ival;
 811   char cval[12];
 812   char uval[128];
 813   char *single_variable;
 814   char *per_host_variable;
 815   unsigned long long num_per_host;
 816
 817   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 818     {
 819       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 820       if ((ival != 0)
 821           && (GNUNET_YES !=
 822               GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 823                                                     single_variable)))
 824         {
 825           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 826           value = cval;
 827         }
 828
 829       GNUNET_free (single_variable);
 830     }
 831
 832   if (0 == strcmp (option, "UNIXPATH"))
 833     {
 834       GNUNET_asprintf (&single_variable, "single_%s_per_host", section);
 835       GNUNET_asprintf (&per_host_variable, "num_%s_per_host", section);
 836       if (GNUNET_YES !=
 837           GNUNET_CONFIGURATION_get_value_yesno (ctx->orig, "testing",
 838                                                 single_variable))
 839         {
 840           GNUNET_snprintf (uval,
 841                            sizeof (uval),
 842                            "/tmp/test-service-%s-%u", section, ctx->upnum++);
 843           value = uval;
 844         }
 845       else if (GNUNET_YES ==
 846                GNUNET_CONFIGURATION_get_value_number (ctx->orig, "testing",
 847                                                       per_host_variable,
 848                                                       &num_per_host))
 849         {
 850           GNUNET_snprintf (uval,
 851                            sizeof (uval),
 852                            "/tmp/test-service-%s-%u",
 853                            section, ctx->fdnum % num_per_host);
 854           value = uval;
 855         }
 856       GNUNET_free (single_variable);
 857       GNUNET_free (per_host_variable);
 858
 859     }
 860
 861   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 862     {
 863       value = ctx->hostname;
 864     }
 865
 866   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 867 }
 868
 869
 870 /**
 871  * Create a new configuration using the given configuration
 872  * as a template; however, each PORT in the existing cfg
 873  * must be renumbered by incrementing "*port".  If we run
 874  * out of "*port" numbers, return NULL.
 875  *
 876  * @param cfg template configuration
 877  * @param port port numbers to use, update to reflect
 878  *             port numbers that were used
 879  * @param upnum number to make unix domain socket names unique
 880  * @param hostname hostname of the controlling host, to allow control connections from
 881  * @param fdnum number used to offset the unix domain socket for grouped processes
 882  *              (such as statistics or peerinfo, which can be shared among others)
 883  *
 884  * @return new configuration, NULL on error
 885  */
 886 static struct GNUNET_CONFIGURATION_Handle *
 887 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 888              uint16_t * port,
 889              uint32_t * upnum, const char *hostname, uint32_t * fdnum)
 890 {
 891   struct UpdateContext uc;
 892   uint16_t orig;
 893   char *control_host;
 894   char *allowed_hosts;
 895
 896   orig = *port;
 897   uc.nport = *port;
 898   uc.upnum = *upnum;
 899   uc.fdnum = *fdnum;
 900   uc.ret = GNUNET_CONFIGURATION_create ();
 901   uc.hostname = hostname;
 902   uc.orig = cfg;
 903
 904   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 905   if (uc.nport >= HIGH_PORT)
 906     {
 907       *port = orig;
 908       GNUNET_CONFIGURATION_destroy (uc.ret);
 909       return NULL;
 910     }
 911
 912   if (GNUNET_CONFIGURATION_get_value_string
 913       (cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 914     {
 915       if (hostname != NULL)
 916         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1; %s;", control_host,
 917                          hostname);
 918       else
 919         GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", control_host);
 920
 921       GNUNET_CONFIGURATION_set_value_string (uc.ret, "core", "ACCEPT_FROM",
 922                                              allowed_hosts);
 923       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport",
 924                                              "ACCEPT_FROM", allowed_hosts);
 925       GNUNET_CONFIGURATION_set_value_string (uc.ret, "dht", "ACCEPT_FROM",
 926                                              allowed_hosts);
 927       GNUNET_CONFIGURATION_set_value_string (uc.ret, "statistics",
 928                                              "ACCEPT_FROM", allowed_hosts);
 929       GNUNET_free_non_null (control_host);
 930       GNUNET_free (allowed_hosts);
 931     }
 932
 933
 934   /* arm needs to know to allow connections from the host on which it is running,
 935    * otherwise gnunet-arm is unable to connect to it in some instances */
 936   if (hostname != NULL)
 937     {
 938       GNUNET_asprintf (&allowed_hosts, "%s; 127.0.0.1;", hostname);
 939       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 940                                              "BINDTO", hostname);
 941       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 942                                              "BINDTO", hostname);
 943       GNUNET_CONFIGURATION_set_value_string (uc.ret, "arm", "ACCEPT_FROM",
 944                                              allowed_hosts);
 945       GNUNET_free (allowed_hosts);
 946     }
 947   else
 948     {
 949       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-tcp",
 950                                              "BINDTO", "127.0.0.1");
 951       GNUNET_CONFIGURATION_set_value_string (uc.ret, "transport-udp",
 952                                              "BINDTO", "127.0.0.1");
 953     }
 954
 955   *port = (uint16_t) uc.nport;
 956   *upnum = uc.upnum;
 957   uc.fdnum++;
 958   *fdnum = uc.fdnum;
 959   return uc.ret;
 960 }
 961
 962
 963 /*
 964  * Add entries to the peers connect list
 965  *
 966  * @param pg the peer group we are working with
 967  * @param first index of the first peer
 968  * @param second index of the second peer
 969  *
 970  * @return the number of connections added
 971  *         technically should only be 0 or 2
 972  *
 973  */
 974 static unsigned int
 975 add_actual_connections (struct GNUNET_TESTING_PeerGroup *pg,
 976                         unsigned int first, unsigned int second)
 977 {
 978   int added;
 979   int add_first;
 980   int add_second;
 981
 982   GNUNET_HashCode hash_first;
 983   GNUNET_HashCode hash_second;
 984
 985   hash_from_uid (first, &hash_first);
 986   hash_from_uid (second, &hash_second);
 987
 988   add_first = GNUNET_NO;
 989   if (GNUNET_NO ==
 990       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].connect_peers,
 991                                               &hash_second))
 992     {
 993       add_first = GNUNET_YES;
 994     }
 995
 996   add_second = GNUNET_NO;
 997   if (GNUNET_NO ==
 998       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].connect_peers,
 999                                               &hash_first))
1000     {
1001       add_second = GNUNET_YES;
1002     }
1003
1004   added = 0;
1005   if (add_first)
1006     {
1007       GNUNET_assert (GNUNET_OK ==
1008                      GNUNET_CONTAINER_multihashmap_put (pg->
1009                                                         peers
1010                                                         [first].connect_peers,
1011                                                         &hash_second,
1012                                                         pg->
1013                                                         peers[second].daemon,
1014                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1015       pg->peers[first].num_connections++;
1016       added++;
1017     }
1018
1019   if (add_second)
1020     {
1021       GNUNET_assert (GNUNET_OK ==
1022                      GNUNET_CONTAINER_multihashmap_put (pg->
1023                                                         peers
1024                                                         [second].connect_peers,
1025                                                         &hash_first,
1026                                                         pg->
1027                                                         peers[first].daemon,
1028                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1029       pg->peers[second].num_connections++;
1030       added++;
1031     }
1032
1033   return added;
1034 }
1035
1036
1037 /*
1038  * Add entries to the peers allowed connections list
1039  *
1040  * @param pg the peer group we are working with
1041  * @param first index of the first peer
1042  * @param second index of the second peer
1043  *
1044  * @return the number of connections added (can be 0, 1 or 2)
1045  *         technically should only be 0 or 2, but the small price
1046  *         of iterating over the lists (hashmaps in the future)
1047  *         for being sure doesn't bother me!
1048  *
1049  */
1050 static unsigned int
1051 add_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg,
1052                          unsigned int first, unsigned int second)
1053 {
1054   int added;
1055 #if OLD
1056   struct PeerConnection *first_iter;
1057   struct PeerConnection *second_iter;
1058   struct PeerConnection *new_first;
1059   struct PeerConnection *new_second;
1060 #endif
1061   int add_first;
1062   int add_second;
1063
1064   GNUNET_HashCode hash_first;
1065   GNUNET_HashCode hash_second;
1066
1067   hash_from_uid (first, &hash_first);
1068   hash_from_uid (second, &hash_second);
1069
1070   add_first = GNUNET_NO;
1071   if (GNUNET_NO ==
1072       GNUNET_CONTAINER_multihashmap_contains (pg->peers[first].allowed_peers,
1073                                               &hash_second))
1074     {
1075       add_first = GNUNET_YES;
1076     }
1077
1078   add_second = GNUNET_NO;
1079   if (GNUNET_NO ==
1080       GNUNET_CONTAINER_multihashmap_contains (pg->peers[second].allowed_peers,
1081                                               &hash_first))
1082     {
1083       add_second = GNUNET_YES;
1084     }
1085 #if OLD
1086   first_iter = pg->peers[first].connected_peers;
1087   while (first_iter != NULL)
1088     {
1089       if (first_iter->daemon == pg->peers[second].daemon)
1090         add_first = GNUNET_NO;
1091       first_iter = first_iter->next;
1092     }
1093
1094   second_iter = pg->peers[second].connected_peers;
1095   add_second = GNUNET_YES;
1096   while (second_iter != NULL)
1097     {
1098       if (second_iter->daemon == pg->peers[first].daemon)
1099         add_second = GNUNET_NO;
1100       second_iter = second_iter->next;
1101     }
1102 #endif
1103
1104   added = 0;
1105   if (add_first)
1106     {
1107       GNUNET_assert (GNUNET_OK ==
1108                      GNUNET_CONTAINER_multihashmap_put (pg->
1109                                                         peers
1110                                                         [first].allowed_peers,
1111                                                         &hash_second,
1112                                                         pg->
1113                                                         peers[second].daemon,
1114                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1115 #if OLD
1116       new_first = GNUNET_malloc (sizeof (struct PeerConnection));
1117       new_first->daemon = pg->peers[second].daemon;
1118       new_first->next = pg->peers[first].connected_peers;
1119       pg->peers[first].connected_peers = new_first;
1120 #endif
1121       pg->peers[first].num_connections++;
1122       added++;
1123     }
1124
1125   if (add_second)
1126     {
1127       GNUNET_assert (GNUNET_OK ==
1128                      GNUNET_CONTAINER_multihashmap_put (pg->
1129                                                         peers
1130                                                         [second].allowed_peers,
1131                                                         &hash_first,
1132                                                         pg->
1133                                                         peers[first].daemon,
1134                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1135 #if OLD
1136       new_second = GNUNET_malloc (sizeof (struct PeerConnection));
1137       new_second->daemon = pg->peers[first].daemon;
1138       new_second->next = pg->peers[second].connected_peers;
1139       pg->peers[second].connected_peers = new_second;
1140       pg->peers[first].num_connections++;
1141 #endif
1142       pg->peers[second].num_connections++;
1143       added++;
1144     }
1145
1146   return added;
1147 }
1148
1149 /*
1150  * Add entries to the peers blacklisted list
1151  *
1152  * @param pg the peer group we are working with
1153  * @param first index of the first peer
1154  * @param second index of the second peer
1155  *
1156  * @return the number of connections added (can be 0, 1 or 2)
1157  *
1158  */
1159 static unsigned int
1160 blacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1161                        unsigned int first, unsigned int second)
1162 {
1163   int added;
1164   int add_first;
1165   int add_second;
1166   GNUNET_HashCode hash_first;
1167   GNUNET_HashCode hash_second;
1168
1169   hash_from_uid (first, &hash_first);
1170   hash_from_uid (second, &hash_second);
1171
1172   add_first = GNUNET_NO;
1173   if (GNUNET_NO ==
1174       GNUNET_CONTAINER_multihashmap_contains (pg->
1175                                               peers[first].blacklisted_peers,
1176                                               &hash_second))
1177     {
1178       add_first = GNUNET_YES;
1179     }
1180
1181   add_second = GNUNET_NO;
1182   if (GNUNET_NO ==
1183       GNUNET_CONTAINER_multihashmap_contains (pg->
1184                                               peers[second].blacklisted_peers,
1185                                               &hash_first))
1186     {
1187       add_second = GNUNET_YES;
1188     }
1189
1190   added = 0;
1191   if (add_first)
1192     {
1193       GNUNET_assert (GNUNET_OK ==
1194                      GNUNET_CONTAINER_multihashmap_put (pg->
1195                                                         peers
1196                                                         [first].blacklisted_peers,
1197                                                         &hash_second,
1198                                                         pg->
1199                                                         peers[second].daemon,
1200                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1201       pg->peers[first].num_connections++;
1202       added++;
1203     }
1204
1205   if (add_second)
1206     {
1207       GNUNET_assert (GNUNET_OK ==
1208                      GNUNET_CONTAINER_multihashmap_put (pg->
1209                                                         peers
1210                                                         [second].blacklisted_peers,
1211                                                         &hash_first,
1212                                                         pg->
1213                                                         peers[first].daemon,
1214                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1215       pg->peers[second].num_connections++;
1216       added++;
1217     }
1218
1219   return added;
1220 }
1221
1222 /*
1223  * Remove entries from the peers blacklisted list
1224  *
1225  * @param pg the peer group we are working with
1226  * @param first index of the first peer
1227  * @param second index of the second peer
1228  *
1229  * @return the number of connections removed (can be 0, 1 or 2)
1230  *
1231  */
1232 static unsigned int
1233 unblacklist_connections (struct GNUNET_TESTING_PeerGroup *pg,
1234                          unsigned int first, unsigned int second)
1235 {
1236   int removed;
1237   int remove_first;
1238   int remove_second;
1239   GNUNET_HashCode hash_first;
1240   GNUNET_HashCode hash_second;
1241
1242   hash_from_uid (first, &hash_first);
1243   hash_from_uid (second, &hash_second);
1244
1245   remove_first =
1246     GNUNET_CONTAINER_multihashmap_contains (pg->
1247                                             peers[first].blacklisted_peers,
1248                                             &hash_second);
1249   remove_second =
1250     GNUNET_CONTAINER_multihashmap_contains (pg->
1251                                             peers[second].blacklisted_peers,
1252                                             &hash_first);
1253
1254   removed = 0;
1255   if (remove_first)
1256     {
1257       GNUNET_assert (GNUNET_YES ==
1258                      GNUNET_CONTAINER_multihashmap_remove (pg->
1259                                                            peers
1260                                                            [first].blacklisted_peers,
1261                                                            &hash_second,
1262                                                            pg->
1263                                                            peers
1264                                                            [second].daemon));
1265       removed++;
1266     }
1267
1268   if (remove_second)
1269     {
1270       GNUNET_assert (GNUNET_YES ==
1271                      GNUNET_CONTAINER_multihashmap_remove (pg->
1272                                                            peers
1273                                                            [second].blacklisted_peers,
1274                                                            &hash_first,
1275                                                            pg->
1276                                                            peers
1277                                                            [first].daemon));
1278       removed++;
1279     }
1280
1281   return removed;
1282 }
1283
1284 /**
1285  * Scale free network construction as described in:
1286  *
1287  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1288  *
1289  * Start with a network of "one" peer, then progressively add
1290  * peers up to the total number.  At each step, iterate over
1291  * all possible peers and connect new peer based on number of
1292  * existing connections of the target peer.
1293  *
1294  * @param pg the peer group we are dealing with
1295  * @param proc the connection processor to use
1296  *
1297  * @return the number of connections created
1298  */
1299 static unsigned int
1300 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg,
1301                    GNUNET_TESTING_ConnectionProcessor proc)
1302 {
1303
1304   unsigned int total_connections;
1305   unsigned int outer_count;
1306   unsigned int i;
1307   unsigned int previous_total_connections;
1308   double random;
1309   double probability;
1310
1311   GNUNET_assert (pg->total > 1);
1312
1313   /* Add a connection between the first two nodes */
1314   total_connections = proc (pg, 0, 1);
1315
1316   for (outer_count = 1; outer_count < pg->total; outer_count++)
1317     {
1318       previous_total_connections = total_connections;
1319       for (i = 0; i < outer_count; i++)
1320         {
1321           probability =
1322             pg->peers[i].num_connections /
1323             (double) previous_total_connections;
1324           random =
1325             ((double)
1326              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1327                                        UINT64_MAX)) / ((double) UINT64_MAX);
1328 #if VERBOSE_TESTING
1329           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1330                       "Considering connecting peer %d to peer %d\n",
1331                       outer_count, i);
1332 #endif
1333           if (random < probability)
1334             {
1335 #if VERBOSE_TESTING
1336               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1337                           "Connecting peer %d to peer %d\n", outer_count, i);
1338 #endif
1339               total_connections += proc (pg, outer_count, i);
1340             }
1341         }
1342     }
1343
1344   return total_connections;
1345 }
1346
1347 /**
1348  * Create a topology given a peer group (set of running peers)
1349  * and a connection processor.
1350  *
1351  * @param pg the peergroup to create the topology on
1352  * @param proc the connection processor to call to actually set
1353  *        up connections between two peers
1354  *
1355  * @return the number of connections that were set up
1356  *
1357  */
1358 static unsigned int
1359 create_small_world_ring (struct GNUNET_TESTING_PeerGroup *pg,
1360                          GNUNET_TESTING_ConnectionProcessor proc)
1361 {
1362   unsigned int i, j;
1363   int nodeToConnect;
1364   unsigned int natLog;
1365   unsigned int randomPeer;
1366   double random, logNModifier, percentage;
1367   unsigned int smallWorldConnections;
1368   int connsPerPeer;
1369   char *p_string;
1370   int max;
1371   int min;
1372   unsigned int useAnd;
1373   int connect_attempts;
1374
1375   logNModifier = 0.5;           /* FIXME: default value? */
1376   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1377                                                           "TESTING",
1378                                                           "LOGNMODIFIER",
1379                                                           &p_string))
1380     {
1381       if (sscanf (p_string, "%lf", &logNModifier) != 1)
1382         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1383                     _
1384                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1385                     p_string, "LOGNMODIFIER", "TESTING");
1386       GNUNET_free (p_string);
1387     }
1388   percentage = 0.5;             /* FIXME: default percentage? */
1389   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1390                                                           "TESTING",
1391                                                           "PERCENTAGE",
1392                                                           &p_string))
1393     {
1394       if (sscanf (p_string, "%lf", &percentage) != 1)
1395         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1396                     _
1397                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1398                     p_string, "PERCENTAGE", "TESTING");
1399       GNUNET_free (p_string);
1400     }
1401   natLog = log (pg->total);
1402   connsPerPeer = ceil (natLog * logNModifier);
1403
1404   if (connsPerPeer % 2 == 1)
1405     connsPerPeer += 1;
1406
1407   smallWorldConnections = 0;
1408   connect_attempts = 0;
1409   for (i = 0; i < pg->total; i++)
1410     {
1411       useAnd = 0;
1412       max = i + connsPerPeer / 2;
1413       min = i - connsPerPeer / 2;
1414
1415       if (max > pg->total - 1)
1416         {
1417           max = max - pg->total;
1418           useAnd = 1;
1419         }
1420
1421       if (min < 0)
1422         {
1423           min = pg->total - 1 + min;
1424           useAnd = 1;
1425         }
1426
1427       for (j = 0; j < connsPerPeer / 2; j++)
1428         {
1429           random =
1430             ((double)
1431              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1432                                        UINT64_MAX) / ((double) UINT64_MAX));
1433           if (random < percentage)
1434             {
1435               /* Connect to uniformly selected random peer */
1436               randomPeer =
1437                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1438                                           pg->total);
1439               while ((((randomPeer < max) && (randomPeer > min))
1440                       && (useAnd == 0)) || (((randomPeer > min)
1441                                              || (randomPeer < max))
1442                                             && (useAnd == 1)))
1443                 {
1444                   randomPeer =
1445                     GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1446                                               pg->total);
1447                 }
1448               smallWorldConnections += proc (pg, i, randomPeer);
1449             }
1450           else
1451             {
1452               nodeToConnect = i + j + 1;
1453               if (nodeToConnect > pg->total - 1)
1454                 {
1455                   nodeToConnect = nodeToConnect - pg->total;
1456                 }
1457               connect_attempts += proc (pg, i, nodeToConnect);
1458             }
1459         }
1460
1461     }
1462
1463   connect_attempts += smallWorldConnections;
1464
1465   return connect_attempts;
1466 }
1467
1468 /**
1469  * Create a topology given a peer group (set of running peers)
1470  * and a connection processor.
1471  *
1472  * @param pg the peergroup to create the topology on
1473  * @param proc the connection processor to call to actually set
1474  *        up connections between two peers
1475  *
1476  * @return the number of connections that were set up
1477  *
1478  */
1479 static unsigned int
1480 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg,
1481                        GNUNET_TESTING_ConnectionProcessor proc)
1482 {
1483   unsigned int outer_count, inner_count;
1484   unsigned int cutoff;
1485   int connect_attempts;
1486   double nat_percentage;
1487   char *p_string;
1488
1489   nat_percentage = 0.6;         /* FIXME: default percentage? */
1490   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1491                                                           "TESTING",
1492                                                           "PERCENTAGE",
1493                                                           &p_string))
1494     {
1495       if (sscanf (p_string, "%lf", &nat_percentage) != 1)
1496         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1497                     _
1498                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1499                     p_string, "PERCENTAGE", "TESTING");
1500       GNUNET_free (p_string);
1501     }
1502
1503
1504
1505   cutoff = (unsigned int) (nat_percentage * pg->total);
1506
1507   connect_attempts = 0;
1508
1509   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1510     {
1511       for (inner_count = outer_count + 1; inner_count < pg->total;
1512            inner_count++)
1513         {
1514           if ((outer_count > cutoff) || (inner_count > cutoff))
1515             {
1516 #if VERBOSE_TESTING
1517               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1518                           "Connecting peer %d to peer %d\n",
1519                           outer_count, inner_count);
1520 #endif
1521               connect_attempts += proc (pg, outer_count, inner_count);
1522             }
1523         }
1524     }
1525
1526   return connect_attempts;
1527
1528 }
1529
1530 /**
1531  * Create a topology given a peer group (set of running peers)
1532  * and a connection processor.
1533  *
1534  * @param pg the peergroup to create the topology on
1535  * @param proc the connection processor to call to actually set
1536  *        up connections between two peers
1537  *
1538  * @return the number of connections that were set up
1539  *
1540  */
1541 static unsigned int
1542 create_small_world (struct GNUNET_TESTING_PeerGroup *pg,
1543                     GNUNET_TESTING_ConnectionProcessor proc)
1544 {
1545   unsigned int i, j, k;
1546   unsigned int square;
1547   unsigned int rows;
1548   unsigned int cols;
1549   unsigned int toggle = 1;
1550   unsigned int nodeToConnect;
1551   unsigned int natLog;
1552   unsigned int node1Row;
1553   unsigned int node1Col;
1554   unsigned int node2Row;
1555   unsigned int node2Col;
1556   unsigned int distance;
1557   double probability, random, percentage;
1558   unsigned int smallWorldConnections;
1559   unsigned int small_world_it;
1560   char *p_string;
1561   int connect_attempts;
1562   square = floor (sqrt (pg->total));
1563   rows = square;
1564   cols = square;
1565
1566   percentage = 0.5;             /* FIXME: default percentage? */
1567   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1568                                                           "TESTING",
1569                                                           "PERCENTAGE",
1570                                                           &p_string))
1571     {
1572       if (sscanf (p_string, "%lf", &percentage) != 1)
1573         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1574                     _
1575                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1576                     p_string, "PERCENTAGE", "TESTING");
1577       GNUNET_free (p_string);
1578     }
1579   if (percentage < 0.0)
1580     {
1581       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1582                   _
1583                   ("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1584                   "PERCENTAGE", "TESTING", percentage);
1585       percentage = 0.5;
1586     }
1587   probability = 0.5;            /* FIXME: default percentage? */
1588   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1589                                                           "TESTING",
1590                                                           "PROBABILITY",
1591                                                           &p_string))
1592     {
1593       if (sscanf (p_string, "%lf", &probability) != 1)
1594         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1595                     _
1596                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1597                     p_string, "PROBABILITY", "TESTING");
1598       GNUNET_free (p_string);
1599     }
1600   if (square * square != pg->total)
1601     {
1602       while (rows * cols < pg->total)
1603         {
1604           if (toggle % 2 == 0)
1605             rows++;
1606           else
1607             cols++;
1608
1609           toggle++;
1610         }
1611     }
1612 #if VERBOSE_TESTING
1613   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1614               _
1615               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1616               rows, cols);
1617 #endif
1618
1619   connect_attempts = 0;
1620   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1621    * to the node to its right and above.  Once this is over, we'll have our torus!
1622    * Special case for the last node (if the rows and columns are not equal), connect
1623    * to the first in the row to maintain topology.
1624    */
1625   for (i = 0; i < pg->total; i++)
1626     {
1627       /* First connect to the node to the right */
1628       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1629         nodeToConnect = i + 1;
1630       else if (i + 1 == pg->total)
1631         nodeToConnect = rows * cols - cols;
1632       else
1633         nodeToConnect = i - cols + 1;
1634
1635       connect_attempts += proc (pg, i, nodeToConnect);
1636
1637       if (i < cols)
1638         nodeToConnect = (rows * cols) - cols + i;
1639       else
1640         nodeToConnect = i - cols;
1641
1642       if (nodeToConnect < pg->total)
1643         connect_attempts += proc (pg, i, nodeToConnect);
1644     }
1645   natLog = log (pg->total);
1646 #if VERBOSE_TESTING > 2
1647   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1648               _("natural log of %d is %d, will run %d iterations\n"),
1649               pg->total, natLog, (int) (natLog * percentage));
1650   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1651               _("Total connections added thus far: %u!\n"), connect_attempts);
1652 #endif
1653   smallWorldConnections = 0;
1654   small_world_it = (unsigned int) (natLog * percentage);
1655   if (small_world_it < 1)
1656     small_world_it = 1;
1657   GNUNET_assert (small_world_it > 0 && small_world_it < (unsigned int) -1);
1658   for (i = 0; i < small_world_it; i++)
1659     {
1660       for (j = 0; j < pg->total; j++)
1661         {
1662           /* Determine the row and column of node at position j on the 2d torus */
1663           node1Row = j / cols;
1664           node1Col = j - (node1Row * cols);
1665           for (k = 0; k < pg->total; k++)
1666             {
1667               /* Determine the row and column of node at position k on the 2d torus */
1668               node2Row = k / cols;
1669               node2Col = k - (node2Row * cols);
1670               /* Simple Cartesian distance */
1671               distance =
1672                 abs (node1Row - node2Row) + abs (node1Col - node2Col);
1673               if (distance > 1)
1674                 {
1675                   /* Calculate probability as 1 over the square of the distance */
1676                   probability = 1.0 / (distance * distance);
1677                   /* Choose a random value between 0 and 1 */
1678                   random =
1679                     ((double)
1680                      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1681                                                UINT64_MAX)) /
1682                     ((double) UINT64_MAX);
1683                   /* If random < probability, then connect the two nodes */
1684                   if (random < probability)
1685                     smallWorldConnections += proc (pg, j, k);
1686
1687                 }
1688             }
1689         }
1690     }
1691   connect_attempts += smallWorldConnections;
1692 #if VERBOSE_TESTING > 2
1693   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1694               _("Total connections added for small world: %d!\n"),
1695               smallWorldConnections);
1696 #endif
1697   return connect_attempts;
1698 }
1699
1700 /**
1701  * Create a topology given a peer group (set of running peers)
1702  * and a connection processor.
1703  *
1704  * @param pg the peergroup to create the topology on
1705  * @param proc the connection processor to call to actually set
1706  *        up connections between two peers
1707  *
1708  * @return the number of connections that were set up
1709  *
1710  */
1711 static unsigned int
1712 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg,
1713                     GNUNET_TESTING_ConnectionProcessor proc)
1714 {
1715   double temp_rand;
1716   unsigned int outer_count;
1717   unsigned int inner_count;
1718   int connect_attempts;
1719   double probability;
1720   char *p_string;
1721
1722   probability = 0.5;            /* FIXME: default percentage? */
1723   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (pg->cfg,
1724                                                           "TESTING",
1725                                                           "PROBABILITY",
1726                                                           &p_string))
1727     {
1728       if (sscanf (p_string, "%lf", &probability) != 1)
1729         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1730                     _
1731                     ("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1732                     p_string, "PROBABILITY", "TESTING");
1733       GNUNET_free (p_string);
1734     }
1735   connect_attempts = 0;
1736   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1737     {
1738       for (inner_count = outer_count + 1; inner_count < pg->total;
1739            inner_count++)
1740         {
1741           temp_rand =
1742             ((double)
1743              GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
1744                                        UINT64_MAX)) / ((double) UINT64_MAX);
1745 #if VERBOSE_TESTING
1746           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1747                       _("rand is %f probability is %f\n"), temp_rand,
1748                       probability);
1749 #endif
1750           if (temp_rand < probability)
1751             {
1752               connect_attempts += proc (pg, outer_count, inner_count);
1753             }
1754         }
1755     }
1756
1757   return connect_attempts;
1758 }
1759
1760 /**
1761  * Create a topology given a peer group (set of running peers)
1762  * and a connection processor.  This particular function creates
1763  * the connections for a 2d-torus, plus additional "closest"
1764  * connections per peer.
1765  *
1766  * @param pg the peergroup to create the topology on
1767  * @param proc the connection processor to call to actually set
1768  *        up connections between two peers
1769  *
1770  * @return the number of connections that were set up
1771  *
1772  */
1773 static unsigned int
1774 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg,
1775                  GNUNET_TESTING_ConnectionProcessor proc)
1776 {
1777   unsigned int i;
1778   unsigned int square;
1779   unsigned int rows;
1780   unsigned int cols;
1781   unsigned int toggle = 1;
1782   unsigned int nodeToConnect;
1783   int connect_attempts;
1784
1785   connect_attempts = 0;
1786
1787   square = floor (sqrt (pg->total));
1788   rows = square;
1789   cols = square;
1790
1791   if (square * square != pg->total)
1792     {
1793       while (rows * cols < pg->total)
1794         {
1795           if (toggle % 2 == 0)
1796             rows++;
1797           else
1798             cols++;
1799
1800           toggle++;
1801         }
1802     }
1803 #if VERBOSE_TESTING
1804   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1805               _
1806               ("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1807               rows, cols);
1808 #endif
1809   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1810    * to the node to its right and above.  Once this is over, we'll have our torus!
1811    * Special case for the last node (if the rows and columns are not equal), connect
1812    * to the first in the row to maintain topology.
1813    */
1814   for (i = 0; i < pg->total; i++)
1815     {
1816       /* First connect to the node to the right */
1817       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1818         nodeToConnect = i + 1;
1819       else if (i + 1 == pg->total)
1820         nodeToConnect = rows * cols - cols;
1821       else
1822         nodeToConnect = i - cols + 1;
1823 #if VERBOSE_TESTING
1824       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1825                   "Connecting peer %d to peer %d\n", i, nodeToConnect);
1826 #endif
1827       connect_attempts += proc (pg, i, nodeToConnect);
1828
1829       /* Second connect to the node immediately above */
1830       if (i < cols)
1831         nodeToConnect = (rows * cols) - cols + i;
1832       else
1833         nodeToConnect = i - cols;
1834
1835       if (nodeToConnect < pg->total)
1836         {
1837 #if VERBOSE_TESTING
1838           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1839                       "Connecting peer %d to peer %d\n", i, nodeToConnect);
1840 #endif
1841           connect_attempts += proc (pg, i, nodeToConnect);
1842         }
1843
1844     }
1845
1846   return connect_attempts;
1847 }
1848
1849
1850 /**
1851  * Create a topology given a peer group (set of running peers)
1852  * and a connection processor.
1853  *
1854  * @param pg the peergroup to create the topology on
1855  * @param proc the connection processor to call to actually set
1856  *        up connections between two peers
1857  *
1858  * @return the number of connections that were set up
1859  *
1860  */
1861 static unsigned int
1862 create_clique (struct GNUNET_TESTING_PeerGroup *pg,
1863                GNUNET_TESTING_ConnectionProcessor proc)
1864 {
1865   unsigned int outer_count;
1866   unsigned int inner_count;
1867   int connect_attempts;
1868
1869   connect_attempts = 0;
1870
1871   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1872     {
1873       for (inner_count = outer_count + 1; inner_count < pg->total;
1874            inner_count++)
1875         {
1876 #if VERBOSE_TESTING
1877           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1878                       "Connecting peer %d to peer %d\n",
1879                       outer_count, inner_count);
1880 #endif
1881           connect_attempts += proc (pg, outer_count, inner_count);
1882         }
1883     }
1884
1885   return connect_attempts;
1886 }
1887
1888 /**
1889  * Create a topology given a peer group (set of running peers)
1890  * and a connection processor.
1891  *
1892  * @param pg the peergroup to create the topology on
1893  * @param proc the connection processor to call to actually set
1894  *        up connections between two peers
1895  *
1896  * @return the number of connections that were set up
1897  *
1898  */
1899 static unsigned int
1900 create_line (struct GNUNET_TESTING_PeerGroup *pg,
1901              GNUNET_TESTING_ConnectionProcessor proc)
1902 {
1903   unsigned int count;
1904   int connect_attempts;
1905
1906   connect_attempts = 0;
1907
1908   /* Connect each peer to the next highest numbered peer */
1909   for (count = 0; count < pg->total - 1; count++)
1910     {
1911 #if VERBOSE_TESTING
1912       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1913                   "Connecting peer %d to peer %d\n", count, count + 1);
1914 #endif
1915       connect_attempts += proc (pg, count, count + 1);
1916     }
1917
1918   return connect_attempts;
1919 }
1920
1921 /**
1922  * Create a topology given a peer group (set of running peers)
1923  * and a connection processor.
1924  *
1925  * @param pg the peergroup to create the topology on
1926  * @param proc the connection processor to call to actually set
1927  *        up connections between two peers
1928  *
1929  * @return the number of connections that were set up
1930  *
1931  */
1932 static unsigned int
1933 create_ring (struct GNUNET_TESTING_PeerGroup *pg,
1934              GNUNET_TESTING_ConnectionProcessor proc)
1935 {
1936   unsigned int count;
1937   int connect_attempts;
1938
1939   connect_attempts = 0;
1940
1941   /* Connect each peer to the next highest numbered peer */
1942   for (count = 0; count < pg->total - 1; count++)
1943     {
1944 #if VERBOSE_TESTING
1945       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1946                   "Connecting peer %d to peer %d\n", count, count + 1);
1947 #endif
1948       connect_attempts += proc (pg, count, count + 1);
1949     }
1950
1951   /* Connect the last peer to the first peer */
1952   connect_attempts += proc (pg, pg->total - 1, 0);
1953
1954   return connect_attempts;
1955 }
1956
1957
1958 /**
1959  * Iterator for writing friends of a peer to a file.
1960  *
1961  * @param cls closure, an open writable file handle
1962  * @param key the key the daemon was stored under
1963  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1964  *
1965  * @return GNUNET_YES to continue iteration
1966  *
1967  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1968  *       with a single file_iterator that takes a closure which contains
1969  *       the prefix to write before the peer.  Then this could be used
1970  *       for blacklisting multiple transports and writing the friend
1971  *       file.  I'm sure *someone* will complain loudly about other
1972  *       things that negate these functions even existing so no point in
1973  *       "fixing" now.
1974  */
1975 static int
1976 friend_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
1977 {
1978   FILE *temp_friend_handle = cls;
1979   struct GNUNET_TESTING_Daemon *peer = value;
1980   struct GNUNET_PeerIdentity *temppeer;
1981   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1982
1983   temppeer = &peer->id;
1984   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
1985   fprintf (temp_friend_handle, "%s\n", (char *) &peer_enc);
1986
1987   return GNUNET_YES;
1988 }
1989
1990 struct BlacklistContext
1991 {
1992   /*
1993    * The (open) file handle to write to
1994    */
1995   FILE *temp_file_handle;
1996
1997   /*
1998    * The transport that this peer will be blacklisted on.
1999    */
2000   char *transport;
2001 };
2002
2003 /**
2004  * Iterator for writing blacklist data to appropriate files.
2005  *
2006  * @param cls closure, an open writable file handle
2007  * @param key the key the daemon was stored under
2008  * @param value the GNUNET_TESTING_Daemon that needs to be written.
2009  *
2010  * @return GNUNET_YES to continue iteration
2011  */
2012 static int
2013 blacklist_file_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2014 {
2015   struct BlacklistContext *blacklist_ctx = cls;
2016   //FILE *temp_blacklist_handle = cls;
2017   struct GNUNET_TESTING_Daemon *peer = value;
2018   struct GNUNET_PeerIdentity *temppeer;
2019   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
2020
2021   temppeer = &peer->id;
2022   GNUNET_CRYPTO_hash_to_enc (&temppeer->hashPubKey, &peer_enc);
2023   fprintf (blacklist_ctx->temp_file_handle, "%s:%s\n",
2024            blacklist_ctx->transport, (char *) &peer_enc);
2025
2026   return GNUNET_YES;
2027 }
2028
2029 /*
2030  * Create the friend files based on the PeerConnection's
2031  * of each peer in the peer group, and copy the files
2032  * to the appropriate place
2033  *
2034  * @param pg the peer group we are dealing with
2035  */
2036 static int
2037 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
2038 {
2039   FILE *temp_friend_handle;
2040   unsigned int pg_iter;
2041   char *temp_service_path;
2042   struct GNUNET_OS_Process **procarr;
2043   char *arg;
2044   char *mytemp;
2045   enum GNUNET_OS_ProcessStatusType type;
2046   unsigned long return_code;
2047   int count;
2048   int ret;
2049   int max_wait = 10;
2050
2051   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2052   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2053     {
2054       mytemp = GNUNET_DISK_mktemp ("friends");
2055       GNUNET_assert (mytemp != NULL);
2056       temp_friend_handle = fopen (mytemp, "wt");
2057       GNUNET_assert (temp_friend_handle != NULL);
2058       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].allowed_peers,
2059                                              &friend_file_iterator,
2060                                              temp_friend_handle);
2061       fclose (temp_friend_handle);
2062
2063       if (GNUNET_OK !=
2064           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2065                                                  daemon->cfg, "PATHS",
2066                                                  "SERVICEHOME",
2067                                                  &temp_service_path))
2068         {
2069           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2070                       _
2071                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2072                       "SERVICEHOME", "PATHS");
2073           if (UNLINK (mytemp) != 0)
2074             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2075                                       mytemp);
2076           GNUNET_free (mytemp);
2077           break;
2078         }
2079
2080       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2081         {
2082           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
2083           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2084                                                       "mv", mytemp, arg,
2085                                                       NULL);
2086 #if VERBOSE_TESTING
2087           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2088                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2089 #endif
2090
2091           GNUNET_free (arg);
2092         }
2093       else                      /* Remote, scp the file to the correct place */
2094         {
2095           if (NULL != pg->peers[pg_iter].daemon->username)
2096             GNUNET_asprintf (&arg, "%s@%s:%s/friends",
2097                              pg->peers[pg_iter].daemon->username,
2098                              pg->peers[pg_iter].daemon->hostname,
2099                              temp_service_path);
2100           else
2101             GNUNET_asprintf (&arg, "%s:%s/friends",
2102                              pg->peers[pg_iter].daemon->hostname,
2103                              temp_service_path);
2104           procarr[pg_iter] =
2105             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2106                                      NULL);
2107
2108 #if VERBOSE_TESTING
2109           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2110                       _("Copying file with command scp %s %s\n"), mytemp,
2111                       arg);
2112 #endif
2113           GNUNET_free (arg);
2114         }
2115       GNUNET_free (temp_service_path);
2116       GNUNET_free (mytemp);
2117     }
2118
2119   count = 0;
2120   ret = GNUNET_SYSERR;
2121   while ((count < max_wait) && (ret != GNUNET_OK))
2122     {
2123       ret = GNUNET_OK;
2124       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2125         {
2126 #if VERBOSE_TESTING
2127           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2128                       _("Checking copy status of file %d\n"), pg_iter);
2129 #endif
2130           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2131             {
2132               if (GNUNET_OS_process_status
2133                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2134                 {
2135                   ret = GNUNET_SYSERR;
2136                 }
2137               else if ((type != GNUNET_OS_PROCESS_EXITED)
2138                        || (return_code != 0))
2139                 {
2140                   ret = GNUNET_SYSERR;
2141                 }
2142               else
2143                 {
2144                   GNUNET_OS_process_close (procarr[pg_iter]);
2145                   procarr[pg_iter] = NULL;
2146 #if VERBOSE_TESTING
2147                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2148                               _("File %d copied\n"), pg_iter);
2149 #endif
2150                 }
2151             }
2152         }
2153       count++;
2154       if (ret == GNUNET_SYSERR)
2155         {
2156           /* FIXME: why sleep here? -CG */
2157           sleep (1);
2158         }
2159     }
2160
2161 #if VERBOSE_TESTING
2162   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2163               _("Finished copying all friend files!\n"));
2164 #endif
2165   GNUNET_free (procarr);
2166   return ret;
2167 }
2168
2169
2170 /*
2171  * Create the blacklist files based on the PeerConnection's
2172  * of each peer in the peer group, and copy the files
2173  * to the appropriate place.
2174  *
2175  * @param pg the peer group we are dealing with
2176  * @param transports space delimited list of transports to blacklist
2177  */
2178 static int
2179 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg,
2180                                  const char *transports)
2181 {
2182   FILE *temp_file_handle;
2183   static struct BlacklistContext blacklist_ctx;
2184   unsigned int pg_iter;
2185   char *temp_service_path;
2186   struct GNUNET_OS_Process **procarr;
2187   char *arg;
2188   char *mytemp;
2189   enum GNUNET_OS_ProcessStatusType type;
2190   unsigned long return_code;
2191   int count;
2192   int ret;
2193   int max_wait = 10;
2194   int transport_len;
2195   unsigned int i;
2196   char *pos;
2197   char *temp_transports;
2198
2199   procarr = GNUNET_malloc (sizeof (struct GNUNET_OS_Process *) * pg->total);
2200   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2201     {
2202       mytemp = GNUNET_DISK_mktemp ("blacklist");
2203       GNUNET_assert (mytemp != NULL);
2204       temp_file_handle = fopen (mytemp, "wt");
2205       GNUNET_assert (temp_file_handle != NULL);
2206       temp_transports = GNUNET_strdup (transports);
2207       blacklist_ctx.temp_file_handle = temp_file_handle;
2208       transport_len = strlen (temp_transports) + 1;
2209       pos = NULL;
2210
2211       for (i = 0; i < transport_len; i++)
2212         {
2213           if ((temp_transports[i] == ' ') && (pos == NULL))
2214             continue;           /* At start of string (whitespace) */
2215           else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
2216             {
2217               temp_transports[i] = '\0';
2218               blacklist_ctx.transport = pos;
2219               GNUNET_CONTAINER_multihashmap_iterate (pg->
2220                                                      peers
2221                                                      [pg_iter].blacklisted_peers,
2222                                                      &blacklist_file_iterator,
2223                                                      &blacklist_ctx);
2224               pos = NULL;
2225             }                   /* At beginning of actual string */
2226           else if (pos == NULL)
2227             {
2228               pos = &temp_transports[i];
2229             }
2230         }
2231
2232       GNUNET_free (temp_transports);
2233       fclose (temp_file_handle);
2234
2235       if (GNUNET_OK !=
2236           GNUNET_CONFIGURATION_get_value_string (pg->peers[pg_iter].
2237                                                  daemon->cfg, "PATHS",
2238                                                  "SERVICEHOME",
2239                                                  &temp_service_path))
2240         {
2241           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2242                       _
2243                       ("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2244                       "SERVICEHOME", "PATHS");
2245           if (UNLINK (mytemp) != 0)
2246             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink",
2247                                       mytemp);
2248           GNUNET_free (mytemp);
2249           break;
2250         }
2251
2252       if (pg->peers[pg_iter].daemon->hostname == NULL)  /* Local, just copy the file */
2253         {
2254           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
2255           procarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2256                                                       "mv", mytemp, arg,
2257                                                       NULL);
2258 #if VERBOSE_TESTING
2259           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2260                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2261 #endif
2262
2263           GNUNET_free (arg);
2264         }
2265       else                      /* Remote, scp the file to the correct place */
2266         {
2267           if (NULL != pg->peers[pg_iter].daemon->username)
2268             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist",
2269                              pg->peers[pg_iter].daemon->username,
2270                              pg->peers[pg_iter].daemon->hostname,
2271                              temp_service_path);
2272           else
2273             GNUNET_asprintf (&arg, "%s:%s/blacklist",
2274                              pg->peers[pg_iter].daemon->hostname,
2275                              temp_service_path);
2276           procarr[pg_iter] =
2277             GNUNET_OS_start_process (NULL, NULL, "scp", "scp", mytemp, arg,
2278                                      NULL);
2279
2280 #if VERBOSE_TESTING
2281           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2282                       _("Copying file with command scp %s %s\n"), mytemp,
2283                       arg);
2284 #endif
2285           GNUNET_free (arg);
2286         }
2287       GNUNET_free (temp_service_path);
2288       GNUNET_free (mytemp);
2289     }
2290
2291   count = 0;
2292   ret = GNUNET_SYSERR;
2293   while ((count < max_wait) && (ret != GNUNET_OK))
2294     {
2295       ret = GNUNET_OK;
2296       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2297         {
2298 #if VERBOSE_TESTING
2299           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2300                       _("Checking copy status of file %d\n"), pg_iter);
2301 #endif
2302           if (procarr[pg_iter] != NULL) /* Check for already completed! */
2303             {
2304               if (GNUNET_OS_process_status
2305                   (procarr[pg_iter], &type, &return_code) != GNUNET_OK)
2306                 {
2307                   ret = GNUNET_SYSERR;
2308                 }
2309               else if ((type != GNUNET_OS_PROCESS_EXITED)
2310                        || (return_code != 0))
2311                 {
2312                   ret = GNUNET_SYSERR;
2313                 }
2314               else
2315                 {
2316                   GNUNET_OS_process_close (procarr[pg_iter]);
2317                   procarr[pg_iter] = NULL;
2318 #if VERBOSE_TESTING
2319                   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2320                               _("File %d copied\n"), pg_iter);
2321 #endif
2322                 }
2323             }
2324         }
2325       count++;
2326       if (ret == GNUNET_SYSERR)
2327         {
2328           /* FIXME: why sleep here? -CG */
2329           sleep (1);
2330         }
2331     }
2332
2333 #if VERBOSE_TESTING
2334   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2335               _("Finished copying all blacklist files!\n"));
2336 #endif
2337   GNUNET_free (procarr);
2338   return ret;
2339 }
2340
2341
2342 /**
2343  * Internal notification of a connection, kept so that we can ensure some connections
2344  * happen instead of flooding all testing daemons with requests to connect.
2345  */
2346 static void
2347 internal_connect_notify (void *cls,
2348                          const struct GNUNET_PeerIdentity *first,
2349                          const struct GNUNET_PeerIdentity *second,
2350                          uint32_t distance,
2351                          const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2352                          const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2353                          struct GNUNET_TESTING_Daemon *first_daemon,
2354                          struct GNUNET_TESTING_Daemon *second_daemon,
2355                          const char *emsg)
2356 {
2357   struct ConnectTopologyContext *ct_ctx = cls;
2358   struct GNUNET_TESTING_PeerGroup *pg = ct_ctx->pg;
2359   outstanding_connects--;
2360   ct_ctx->remaining_connections--;
2361   if (ct_ctx->remaining_connections == 0)
2362     {
2363       if (ct_ctx->notify_connections_done != NULL)
2364         ct_ctx->notify_connections_done (ct_ctx->notify_cls, NULL);
2365       GNUNET_free (ct_ctx);
2366     }
2367
2368   if (pg->notify_connection != NULL)
2369     pg->notify_connection (pg->notify_connection_cls, first, second, distance,
2370                            first_cfg, second_cfg, first_daemon, second_daemon,
2371                            emsg);
2372 }
2373
2374
2375 /**
2376  * Either delay a connection (because there are too many outstanding)
2377  * or schedule it for right now.
2378  *
2379  * @param cls a connection context
2380  * @param tc the task runtime context
2381  */
2382 static void
2383 schedule_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2384 {
2385   struct ConnectContext *connect_context = cls;
2386
2387   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2388     return;
2389
2390   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2391     {
2392 #if VERBOSE_TESTING > 2
2393       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2394                   _
2395                   ("Delaying connect, we have too many outstanding connections!\n"));
2396 #endif
2397       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
2398                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
2399                                     &schedule_connect, connect_context);
2400     }
2401   else
2402     {
2403 #if VERBOSE_TESTING > 2
2404       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2405                   _("Creating connection, outstanding_connections is %d\n"),
2406                   outstanding_connects);
2407 #endif
2408       outstanding_connects++;
2409       GNUNET_TESTING_daemons_connect (connect_context->first,
2410                                       connect_context->second,
2411                                       CONNECT_TIMEOUT,
2412                                       CONNECT_ATTEMPTS,
2413                                       &internal_connect_notify,
2414                                       connect_context->ct_ctx);
2415       GNUNET_free (connect_context);
2416     }
2417 }
2418
2419
2420 /**
2421  * Iterator for actually scheduling connections to be created
2422  * between two peers.
2423  *
2424  * @param cls closure, a GNUNET_TESTING_Daemon
2425  * @param key the key the second Daemon was stored under
2426  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2427  *
2428  * @return GNUNET_YES to continue iteration
2429  */
2430 static int
2431 connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2432 {
2433   struct ConnectTopologyContext *ct_ctx = cls;
2434   struct PeerData *first = ct_ctx->first;
2435   struct GNUNET_TESTING_Daemon *second = value;
2436   struct ConnectContext *connect_context;
2437
2438   connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2439   connect_context->first = first->daemon;
2440   connect_context->second = second;
2441   connect_context->ct_ctx = ct_ctx;
2442   GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2443
2444   return GNUNET_YES;
2445 }
2446
2447
2448 /**
2449  * Iterator for copying all entries in the allowed hashmap to the
2450  * connect hashmap.
2451  *
2452  * @param cls closure, a GNUNET_TESTING_Daemon
2453  * @param key the key the second Daemon was stored under
2454  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2455  *
2456  * @return GNUNET_YES to continue iteration
2457  */
2458 static int
2459 copy_topology_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2460 {
2461   struct PeerData *first = cls;
2462
2463   GNUNET_assert (GNUNET_OK ==
2464                  GNUNET_CONTAINER_multihashmap_put (first->connect_peers, key,
2465                                                     value,
2466                                                     GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2467
2468   return GNUNET_YES;
2469 }
2470
2471 /**
2472  * Make the peers to connect the same as those that are allowed to be
2473  * connected.
2474  *
2475  * @param pg the peer group
2476  */
2477 static int
2478 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2479 {
2480   unsigned int pg_iter;
2481   int ret;
2482   int total;
2483
2484   total = 0;
2485   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2486     {
2487       ret =
2488         GNUNET_CONTAINER_multihashmap_iterate (pg->
2489                                                peers[pg_iter].allowed_peers,
2490                                                &copy_topology_iterator,
2491                                                &pg->peers[pg_iter]);
2492       if (GNUNET_SYSERR == ret)
2493         return GNUNET_SYSERR;
2494
2495       total = total + ret;
2496     }
2497
2498   return total;
2499 }
2500
2501
2502 /**
2503  * Connect the topology as specified by the PeerConnection's
2504  * of each peer in the peer group
2505  *
2506  * @param pg the peer group we are dealing with
2507  * @return the number of connections that will be attempted
2508  */
2509 static int
2510 connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
2511                   GNUNET_TESTING_NotifyCompletion notify_callback,
2512                   void *notify_cls)
2513 {
2514   unsigned int pg_iter;
2515   int ret;
2516   unsigned int total;
2517   struct ConnectTopologyContext *ct_ctx;
2518 #if OLD
2519   struct PeerConnection *connection_iter;
2520   struct ConnectContext *connect_context;
2521 #endif
2522
2523   total = 0;
2524   ct_ctx = GNUNET_malloc (sizeof (struct ConnectTopologyContext));
2525   ct_ctx->notify_connections_done = notify_callback;
2526   ct_ctx->notify_cls = notify_cls;
2527   ct_ctx->pg = pg;
2528
2529   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2530     {
2531       total +=
2532         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
2533     }
2534
2535   if (total == 0)
2536     {
2537       GNUNET_free (ct_ctx);
2538       return total;
2539     }
2540   ct_ctx->remaining_connections = total;
2541   total = 0;
2542
2543   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2544     {
2545       ct_ctx->first = &pg->peers[pg_iter];
2546       ret =
2547         GNUNET_CONTAINER_multihashmap_iterate (pg->
2548                                                peers[pg_iter].connect_peers,
2549                                                &connect_iterator, ct_ctx);
2550       GNUNET_assert (GNUNET_SYSERR != ret && ret >= 0);
2551       total = total + ret;
2552
2553 #if OLD
2554       connection_iter = FIXME;
2555       while (connection_iter != NULL)
2556         {
2557           connect_context = GNUNET_malloc (sizeof (struct ConnectContext));
2558           connect_context->pg = pg;
2559           connect_context->first = FIXME;
2560           connect_context->second = connection_iter->daemon;
2561           GNUNET_SCHEDULER_add_now (&schedule_connect, connect_context);
2562           connection_iter = connection_iter->next;
2563         }
2564 #endif
2565     }
2566   return total;
2567 }
2568
2569
2570 /**
2571  * Takes a peer group and creates a topology based on the
2572  * one specified.  Creates a topology means generates friend
2573  * files for the peers so they can only connect to those allowed
2574  * by the topology.  This will only have an effect once peers
2575  * are started if the FRIENDS_ONLY option is set in the base
2576  * config.  Also takes an optional restrict topology which
2577  * disallows connections based on particular transports
2578  * UNLESS they are specified in the restricted topology.
2579  *
2580  * @param pg the peer group struct representing the running peers
2581  * @param topology which topology to connect the peers in
2582  * @param restrict_topology disallow restrict_transports transport
2583  *                          connections to peers NOT in this topology
2584  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2585  * @param restrict_transports space delimited list of transports to blacklist
2586  *                            to create restricted topology
2587  *
2588  * @return the maximum number of connections were all allowed peers
2589  *         connected to each other
2590  */
2591 unsigned int
2592 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2593                                 enum GNUNET_TESTING_Topology topology,
2594                                 enum GNUNET_TESTING_Topology
2595                                 restrict_topology,
2596                                 const char *restrict_transports)
2597 {
2598   int ret;
2599   unsigned int num_connections;
2600   int unblacklisted_connections;
2601
2602   switch (topology)
2603     {
2604     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2605 #if VERBOSE_TESTING
2606       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating clique topology\n"));
2607 #endif
2608       num_connections = create_clique (pg, &add_allowed_connections);
2609       break;
2610     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2611 #if VERBOSE_TESTING
2612       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2613                   _("Creating small world (ring) topology\n"));
2614 #endif
2615       num_connections =
2616         create_small_world_ring (pg, &add_allowed_connections);
2617       break;
2618     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2619 #if VERBOSE_TESTING
2620       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2621                   _("Creating small world (2d-torus) topology\n"));
2622 #endif
2623       num_connections = create_small_world (pg, &add_allowed_connections);
2624       break;
2625     case GNUNET_TESTING_TOPOLOGY_RING:
2626 #if VERBOSE_TESTING
2627       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating ring topology\n"));
2628 #endif
2629       num_connections = create_ring (pg, &add_allowed_connections);
2630       break;
2631     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2632 #if VERBOSE_TESTING
2633       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating 2d torus topology\n"));
2634 #endif
2635       num_connections = create_2d_torus (pg, &add_allowed_connections);
2636       break;
2637     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2638 #if VERBOSE_TESTING
2639       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2640                   _("Creating Erdos-Renyi topology\n"));
2641 #endif
2642       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2643       break;
2644     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2645 #if VERBOSE_TESTING
2646       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Creating InterNAT topology\n"));
2647 #endif
2648       num_connections = create_nated_internet (pg, &add_allowed_connections);
2649       break;
2650     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2651 #if VERBOSE_TESTING
2652       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2653                   _("Creating Scale Free topology\n"));
2654 #endif
2655       num_connections = create_scale_free (pg, &add_allowed_connections);
2656       break;
2657     case GNUNET_TESTING_TOPOLOGY_LINE:
2658 #if VERBOSE_TESTING
2659       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2660                   _("Creating straight line topology\n"));
2661 #endif
2662       num_connections = create_line (pg, &add_allowed_connections);
2663       break;
2664     case GNUNET_TESTING_TOPOLOGY_NONE:
2665 #if VERBOSE_TESTING
2666       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2667                   _
2668                   ("Creating no allowed topology (all peers can connect at core level)\n"));
2669 #endif
2670       num_connections = 0;
2671       break;
2672     default:
2673       num_connections = 0;
2674       break;
2675     }
2676
2677   if (GNUNET_YES ==
2678       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2679     {
2680       ret = create_and_copy_friend_files (pg);
2681       if (ret != GNUNET_OK)
2682         {
2683 #if VERBOSE_TESTING
2684           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2685                       _("Failed during friend file copying!\n"));
2686 #endif
2687           return GNUNET_SYSERR;
2688         }
2689       else
2690         {
2691 #if VERBOSE_TESTING
2692           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2693                       _("Friend files created/copied successfully!\n"));
2694 #endif
2695         }
2696     }
2697
2698   /* Use the create clique method to initially set all connections as blacklisted. */
2699   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2700     create_clique (pg, &blacklist_connections);
2701
2702   unblacklisted_connections = 0;
2703   /* Un-blacklist connections as per the topology specified */
2704   switch (restrict_topology)
2705     {
2706     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2707 #if VERBOSE_TESTING
2708       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2709                   _("Blacklisting all but clique topology\n"));
2710 #endif
2711       unblacklisted_connections =
2712         create_clique (pg, &unblacklist_connections);
2713       break;
2714     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2715 #if VERBOSE_TESTING
2716       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2717                   _("Blacklisting all but small world (ring) topology\n"));
2718 #endif
2719       unblacklisted_connections =
2720         create_small_world_ring (pg, &unblacklist_connections);
2721       break;
2722     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2723 #if VERBOSE_TESTING
2724       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2725                   _
2726                   ("Blacklisting all but small world (2d-torus) topology\n"));
2727 #endif
2728       unblacklisted_connections =
2729         create_small_world (pg, &unblacklist_connections);
2730       break;
2731     case GNUNET_TESTING_TOPOLOGY_RING:
2732 #if VERBOSE_TESTING
2733       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2734                   _("Blacklisting all but ring topology\n"));
2735 #endif
2736       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2737       break;
2738     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2739 #if VERBOSE_TESTING
2740       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2741                   _("Blacklisting all but 2d torus topology\n"));
2742 #endif
2743       unblacklisted_connections =
2744         create_2d_torus (pg, &unblacklist_connections);
2745       break;
2746     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2747 #if VERBOSE_TESTING
2748       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2749                   _("Blacklisting all but Erdos-Renyi topology\n"));
2750 #endif
2751       unblacklisted_connections =
2752         create_erdos_renyi (pg, &unblacklist_connections);
2753       break;
2754     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2755 #if VERBOSE_TESTING
2756       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2757                   _("Blacklisting all but InterNAT topology\n"));
2758 #endif
2759       unblacklisted_connections =
2760         create_nated_internet (pg, &unblacklist_connections);
2761       break;
2762     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2763 #if VERBOSE_TESTING
2764       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2765                   _("Blacklisting all but Scale Free topology\n"));
2766 #endif
2767       unblacklisted_connections =
2768         create_scale_free (pg, &unblacklist_connections);
2769       break;
2770     case GNUNET_TESTING_TOPOLOGY_LINE:
2771 #if VERBOSE_TESTING
2772       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2773                   _("Blacklisting all but straight line topology\n"));
2774 #endif
2775       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2776       break;
2777     case GNUNET_TESTING_TOPOLOGY_NONE:
2778 #if VERBOSE_TESTING
2779       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2780                   _
2781                   ("Creating no blacklist topology (all peers can connect at transport level)\n"));
2782 #endif
2783     default:
2784       break;
2785     }
2786
2787   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2788     {
2789       ret = create_and_copy_blacklist_files (pg, restrict_transports);
2790       if (ret != GNUNET_OK)
2791         {
2792 #if VERBOSE_TESTING
2793           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2794                       _("Failed during blacklist file copying!\n"));
2795 #endif
2796           return 0;
2797         }
2798       else
2799         {
2800 #if VERBOSE_TESTING
2801           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2802                       _("Blacklist files created/copied successfully!\n"));
2803 #endif
2804         }
2805     }
2806   return num_connections;
2807 }
2808
2809 struct RandomContext
2810 {
2811   /**
2812    * The peergroup
2813    */
2814   struct GNUNET_TESTING_PeerGroup *pg;
2815
2816   /**
2817    * uid of the first peer
2818    */
2819   uint32_t first_uid;
2820
2821   /**
2822    * Peer data for first peer.
2823    */
2824   struct PeerData *first;
2825
2826   /**
2827    * Random percentage to use
2828    */
2829   double percentage;
2830 };
2831
2832 struct MinimumContext
2833 {
2834   /**
2835    * The peergroup
2836    */
2837   struct GNUNET_TESTING_PeerGroup *pg;
2838
2839   /**
2840    * uid of the first peer
2841    */
2842   uint32_t first_uid;
2843
2844   /**
2845    * Peer data for first peer.
2846    */
2847   struct PeerData *first;
2848
2849   /**
2850    * Number of conns per peer
2851    */
2852   unsigned int num_to_add;
2853
2854   /**
2855    * Permuted array of all possible connections.  Only add the Nth
2856    * peer if it's in the Nth position.
2857    */
2858   unsigned int *pg_array;
2859
2860   /**
2861    * What number is the current element we are iterating over?
2862    */
2863   unsigned int current;
2864 };
2865
2866 struct DFSContext
2867 {
2868   /**
2869    * The peergroup
2870    */
2871   struct GNUNET_TESTING_PeerGroup *pg;
2872
2873   /**
2874    * uid of the first peer
2875    */
2876   uint32_t first_uid;
2877
2878   /**
2879    * uid of the second peer
2880    */
2881   uint32_t second_uid;
2882
2883   /**
2884    * Peer data for first peer.
2885    */
2886   struct PeerData *first;
2887
2888   /**
2889    * Which peer has been chosen as the one to add?
2890    */
2891   unsigned int chosen;
2892
2893   /**
2894    * What number is the current element we are iterating over?
2895    */
2896   unsigned int current;
2897 };
2898
2899 /**
2900  * Iterator for choosing random peers to connect.
2901  *
2902  * @param cls closure, a RandomContext
2903  * @param key the key the second Daemon was stored under
2904  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2905  *
2906  * @return GNUNET_YES to continue iteration
2907  */
2908 static int
2909 random_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2910 {
2911   struct RandomContext *random_ctx = cls;
2912   double random_number;
2913   uint32_t second_pos;
2914   GNUNET_HashCode first_hash;
2915   random_number =
2916     ((double)
2917      GNUNET_CRYPTO_random_u64 (GNUNET_CRYPTO_QUALITY_WEAK,
2918                                UINT64_MAX)) / ((double) UINT64_MAX);
2919   if (random_number < random_ctx->percentage)
2920     {
2921       GNUNET_assert (GNUNET_OK ==
2922                      GNUNET_CONTAINER_multihashmap_put (random_ctx->
2923                                                         first->connect_peers_working_set,
2924                                                         key, value,
2925                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2926     }
2927   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2928   uid_from_hash (key, &second_pos);
2929   hash_from_uid (random_ctx->first_uid, &first_hash);
2930   GNUNET_assert (random_ctx->pg->total > second_pos);
2931   GNUNET_assert (GNUNET_YES ==
2932                  GNUNET_CONTAINER_multihashmap_remove (random_ctx->
2933                                                        pg->peers
2934                                                        [second_pos].connect_peers,
2935                                                        &first_hash,
2936                                                        random_ctx->
2937                                                        first->daemon));
2938
2939   return GNUNET_YES;
2940 }
2941
2942 /**
2943  * Iterator for adding at least X peers to a peers connection set.
2944  *
2945  * @param cls closure, MinimumContext
2946  * @param key the key the second Daemon was stored under
2947  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2948  *
2949  * @return GNUNET_YES to continue iteration
2950  */
2951 static int
2952 minimum_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
2953 {
2954   struct MinimumContext *min_ctx = cls;
2955   uint32_t second_pos;
2956   GNUNET_HashCode first_hash;
2957   unsigned int i;
2958
2959   if (GNUNET_CONTAINER_multihashmap_size
2960       (min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2961     {
2962       for (i = 0; i < min_ctx->num_to_add; i++)
2963         {
2964           if (min_ctx->pg_array[i] == min_ctx->current)
2965             {
2966               GNUNET_assert (GNUNET_OK ==
2967                              GNUNET_CONTAINER_multihashmap_put
2968                              (min_ctx->first->connect_peers_working_set, key,
2969                               value,
2970                               GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2971               uid_from_hash (key, &second_pos);
2972               hash_from_uid (min_ctx->first_uid, &first_hash);
2973               GNUNET_assert (min_ctx->pg->total > second_pos);
2974               GNUNET_assert (GNUNET_OK ==
2975                              GNUNET_CONTAINER_multihashmap_put (min_ctx->
2976                                                                 pg->peers
2977                                                                 [second_pos].connect_peers_working_set,
2978                                                                 &first_hash,
2979                                                                 min_ctx->first->
2980                                                                 daemon,
2981                                                                 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2982               /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2983               GNUNET_assert (GNUNET_YES ==
2984                              GNUNET_CONTAINER_multihashmap_remove
2985                              (min_ctx->pg->peers[second_pos].connect_peers,
2986                               &first_hash, min_ctx->first->daemon));
2987             }
2988         }
2989       min_ctx->current++;
2990       return GNUNET_YES;
2991     }
2992   else
2993     return GNUNET_NO;           /* We can stop iterating, we have enough peers! */
2994
2995 }
2996
2997
2998 /**
2999  * Iterator for adding peers to a connection set based on a depth first search.
3000  *
3001  * @param cls closure, MinimumContext
3002  * @param key the key the second daemon was stored under
3003  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
3004  *
3005  * @return GNUNET_YES to continue iteration
3006  */
3007 static int
3008 dfs_connect_iterator (void *cls, const GNUNET_HashCode * key, void *value)
3009 {
3010   struct DFSContext *dfs_ctx = cls;
3011   GNUNET_HashCode first_hash;
3012
3013   if (dfs_ctx->current == dfs_ctx->chosen)
3014     {
3015       GNUNET_assert (GNUNET_OK ==
3016                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3017                                                         first->connect_peers_working_set,
3018                                                         key, value,
3019                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3020       uid_from_hash (key, &dfs_ctx->second_uid);
3021       hash_from_uid (dfs_ctx->first_uid, &first_hash);
3022       GNUNET_assert (GNUNET_OK ==
3023                      GNUNET_CONTAINER_multihashmap_put (dfs_ctx->
3024                                                         pg->peers
3025                                                         [dfs_ctx->second_uid].connect_peers_working_set,
3026                                                         &first_hash,
3027                                                         dfs_ctx->
3028                                                         first->daemon,
3029                                                         GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
3030       GNUNET_assert (GNUNET_YES ==
3031                      GNUNET_CONTAINER_multihashmap_remove (dfs_ctx->
3032                                                            pg->peers
3033                                                            [dfs_ctx->second_uid].connect_peers,
3034                                                            &first_hash,
3035                                                            dfs_ctx->
3036                                                            first->daemon));
3037       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
3038       return GNUNET_NO;         /* We have found our peer, don't iterate more */
3039     }
3040
3041   dfs_ctx->current++;
3042   return GNUNET_YES;
3043 }
3044
3045
3046 /**
3047  * From the set of connections possible, choose percentage percent of connections
3048  * to actually connect.
3049  *
3050  * @param pg the peergroup we are dealing with
3051  * @param percentage what percent of total connections to make
3052  */
3053 void
3054 choose_random_connections (struct GNUNET_TESTING_PeerGroup *pg,
3055                            double percentage)
3056 {
3057   struct RandomContext random_ctx;
3058   uint32_t pg_iter;
3059
3060   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3061     {
3062       random_ctx.first_uid = pg_iter;
3063       random_ctx.first = &pg->peers[pg_iter];
3064       random_ctx.percentage = percentage;
3065       random_ctx.pg = pg;
3066       pg->peers[pg_iter].connect_peers_working_set =
3067         GNUNET_CONTAINER_multihashmap_create (pg->total);
3068       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3069                                              &random_connect_iterator,
3070                                              &random_ctx);
3071       /* Now remove the old connections */
3072       GNUNET_CONTAINER_multihashmap_destroy (pg->
3073                                              peers[pg_iter].connect_peers);
3074       /* And replace with the random set */
3075       pg->peers[pg_iter].connect_peers =
3076         pg->peers[pg_iter].connect_peers_working_set;
3077     }
3078 }
3079
3080 /**
3081  * From the set of connections possible, choose at least num connections per
3082  * peer.
3083  *
3084  * @param pg the peergroup we are dealing with
3085  * @param num how many connections at least should each peer have (if possible)?
3086  */
3087 static void
3088 choose_minimum (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3089 {
3090   struct MinimumContext minimum_ctx;
3091   uint32_t pg_iter;
3092
3093   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3094     {
3095       pg->peers[pg_iter].connect_peers_working_set =
3096         GNUNET_CONTAINER_multihashmap_create (num);
3097     }
3098
3099   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3100     {
3101       minimum_ctx.first_uid = pg_iter;
3102       minimum_ctx.pg_array =
3103         GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK,
3104                                       GNUNET_CONTAINER_multihashmap_size
3105                                       (pg->peers[pg_iter].connect_peers));
3106       minimum_ctx.first = &pg->peers[pg_iter];
3107       minimum_ctx.pg = pg;
3108       minimum_ctx.num_to_add = num;
3109       minimum_ctx.current = 0;
3110       GNUNET_CONTAINER_multihashmap_iterate (pg->peers[pg_iter].connect_peers,
3111                                              &minimum_connect_iterator,
3112                                              &minimum_ctx);
3113     }
3114
3115   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3116     {
3117       /* Remove the "old" connections */
3118       GNUNET_CONTAINER_multihashmap_destroy (pg->
3119                                              peers[pg_iter].connect_peers);
3120       /* And replace with the working set */
3121       pg->peers[pg_iter].connect_peers =
3122         pg->peers[pg_iter].connect_peers_working_set;
3123     }
3124
3125 }
3126
3127
3128 static unsigned int
3129 count_workingset_connections (struct GNUNET_TESTING_PeerGroup *pg)
3130 {
3131   unsigned int count;
3132   unsigned int pg_iter;
3133
3134   count = 0;
3135
3136   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3137     {
3138       count +=
3139         GNUNET_CONTAINER_multihashmap_size (pg->
3140                                             peers
3141                                             [pg_iter].connect_peers_working_set);
3142     }
3143
3144   return count;
3145 }
3146
3147
3148 static unsigned int
3149 count_allowed_connections (struct GNUNET_TESTING_PeerGroup *pg)
3150 {
3151   unsigned int count;
3152   unsigned int pg_iter;
3153
3154   count = 0;
3155
3156   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3157     {
3158       count +=
3159         GNUNET_CONTAINER_multihashmap_size (pg->peers[pg_iter].connect_peers);
3160     }
3161
3162   return count;
3163 }
3164
3165
3166 struct FindClosestContext
3167 {
3168   /**
3169    * The currently known closest peer.
3170    */
3171   struct GNUNET_TESTING_Daemon *closest;
3172
3173   /**
3174    * The info for the peer we are adding connections for.
3175    */
3176   struct PeerData *curr_peer;
3177
3178   /**
3179    * The distance (bits) between the current
3180    * peer and the currently known closest.
3181    */
3182   unsigned int closest_dist;
3183
3184   /**
3185    * The offset of the closest known peer in
3186    * the peer group.
3187    */
3188   unsigned int closest_num;
3189 };
3190
3191 /**
3192  * Iterator over hash map entries of the allowed
3193  * peer connections.  Find the closest, not already
3194  * connected peer and return it.
3195  *
3196  * @param cls closure (struct FindClosestContext)
3197  * @param key current key code (hash of offset in pg)
3198  * @param value value in the hash map - a GNUNET_TESTING_Daemon
3199  * @return GNUNET_YES if we should continue to
3200  *         iterate,
3201  *         GNUNET_NO if not.
3202  */
3203 static int
3204 find_closest_peers (void *cls, const GNUNET_HashCode * key, void *value)
3205 {
3206   struct FindClosestContext *closest_ctx = cls;
3207   struct GNUNET_TESTING_Daemon *daemon = value;
3208
3209   if (((closest_ctx->closest == NULL) ||
3210        (GNUNET_CRYPTO_hash_matching_bits
3211         (&daemon->id.hashPubKey,
3212          &closest_ctx->curr_peer->daemon->id.hashPubKey) >
3213         closest_ctx->closest_dist))
3214       && (GNUNET_YES !=
3215           GNUNET_CONTAINER_multihashmap_contains (closest_ctx->
3216                                                   curr_peer->connect_peers,
3217                                                   key)))
3218     {
3219       closest_ctx->closest_dist =
3220         GNUNET_CRYPTO_hash_matching_bits (&daemon->id.hashPubKey,
3221                                           &closest_ctx->curr_peer->daemon->
3222                                           id.hashPubKey);
3223       closest_ctx->closest = daemon;
3224       uid_from_hash (key, &closest_ctx->closest_num);
3225     }
3226   return GNUNET_YES;
3227 }
3228
3229 /**
3230  * From the set of connections possible, choose at num connections per
3231  * peer based on depth which are closest out of those allowed.  Guaranteed
3232  * to add num peers to connect to, provided there are that many peers
3233  * in the underlay topology to connect to.
3234  *
3235  * @param pg the peergroup we are dealing with
3236  * @param num how many connections at least should each peer have (if possible)?
3237  * @param proc processor to actually add the connections
3238  */
3239 void
3240 add_closest (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num,
3241              GNUNET_TESTING_ConnectionProcessor proc)
3242 {
3243   struct FindClosestContext closest_ctx;
3244   uint32_t pg_iter;
3245   uint32_t i;
3246
3247   for (i = 0; i < num; i++)     /* Each time find a closest peer (from those available) */
3248     {
3249       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3250         {
3251           closest_ctx.curr_peer = &pg->peers[pg_iter];
3252           closest_ctx.closest = NULL;
3253           closest_ctx.closest_dist = 0;
3254           closest_ctx.closest_num = 0;
3255           GNUNET_CONTAINER_multihashmap_iterate (pg->
3256                                                  peers[pg_iter].allowed_peers,
3257                                                  &find_closest_peers,
3258                                                  &closest_ctx);
3259           if (closest_ctx.closest != NULL)
3260             {
3261               GNUNET_assert (closest_ctx.closest_num < pg->total);
3262               proc (pg, pg_iter, closest_ctx.closest_num);
3263             }
3264         }
3265     }
3266 }
3267
3268 /**
3269  * From the set of connections possible, choose at least num connections per
3270  * peer based on depth first traversal of peer connections.  If DFS leaves
3271  * peers unconnected, ensure those peers get connections.
3272  *
3273  * @param pg the peergroup we are dealing with
3274  * @param num how many connections at least should each peer have (if possible)?
3275  */
3276 void
3277 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
3278 {
3279   struct DFSContext dfs_ctx;
3280   uint32_t pg_iter;
3281   uint32_t dfs_count;
3282   uint32_t starting_peer;
3283   uint32_t least_connections;
3284   GNUNET_HashCode second_hash;
3285
3286   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3287     {
3288       pg->peers[pg_iter].connect_peers_working_set =
3289         GNUNET_CONTAINER_multihashmap_create (num);
3290     }
3291
3292   starting_peer = 0;
3293   dfs_count = 0;
3294   while ((count_workingset_connections (pg) < num * pg->total)
3295          && (count_allowed_connections (pg) > 0))
3296     {
3297       if (dfs_count % pg->total == 0)   /* Restart the DFS at some weakly connected peer */
3298         {
3299           least_connections = -1;       /* Set to very high number */
3300           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3301             {
3302               if (GNUNET_CONTAINER_multihashmap_size
3303                   (pg->peers[pg_iter].connect_peers_working_set) <
3304                   least_connections)
3305                 {
3306                   starting_peer = pg_iter;
3307                   least_connections =
3308                     GNUNET_CONTAINER_multihashmap_size (pg->
3309                                                         peers
3310                                                         [pg_iter].connect_peers_working_set);
3311                 }
3312             }
3313         }
3314
3315       if (GNUNET_CONTAINER_multihashmap_size (pg->peers[starting_peer].connect_peers) == 0)     /* Ensure there is at least one peer left to connect! */
3316         {
3317           dfs_count = 0;
3318           continue;
3319         }
3320
3321       /* Choose a random peer from the chosen peers set of connections to add */
3322       dfs_ctx.chosen =
3323         GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
3324                                   GNUNET_CONTAINER_multihashmap_size
3325                                   (pg->peers[starting_peer].connect_peers));
3326       dfs_ctx.first_uid = starting_peer;
3327       dfs_ctx.first = &pg->peers[starting_peer];
3328       dfs_ctx.pg = pg;
3329       dfs_ctx.current = 0;
3330
3331       GNUNET_CONTAINER_multihashmap_iterate (pg->
3332                                              peers
3333                                              [starting_peer].connect_peers,
3334                                              &dfs_connect_iterator, &dfs_ctx);
3335       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
3336       hash_from_uid (dfs_ctx.second_uid, &second_hash);
3337       GNUNET_assert (GNUNET_YES ==
3338                      GNUNET_CONTAINER_multihashmap_remove (pg->peers
3339                                                            [starting_peer].connect_peers,
3340                                                            &second_hash,
3341                                                            pg->
3342                                                            peers
3343                                                            [dfs_ctx.second_uid].daemon));
3344       starting_peer = dfs_ctx.second_uid;
3345     }
3346
3347   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
3348     {
3349       /* Remove the "old" connections */
3350       GNUNET_CONTAINER_multihashmap_destroy (pg->
3351                                              peers[pg_iter].connect_peers);
3352       /* And replace with the working set */
3353       pg->peers[pg_iter].connect_peers =
3354         pg->peers[pg_iter].connect_peers_working_set;
3355     }
3356 }
3357
3358 /**
3359  * Internal callback for topology information for a particular peer.
3360  */
3361 static void
3362 internal_topology_callback (void *cls,
3363                             const struct GNUNET_PeerIdentity *peer,
3364                             const struct GNUNET_TRANSPORT_ATS_Information
3365                             *atsi)
3366 {
3367   struct CoreContext *core_ctx = cls;
3368   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
3369
3370   if (peer == NULL)             /* Either finished, or something went wrong */
3371     {
3372       iter_ctx->completed++;
3373       iter_ctx->connected--;
3374       /* One core context allocated per iteration, must free! */
3375       GNUNET_free (core_ctx);
3376     }
3377   else
3378     {
3379       iter_ctx->topology_cb (iter_ctx->cls, &core_ctx->daemon->id,
3380                              peer, NULL);
3381     }
3382
3383   if (iter_ctx->completed == iter_ctx->total)
3384     {
3385       iter_ctx->topology_cb (iter_ctx->cls, NULL, NULL, NULL);
3386       /* Once all are done, free the iteration context */
3387       GNUNET_free (iter_ctx);
3388     }
3389 }
3390
3391
3392 /**
3393  * Check running topology iteration tasks, if below max start a new one, otherwise
3394  * schedule for some time in the future.
3395  */
3396 static void
3397 schedule_get_topology (void *cls,
3398                        const struct GNUNET_SCHEDULER_TaskContext *tc)
3399 {
3400   struct CoreContext *core_context = cls;
3401   struct TopologyIterateContext *topology_context =
3402     (struct TopologyIterateContext *) core_context->iter_context;
3403   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3404     return;
3405
3406   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3407     {
3408 #if VERBOSE_TESTING > 2
3409       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3410                   _
3411                   ("Delaying connect, we have too many outstanding connections!\n"));
3412 #endif
3413       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3414                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3415                                     &schedule_get_topology, core_context);
3416     }
3417   else
3418     {
3419 #if VERBOSE_TESTING > 2
3420       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3421                   _("Creating connection, outstanding_connections is %d\n"),
3422                   outstanding_connects);
3423 #endif
3424       topology_context->connected++;
3425
3426       if (GNUNET_OK !=
3427           GNUNET_CORE_iterate_peers (core_context->daemon->cfg,
3428                                      &internal_topology_callback,
3429                                      core_context))
3430         {
3431           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Topology iteration failed.\n");
3432           internal_topology_callback (core_context, NULL, NULL);
3433         }
3434     }
3435 }
3436
3437 /**
3438  * Iterate over all (running) peers in the peer group, retrieve
3439  * all connections that each currently has.
3440  */
3441 void
3442 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg,
3443                              GNUNET_TESTING_NotifyTopology cb, void *cls)
3444 {
3445   struct TopologyIterateContext *topology_context;
3446   struct CoreContext *core_ctx;
3447   unsigned int i;
3448   unsigned int total_count;
3449
3450   /* Allocate a single topology iteration context */
3451   topology_context = GNUNET_malloc (sizeof (struct TopologyIterateContext));
3452   topology_context->topology_cb = cb;
3453   topology_context->cls = cls;
3454   total_count = 0;
3455   for (i = 0; i < pg->total; i++)
3456     {
3457       if (pg->peers[i].daemon->running == GNUNET_YES)
3458         {
3459           /* Allocate one core context per core we need to connect to */
3460           core_ctx = GNUNET_malloc (sizeof (struct CoreContext));
3461           core_ctx->daemon = pg->peers[i].daemon;
3462           /* Set back pointer to topology iteration context */
3463           core_ctx->iter_context = topology_context;
3464           GNUNET_SCHEDULER_add_now (&schedule_get_topology, core_ctx);
3465           total_count++;
3466         }
3467     }
3468   if (total_count == 0)
3469     {
3470       cb (cls, NULL, NULL, "Cannot iterate over topology, no running peers!");
3471       GNUNET_free (topology_context);
3472     }
3473   else
3474     topology_context->total = total_count;
3475   return;
3476 }
3477
3478 /**
3479  * Callback function to process statistic values.
3480  * This handler is here only really to insert a peer
3481  * identity (or daemon) so the statistics can be uniquely
3482  * tied to a single running peer.
3483  *
3484  * @param cls closure
3485  * @param subsystem name of subsystem that created the statistic
3486  * @param name the name of the datum
3487  * @param value the current value
3488  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
3489  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
3490  */
3491 static int
3492 internal_stats_callback (void *cls,
3493                          const char *subsystem,
3494                          const char *name, uint64_t value, int is_persistent)
3495 {
3496   struct StatsCoreContext *core_context = cls;
3497   struct StatsIterateContext *stats_context =
3498     (struct StatsIterateContext *) core_context->iter_context;
3499
3500   return stats_context->proc (stats_context->cls, &core_context->daemon->id,
3501                               subsystem, name, value, is_persistent);
3502 }
3503
3504 /**
3505  * Internal continuation call for statistics iteration.
3506  *
3507  * @param cls closure, the CoreContext for this iteration
3508  * @param success whether or not the statistics iterations
3509  *        was canceled or not (we don't care)
3510  */
3511 static void
3512 internal_stats_cont (void *cls, int success)
3513 {
3514   struct StatsCoreContext *core_context = cls;
3515   struct StatsIterateContext *stats_context =
3516     (struct StatsIterateContext *) core_context->iter_context;
3517
3518   stats_context->connected--;
3519   stats_context->completed++;
3520
3521   if (stats_context->completed == stats_context->total)
3522     {
3523       stats_context->cont (stats_context->cls, GNUNET_YES);
3524       GNUNET_free (stats_context);
3525     }
3526
3527   if (core_context->stats_handle != NULL)
3528     GNUNET_STATISTICS_destroy (core_context->stats_handle, GNUNET_NO);
3529
3530   GNUNET_free (core_context);
3531 }
3532
3533 /**
3534  * Check running topology iteration tasks, if below max start a new one, otherwise
3535  * schedule for some time in the future.
3536  */
3537 static void
3538 schedule_get_statistics (void *cls,
3539                          const struct GNUNET_SCHEDULER_TaskContext *tc)
3540 {
3541   struct StatsCoreContext *core_context = cls;
3542   struct StatsIterateContext *stats_context =
3543     (struct StatsIterateContext *) core_context->iter_context;
3544
3545   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3546     return;
3547
3548   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3549     {
3550 #if VERBOSE_TESTING > 2
3551       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3552                   _
3553                   ("Delaying connect, we have too many outstanding connections!\n"));
3554 #endif
3555       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3556                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3557                                     &schedule_get_statistics, core_context);
3558     }
3559   else
3560     {
3561 #if VERBOSE_TESTING > 2
3562       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3563                   _("Creating connection, outstanding_connections is %d\n"),
3564                   outstanding_connects);
3565 #endif
3566
3567       stats_context->connected++;
3568       core_context->stats_handle =
3569         GNUNET_STATISTICS_create ("testing", core_context->daemon->cfg);
3570       if (core_context->stats_handle == NULL)
3571         {
3572           internal_stats_cont (core_context, GNUNET_NO);
3573           return;
3574         }
3575
3576       core_context->stats_get_handle =
3577         GNUNET_STATISTICS_get (core_context->stats_handle, NULL, NULL,
3578                                GNUNET_TIME_relative_get_forever (),
3579                                &internal_stats_cont, &internal_stats_callback,
3580                                core_context);
3581       if (core_context->stats_get_handle == NULL)
3582         internal_stats_cont (core_context, GNUNET_NO);
3583
3584     }
3585 }
3586
3587 struct DuplicateStats
3588 {
3589   /**
3590    * Next item in the list
3591    */
3592   struct DuplicateStats *next;
3593
3594   /**
3595    * Nasty string, concatenation of relevant information.
3596    */
3597   char *unique_string;
3598 };
3599
3600 /**
3601  * Check whether the combination of port/host/unix domain socket
3602  * already exists in the list of peers being checked for statistics.
3603  *
3604  * @param pg the peergroup in question
3605  * @param specific_peer the peer we're concerned with
3606  * @param stats_list the list to return to the caller
3607  *
3608  * @return GNUNET_YES if the statistics instance has been seen already,
3609  *         GNUNET_NO if not (and we may have added it to the list)
3610  */
3611 static int
3612 stats_check_existing (struct GNUNET_TESTING_PeerGroup *pg,
3613                       struct PeerData *specific_peer,
3614                       struct DuplicateStats **stats_list)
3615 {
3616   struct DuplicateStats *pos;
3617   char *unix_domain_socket;
3618   unsigned long long port;
3619   char *to_match;
3620   if (GNUNET_YES !=
3621       GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "testing",
3622                                             "single_statistics_per_host"))
3623     return GNUNET_NO;           /* Each peer has its own statistics instance, do nothing! */
3624
3625   pos = *stats_list;
3626   if (GNUNET_OK !=
3627       GNUNET_CONFIGURATION_get_value_string (specific_peer->cfg, "statistics",
3628                                              "unixpath", &unix_domain_socket))
3629     return GNUNET_NO;
3630
3631   if (GNUNET_OK !=
3632       GNUNET_CONFIGURATION_get_value_number (specific_peer->cfg, "statistics",
3633                                              "port", &port))
3634     return GNUNET_NO;
3635
3636   if (specific_peer->daemon->hostname != NULL)
3637     GNUNET_asprintf (&to_match, "%s%s%llu", specific_peer->daemon->hostname,
3638                      unix_domain_socket, port);
3639   else
3640     GNUNET_asprintf (&to_match, "%s%llu", unix_domain_socket, port);
3641
3642   while (pos != NULL)
3643     {
3644       if (0 == strcmp (to_match, pos->unique_string))
3645         {
3646           GNUNET_free (unix_domain_socket);
3647           GNUNET_free (to_match);
3648           return GNUNET_YES;
3649         }
3650       pos = pos->next;
3651     }
3652   pos = GNUNET_malloc (sizeof (struct DuplicateStats));
3653   pos->unique_string = to_match;
3654   pos->next = *stats_list;
3655   *stats_list = pos;
3656   GNUNET_free (unix_domain_socket);
3657   return GNUNET_NO;
3658 }
3659
3660 /**
3661  * Iterate over all (running) peers in the peer group, retrieve
3662  * all statistics from each.
3663  */
3664 void
3665 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3666                                GNUNET_STATISTICS_Callback cont,
3667                                GNUNET_TESTING_STATISTICS_Iterator proc,
3668                                void *cls)
3669 {
3670   struct StatsIterateContext *stats_context;
3671   struct StatsCoreContext *core_ctx;
3672   unsigned int i;
3673   unsigned int total_count;
3674   struct DuplicateStats *stats_list;
3675   struct DuplicateStats *pos;
3676   stats_list = NULL;
3677
3678   /* Allocate a single stats iteration context */
3679   stats_context = GNUNET_malloc (sizeof (struct StatsIterateContext));
3680   stats_context->cont = cont;
3681   stats_context->proc = proc;
3682   stats_context->cls = cls;
3683   total_count = 0;
3684
3685   for (i = 0; i < pg->total; i++)
3686     {
3687       if ((pg->peers[i].daemon->running == GNUNET_YES)
3688           && (GNUNET_NO ==
3689               stats_check_existing (pg, &pg->peers[i], &stats_list)))
3690         {
3691           /* Allocate one core context per core we need to connect to */
3692           core_ctx = GNUNET_malloc (sizeof (struct StatsCoreContext));
3693           core_ctx->daemon = pg->peers[i].daemon;
3694           /* Set back pointer to topology iteration context */
3695           core_ctx->iter_context = stats_context;
3696           GNUNET_SCHEDULER_add_now (&schedule_get_statistics, core_ctx);
3697           total_count++;
3698         }
3699     }
3700
3701   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3702               "Retrieving stats from %u total instances.\n", total_count);
3703   stats_context->total = total_count;
3704   if (stats_list != NULL)
3705     {
3706       pos = stats_list;
3707       while (pos != NULL)
3708         {
3709           GNUNET_free (pos->unique_string);
3710           stats_list = pos->next;
3711           GNUNET_free (pos);
3712           pos = stats_list->next;
3713         }
3714     }
3715   return;
3716 }
3717
3718 /**
3719  * There are many ways to connect peers that are supported by this function.
3720  * To connect peers in the same topology that was created via the
3721  * GNUNET_TESTING_create_topology, the topology variable must be set to
3722  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3723  * a new instance of that topology will be generated and attempted to be
3724  * connected.  This could result in some connections being impossible,
3725  * because some topologies are non-deterministic.
3726  *
3727  * @param pg the peer group struct representing the running peers
3728  * @param topology which topology to connect the peers in
3729  * @param options options for connecting the topology
3730  * @param option_modifier modifier for options that take a parameter
3731  * @param notify_callback notification to be called once all connections completed
3732  * @param notify_cls closure for notification callback
3733  *
3734  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3735  */
3736 int
3737 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3738                                  enum GNUNET_TESTING_Topology topology,
3739                                  enum GNUNET_TESTING_TopologyOption options,
3740                                  double option_modifier,
3741                                  GNUNET_TESTING_NotifyCompletion
3742                                  notify_callback, void *notify_cls)
3743 {
3744   switch (topology)
3745     {
3746     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3747 #if VERBOSE_TOPOLOGY
3748       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3749                   _("Creating clique CONNECT topology\n"));
3750 #endif
3751       create_clique (pg, &add_actual_connections);
3752       break;
3753     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3754 #if VERBOSE_TOPOLOGY
3755       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3756                   _("Creating small world (ring) CONNECT topology\n"));
3757 #endif
3758       create_small_world_ring (pg, &add_actual_connections);
3759       break;
3760     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3761 #if VERBOSE_TOPOLOGY
3762       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3763                   _("Creating small world (2d-torus) CONNECT topology\n"));
3764 #endif
3765       create_small_world (pg, &add_actual_connections);
3766       break;
3767     case GNUNET_TESTING_TOPOLOGY_RING:
3768 #if VERBOSE_TOPOLOGY
3769       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3770                   _("Creating ring CONNECT topology\n"));
3771 #endif
3772       create_ring (pg, &add_actual_connections);
3773       break;
3774     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3775 #if VERBOSE_TOPOLOGY
3776       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3777                   _("Creating 2d torus CONNECT topology\n"));
3778 #endif
3779       create_2d_torus (pg, &add_actual_connections);
3780       break;
3781     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3782 #if VERBOSE_TOPOLOGY
3783       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3784                   _("Creating Erdos-Renyi CONNECT topology\n"));
3785 #endif
3786       create_erdos_renyi (pg, &add_actual_connections);
3787       break;
3788     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3789 #if VERBOSE_TOPOLOGY
3790       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3791                   _("Creating InterNAT CONNECT topology\n"));
3792 #endif
3793       create_nated_internet (pg, &add_actual_connections);
3794       break;
3795     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3796 #if VERBOSE_TOPOLOGY
3797       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3798                   _("Creating Scale Free CONNECT topology\n"));
3799 #endif
3800       create_scale_free (pg, &add_actual_connections);
3801       break;
3802     case GNUNET_TESTING_TOPOLOGY_LINE:
3803 #if VERBOSE_TOPOLOGY
3804       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3805                   _("Creating straight line CONNECT topology\n"));
3806 #endif
3807       create_line (pg, &add_actual_connections);
3808       break;
3809     case GNUNET_TESTING_TOPOLOGY_NONE:
3810 #if VERBOSE_TOPOLOGY
3811       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3812                   _("Creating no CONNECT topology\n"));
3813 #endif
3814       copy_allowed_topology (pg);
3815       break;
3816     default:
3817       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3818                   _
3819                   ("Unknown topology specification, can't connect peers!\n"));
3820       return GNUNET_SYSERR;
3821     }
3822
3823   switch (options)
3824     {
3825     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3826 #if VERBOSE_TOPOLOGY
3827       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3828                   _
3829                   ("Connecting random subset (%'.2f percent) of possible peers\n"),
3830                   100 * option_modifier);
3831 #endif
3832       choose_random_connections (pg, option_modifier);
3833       break;
3834     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3835 #if VERBOSE_TOPOLOGY
3836       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3837                   _("Connecting a minimum of %u peers each (if possible)\n"),
3838                   (unsigned int) option_modifier);
3839 #endif
3840       choose_minimum (pg, (unsigned int) option_modifier);
3841       break;
3842     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3843 #if VERBOSE_TOPOLOGY
3844       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3845                   _
3846                   ("Using DFS to connect a minimum of %u peers each (if possible)\n"),
3847                   (unsigned int) option_modifier);
3848 #endif
3849       perform_dfs (pg, (int) option_modifier);
3850       break;
3851     case GNUNET_TESTING_TOPOLOGY_OPTION_ADD_CLOSEST:
3852 #if VERBOSE_TOPOLOGY
3853       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3854                   _
3855                   ("Finding additional %u closest peers each (if possible)\n"),
3856                   (unsigned int) option_modifier);
3857 #endif
3858       add_closest (pg, (unsigned int) option_modifier,
3859                    &add_actual_connections);
3860       break;
3861     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3862       break;
3863     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3864       break;
3865     default:
3866       break;
3867     }
3868
3869   return connect_topology (pg, notify_callback, notify_cls);
3870 }
3871
3872 /**
3873  * Callback that is called whenever a hostkey is generated
3874  * for a peer.  Call the real callback and decrement the
3875  * starting counter for the peergroup.
3876  *
3877  * @param cls closure
3878  * @param id identifier for the daemon, NULL on error
3879  * @param d handle for the daemon
3880  * @param emsg error message (NULL on success)
3881  */
3882 static void
3883 internal_hostkey_callback (void *cls,
3884                            const struct GNUNET_PeerIdentity *id,
3885                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3886 {
3887   struct InternalStartContext *internal_context = cls;
3888   internal_context->peer->pg->starting--;
3889   internal_context->peer->pg->started++;
3890   if (internal_context->hostkey_callback != NULL)
3891     internal_context->hostkey_callback (internal_context->hostkey_cls, id, d,
3892                                         emsg);
3893   else if (internal_context->peer->pg->started ==
3894            internal_context->peer->pg->total)
3895     {
3896       internal_context->peer->pg->started = 0;  /* Internal startup may use this counter! */
3897       GNUNET_TESTING_daemons_continue_startup (internal_context->peer->pg);
3898     }
3899 }
3900
3901 /**
3902  * Callback that is called whenever a peer has finished starting.
3903  * Call the real callback and decrement the starting counter
3904  * for the peergroup.
3905  *
3906  * @param cls closure
3907  * @param id identifier for the daemon, NULL on error
3908  * @param d handle for the daemon
3909  * @param emsg error message (NULL on success)
3910  */
3911 static void
3912 internal_startup_callback (void *cls,
3913                            const struct GNUNET_PeerIdentity *id,
3914                            const struct GNUNET_CONFIGURATION_Handle *cfg,
3915                            struct GNUNET_TESTING_Daemon *d, const char *emsg)
3916 {
3917   struct InternalStartContext *internal_context = cls;
3918   internal_context->peer->pg->starting--;
3919   if (internal_context->start_cb != NULL)
3920     internal_context->start_cb (internal_context->start_cb_cls, id, cfg, d,
3921                                 emsg);
3922 }
3923
3924 static void
3925 internal_continue_startup (void *cls,
3926                            const struct GNUNET_SCHEDULER_TaskContext *tc)
3927 {
3928   struct InternalStartContext *internal_context = cls;
3929
3930   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3931     {
3932       return;
3933     }
3934
3935   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3936     {
3937       internal_context->peer->pg->starting++;
3938       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3939     }
3940   else
3941     {
3942       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
3943                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
3944                                     &internal_continue_startup,
3945                                     internal_context);
3946     }
3947 }
3948
3949
3950 /**
3951  * Callback for informing us about a successful
3952  * or unsuccessful churn start call.
3953  *
3954  * @param cls a ChurnContext
3955  * @param id the peer identity of the started peer
3956  * @param cfg the handle to the configuration of the peer
3957  * @param d handle to the daemon for the peer
3958  * @param emsg NULL on success, non-NULL on failure
3959  *
3960  */
3961 void
3962 churn_start_callback (void *cls,
3963                       const struct GNUNET_PeerIdentity *id,
3964                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3965                       struct GNUNET_TESTING_Daemon *d, const char *emsg)
3966 {
3967   struct ChurnRestartContext *startup_ctx = cls;
3968   struct ChurnContext *churn_ctx = startup_ctx->churn_ctx;
3969
3970   unsigned int total_left;
3971   char *error_message;
3972
3973   error_message = NULL;
3974   if (emsg != NULL)
3975     {
3976       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3977                   "Churn stop callback failed with error `%s'\n", emsg);
3978       churn_ctx->num_failed_start++;
3979     }
3980   else
3981     {
3982       churn_ctx->num_to_start--;
3983     }
3984
3985 #if DEBUG_CHURN
3986   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3987               "Started peer, %d left.\n", churn_ctx->num_to_start);
3988 #endif
3989
3990   total_left =
3991     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
3992     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3993
3994   if (total_left == 0)
3995     {
3996       if ((churn_ctx->num_failed_stop > 0)
3997           || (churn_ctx->num_failed_start > 0))
3998         GNUNET_asprintf (&error_message,
3999                          "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4000                          churn_ctx->num_failed_start,
4001                          churn_ctx->num_failed_stop);
4002       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4003       GNUNET_free_non_null (error_message);
4004       GNUNET_free (churn_ctx);
4005       GNUNET_free (startup_ctx);
4006     }
4007 }
4008
4009
4010 static void
4011 schedule_churn_restart (void *cls,
4012                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4013 {
4014   struct PeerRestartContext *peer_restart_ctx = cls;
4015   struct ChurnRestartContext *startup_ctx =
4016     peer_restart_ctx->churn_restart_ctx;
4017
4018   if (startup_ctx->outstanding > MAX_CONCURRENT_STARTING)
4019     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4020                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4021                                   &schedule_churn_restart, peer_restart_ctx);
4022   else
4023     {
4024       GNUNET_TESTING_daemon_start_stopped (peer_restart_ctx->daemon,
4025                                            startup_ctx->timeout,
4026                                            &churn_start_callback,
4027                                            startup_ctx);
4028       GNUNET_free (peer_restart_ctx);
4029     }
4030 }
4031
4032 static void
4033 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
4034 {
4035   struct InternalStartContext *internal_context = cls;
4036
4037   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
4038     {
4039       return;
4040     }
4041
4042   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
4043     {
4044       internal_context->peer->pg->starting++;
4045       internal_context->peer->daemon =
4046         GNUNET_TESTING_daemon_start (internal_context->peer->cfg,
4047                                      internal_context->timeout,
4048                                      internal_context->hostname,
4049                                      internal_context->username,
4050                                      internal_context->sshport,
4051                                      &internal_hostkey_callback,
4052                                      internal_context,
4053                                      &internal_startup_callback,
4054                                      internal_context);
4055     }
4056   else
4057     {
4058       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4059                                     (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4060                                     &internal_start, internal_context);
4061     }
4062 }
4063
4064 /**
4065  * Function which continues a peer group starting up
4066  * after successfully generating hostkeys for each peer.
4067  *
4068  * @param pg the peer group to continue starting
4069  *
4070  */
4071 void
4072 GNUNET_TESTING_daemons_continue_startup (struct GNUNET_TESTING_PeerGroup *pg)
4073 {
4074   unsigned int i;
4075
4076   pg->starting = 0;
4077   for (i = 0; i < pg->total; i++)
4078     {
4079       GNUNET_SCHEDULER_add_now (&internal_continue_startup,
4080                                 &pg->peers[i].internal_context);
4081       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
4082     }
4083 }
4084
4085 /**
4086  * Start count gnunet instances with the same set of transports and
4087  * applications.  The port numbers (any option called "PORT") will be
4088  * adjusted to ensure that no two peers running on the same system
4089  * have the same port(s) in their respective configurations.
4090  *
4091  * @param cfg configuration template to use
4092  * @param total number of daemons to start
4093  * @param timeout total time allowed for peers to start
4094  * @param hostkey_callback function to call on each peers hostkey generation
4095  *        if NULL, peers will be started by this call, if non-null,
4096  *        GNUNET_TESTING_daemons_continue_startup must be called after
4097  *        successful hostkey generation
4098  * @param hostkey_cls closure for hostkey callback
4099  * @param cb function to call on each daemon that was started
4100  * @param cb_cls closure for cb
4101  * @param connect_callback function to call each time two hosts are connected
4102  * @param connect_callback_cls closure for connect_callback
4103  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
4104  *
4105  * @return NULL on error, otherwise handle to control peer group
4106  */
4107 struct GNUNET_TESTING_PeerGroup *
4108 GNUNET_TESTING_daemons_start (const struct GNUNET_CONFIGURATION_Handle *cfg,
4109                               unsigned int total,
4110                               struct GNUNET_TIME_Relative timeout,
4111                               GNUNET_TESTING_NotifyHostkeyCreated
4112                               hostkey_callback, void *hostkey_cls,
4113                               GNUNET_TESTING_NotifyDaemonRunning cb,
4114                               void *cb_cls,
4115                               GNUNET_TESTING_NotifyConnection
4116                               connect_callback, void *connect_callback_cls,
4117                               const struct GNUNET_TESTING_Host *hostnames)
4118 {
4119   struct GNUNET_TESTING_PeerGroup *pg;
4120   const struct GNUNET_TESTING_Host *hostpos;
4121 #if 0
4122   char *pos;
4123   const char *rpos;
4124   char *start;
4125 #endif
4126   const char *hostname;
4127   const char *username;
4128   char *baseservicehome;
4129   char *newservicehome;
4130   char *tmpdir;
4131   struct GNUNET_CONFIGURATION_Handle *pcfg;
4132   unsigned int off;
4133   unsigned int hostcnt;
4134   uint16_t minport;
4135   uint16_t sshport;
4136   uint32_t upnum;
4137   uint32_t fdnum;
4138
4139   if (0 == total)
4140     {
4141       GNUNET_break (0);
4142       return NULL;
4143     }
4144   upnum = 0;
4145   fdnum = 0;
4146   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
4147   pg->cfg = cfg;
4148   pg->notify_connection = connect_callback;
4149   pg->notify_connection_cls = connect_callback_cls;
4150   pg->total = total;
4151   pg->max_timeout = GNUNET_TIME_relative_to_absolute (timeout);
4152   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
4153   if (NULL != hostnames)
4154     {
4155       off = 0;
4156       hostpos = hostnames;
4157       while (hostpos != NULL)
4158         {
4159           hostpos = hostpos->next;
4160           off++;
4161         }
4162       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4163       off = 0;
4164
4165       hostpos = hostnames;
4166       while (hostpos != NULL)
4167         {
4168           pg->hosts[off].minport = LOW_PORT;
4169           pg->hosts[off].hostname = GNUNET_strdup (hostpos->hostname);
4170           if (hostpos->username != NULL)
4171             pg->hosts[off].username = GNUNET_strdup (hostpos->username);
4172           pg->hosts[off].sshport = hostpos->port;
4173           hostpos = hostpos->next;
4174           off++;
4175         }
4176
4177       if (off == 0)
4178         {
4179           pg->hosts = NULL;
4180         }
4181       hostcnt = off;
4182       minport = 0;
4183       pg->num_hosts = off;
4184
4185 #if NO_LL
4186       off = 2;
4187       /* skip leading spaces */
4188       while ((0 != *hostnames) && (isspace ((unsigned char) *hostnames)))
4189         hostnames++;
4190       rpos = hostnames;
4191       while ('\0' != *rpos)
4192         {
4193           if (isspace ((unsigned char) *rpos))
4194             off++;
4195           rpos++;
4196         }
4197       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
4198       off = 0;
4199       start = GNUNET_strdup (hostnames);
4200       pos = start;
4201       while ('\0' != *pos)
4202         {
4203           if (isspace ((unsigned char) *pos))
4204             {
4205               *pos = '\0';
4206               if (strlen (start) > 0)
4207                 {
4208                   pg->hosts[off].minport = LOW_PORT;
4209                   pg->hosts[off++].hostname = start;
4210                 }
4211               start = pos + 1;
4212             }
4213           pos++;
4214         }
4215       if (strlen (start) > 0)
4216         {
4217           pg->hosts[off].minport = LOW_PORT;
4218           pg->hosts[off++].hostname = start;
4219         }
4220       if (off == 0)
4221         {
4222           GNUNET_free (start);
4223           GNUNET_free (pg->hosts);
4224           pg->hosts = NULL;
4225         }
4226       hostcnt = off;
4227       minport = 0;              /* make gcc happy */
4228 #endif
4229     }
4230   else
4231     {
4232       hostcnt = 0;
4233       minport = LOW_PORT;
4234     }
4235   for (off = 0; off < total; off++)
4236     {
4237       if (hostcnt > 0)
4238         {
4239           hostname = pg->hosts[off % hostcnt].hostname;
4240           username = pg->hosts[off % hostcnt].username;
4241           sshport = pg->hosts[off % hostcnt].sshport;
4242           pcfg = make_config (cfg,
4243                               &pg->hosts[off % hostcnt].minport,
4244                               &upnum, hostname, &fdnum);
4245         }
4246       else
4247         {
4248           hostname = NULL;
4249           username = NULL;
4250           sshport = 0;
4251           pcfg = make_config (cfg, &minport, &upnum, hostname, &fdnum);
4252         }
4253
4254       if (NULL == pcfg)
4255         {
4256           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4257                       _
4258                       ("Could not create configuration for peer number %u on `%s'!\n"),
4259                       off, hostname == NULL ? "localhost" : hostname);
4260           continue;
4261         }
4262
4263       if (GNUNET_YES ==
4264           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
4265                                                  &baseservicehome))
4266         {
4267           GNUNET_asprintf (&newservicehome, "%s/%d/", baseservicehome, off);
4268           GNUNET_free (baseservicehome);
4269         }
4270       else
4271         {
4272           tmpdir = getenv ("TMPDIR");
4273           tmpdir = tmpdir ? tmpdir : "/tmp";
4274           GNUNET_asprintf (&newservicehome,
4275                            "%s/%s/%d/",
4276                            tmpdir, "gnunet-testing-test-test", off);
4277         }
4278       GNUNET_CONFIGURATION_set_value_string (pcfg,
4279                                              "PATHS",
4280                                              "SERVICEHOME", newservicehome);
4281       GNUNET_free (newservicehome);
4282       pg->peers[off].cfg = pcfg;
4283       pg->peers[off].allowed_peers =
4284         GNUNET_CONTAINER_multihashmap_create (total);
4285       pg->peers[off].connect_peers =
4286         GNUNET_CONTAINER_multihashmap_create (total);
4287       pg->peers[off].blacklisted_peers =
4288         GNUNET_CONTAINER_multihashmap_create (total);
4289       pg->peers[off].pg = pg;
4290
4291       pg->peers[off].internal_context.peer = &pg->peers[off];
4292       pg->peers[off].internal_context.timeout = timeout;
4293       pg->peers[off].internal_context.hostname = hostname;
4294       pg->peers[off].internal_context.username = username;
4295       pg->peers[off].internal_context.sshport = sshport;
4296       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
4297       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
4298       pg->peers[off].internal_context.start_cb = cb;
4299       pg->peers[off].internal_context.start_cb_cls = cb_cls;
4300
4301       GNUNET_SCHEDULER_add_now (&internal_start,
4302                                 &pg->peers[off].internal_context);
4303
4304     }
4305   return pg;
4306 }
4307
4308 /*
4309  * Get a daemon by number, so callers don't have to do nasty
4310  * offsetting operation.
4311  */
4312 struct GNUNET_TESTING_Daemon *
4313 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg,
4314                            unsigned int position)
4315 {
4316   if (position < pg->total)
4317     return pg->peers[position].daemon;
4318   else
4319     return NULL;
4320 }
4321
4322 /*
4323  * Get a daemon by peer identity, so callers can
4324  * retrieve the daemon without knowing it's offset.
4325  *
4326  * @param pg the peer group to retrieve the daemon from
4327  * @param peer_id the peer identity of the daemon to retrieve
4328  *
4329  * @return the daemon on success, or NULL if no such peer identity is found
4330  */
4331 struct GNUNET_TESTING_Daemon *
4332 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg,
4333                                  struct GNUNET_PeerIdentity *peer_id)
4334 {
4335   unsigned int i;
4336
4337   for (i = 0; i < pg->total; i++)
4338     {
4339       if (0 ==
4340           memcmp (&pg->peers[i].daemon->id, peer_id,
4341                   sizeof (struct GNUNET_PeerIdentity)))
4342         return pg->peers[i].daemon;
4343     }
4344
4345   return NULL;
4346 }
4347
4348 /**
4349  * Prototype of a function that will be called when a
4350  * particular operation was completed the testing library.
4351  *
4352  * @param cls closure (a struct RestartContext)
4353  * @param id id of the peer that was restarted
4354  * @param cfg handle to the configuration of the peer
4355  * @param d handle to the daemon that was restarted
4356  * @param emsg NULL on success
4357  */
4358 void
4359 restart_callback (void *cls,
4360                   const struct GNUNET_PeerIdentity *id,
4361                   const struct GNUNET_CONFIGURATION_Handle *cfg,
4362                   struct GNUNET_TESTING_Daemon *d, const char *emsg)
4363 {
4364   struct RestartContext *restart_context = cls;
4365
4366   if (emsg == NULL)
4367     {
4368       restart_context->peers_restarted++;
4369     }
4370   else
4371     {
4372       restart_context->peers_restart_failed++;
4373     }
4374
4375   if (restart_context->peers_restarted == restart_context->peer_group->total)
4376     {
4377       restart_context->callback (restart_context->callback_cls, NULL);
4378       GNUNET_free (restart_context);
4379     }
4380   else if (restart_context->peers_restart_failed +
4381            restart_context->peers_restarted ==
4382            restart_context->peer_group->total)
4383     {
4384       restart_context->callback (restart_context->callback_cls,
4385                                  "Failed to restart peers!");
4386       GNUNET_free (restart_context);
4387     }
4388
4389 }
4390
4391 /**
4392  * Callback for informing us about a successful
4393  * or unsuccessful churn stop call.
4394  *
4395  * @param cls a ChurnContext
4396  * @param emsg NULL on success, non-NULL on failure
4397  *
4398  */
4399 void
4400 churn_stop_callback (void *cls, const char *emsg)
4401 {
4402   struct ShutdownContext *shutdown_ctx = cls;
4403   struct ChurnContext *churn_ctx = shutdown_ctx->cb_cls;
4404   unsigned int total_left;
4405   char *error_message;
4406
4407   error_message = NULL;
4408   shutdown_ctx->outstanding--;
4409
4410   if (emsg != NULL)
4411     {
4412       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4413                   "Churn stop callback failed with error `%s'\n", emsg);
4414       churn_ctx->num_failed_stop++;
4415     }
4416   else
4417     {
4418       churn_ctx->num_to_stop--;
4419     }
4420
4421 #if DEBUG_CHURN
4422   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4423               "Stopped peer, %d left.\n", churn_ctx->num_to_stop);
4424 #endif
4425   total_left =
4426     (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) +
4427     (churn_ctx->num_to_start - churn_ctx->num_failed_start);
4428
4429   if (total_left == 0)
4430     {
4431       if ((churn_ctx->num_failed_stop > 0)
4432           || (churn_ctx->num_failed_start > 0))
4433         {
4434           GNUNET_asprintf (&error_message,
4435                            "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
4436                            churn_ctx->num_failed_start,
4437                            churn_ctx->num_failed_stop);
4438         }
4439       churn_ctx->cb (churn_ctx->cb_cls, error_message);
4440       GNUNET_free_non_null (error_message);
4441       GNUNET_free (churn_ctx);
4442       GNUNET_free (shutdown_ctx);
4443     }
4444 }
4445
4446 /**
4447  * Count the number of running peers.
4448  *
4449  * @param pg handle for the peer group
4450  *
4451  * @return the number of currently running peers in the peer group
4452  */
4453 unsigned int
4454 GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
4455 {
4456   unsigned int i;
4457   unsigned int running = 0;
4458   for (i = 0; i < pg->total; i++)
4459     {
4460       if (pg->peers[i].daemon->running == GNUNET_YES)
4461         {
4462           GNUNET_assert (running != -1);
4463           running++;
4464         }
4465     }
4466   return running;
4467 }
4468
4469 /**
4470  * Task to rate limit the number of outstanding peer shutdown
4471  * requests.  This is necessary for making sure we don't do
4472  * too many ssh connections at once, but is generally nicer
4473  * to any system as well (graduated task starts, as opposed
4474  * to calling gnunet-arm N times all at once).
4475  */
4476 static void
4477 schedule_churn_shutdown_task (void *cls,
4478                               const struct GNUNET_SCHEDULER_TaskContext *tc)
4479 {
4480   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4481   struct ShutdownContext *shutdown_ctx;
4482
4483   GNUNET_assert (peer_shutdown_ctx != NULL);
4484   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4485   GNUNET_assert (shutdown_ctx != NULL);
4486
4487   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4488     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4489                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4490                                   &schedule_churn_shutdown_task,
4491                                   peer_shutdown_ctx);
4492   else
4493     {
4494       shutdown_ctx->outstanding++;
4495       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4496                                   shutdown_ctx->timeout, shutdown_ctx->cb,
4497                                   shutdown_ctx, GNUNET_NO, GNUNET_YES);
4498       GNUNET_free (peer_shutdown_ctx);
4499     }
4500 }
4501
4502 /**
4503  * Simulate churn by stopping some peers (and possibly
4504  * re-starting others if churn is called multiple times).  This
4505  * function can only be used to create leave-join churn (peers "never"
4506  * leave for good).  First "voff" random peers that are currently
4507  * online will be taken offline; then "von" random peers that are then
4508  * offline will be put back online.  No notifications will be
4509  * generated for any of these operations except for the callback upon
4510  * completion.
4511  *
4512  * @param pg handle for the peer group
4513  * @param voff number of peers that should go offline
4514  * @param von number of peers that should come back online;
4515  *            must be zero on first call (since "testbed_start"
4516  *            always starts all of the peers)
4517  * @param timeout how long to wait for operations to finish before
4518  *        giving up
4519  * @param cb function to call at the end
4520  * @param cb_cls closure for cb
4521  */
4522 void
4523 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
4524                               unsigned int voff,
4525                               unsigned int von,
4526                               struct GNUNET_TIME_Relative timeout,
4527                               GNUNET_TESTING_NotifyCompletion cb,
4528                               void *cb_cls)
4529 {
4530   struct ChurnContext *churn_ctx;
4531   struct ShutdownContext *shutdown_ctx;
4532   struct PeerShutdownContext *peer_shutdown_ctx;
4533   struct PeerRestartContext *peer_restart_ctx;
4534   struct ChurnRestartContext *churn_startup_ctx;
4535
4536   unsigned int running;
4537   unsigned int stopped;
4538   unsigned int total_running;
4539   unsigned int total_stopped;
4540   unsigned int i;
4541   unsigned int *running_arr;
4542   unsigned int *stopped_arr;
4543   unsigned int *running_permute;
4544   unsigned int *stopped_permute;
4545
4546   running = 0;
4547   stopped = 0;
4548
4549   if ((von == 0) && (voff == 0))        /* No peers at all? */
4550     {
4551       cb (cb_cls, NULL);
4552       return;
4553     }
4554
4555   for (i = 0; i < pg->total; i++)
4556     {
4557       if (pg->peers[i].daemon->running == GNUNET_YES)
4558         {
4559           GNUNET_assert (running != -1);
4560           running++;
4561         }
4562       else
4563         {
4564           GNUNET_assert (stopped != -1);
4565           stopped++;
4566         }
4567     }
4568
4569   if (voff > running)
4570     {
4571       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4572                   "Trying to stop more peers than are currently running!\n");
4573       cb (cb_cls, "Trying to stop more peers than are currently running!");
4574       return;
4575     }
4576
4577   if (von > stopped)
4578     {
4579       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4580                   "Trying to start more peers than are currently stopped!\n");
4581       cb (cb_cls, "Trying to start more peers than are currently stopped!");
4582       return;
4583     }
4584
4585   churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4586
4587   running_arr = NULL;
4588   if (running > 0)
4589     running_arr = GNUNET_malloc (running * sizeof (unsigned int));
4590
4591   stopped_arr = NULL;
4592   if (stopped > 0)
4593     stopped_arr = GNUNET_malloc (stopped * sizeof (unsigned int));
4594
4595   running_permute = NULL;
4596   stopped_permute = NULL;
4597
4598   if (running > 0)
4599     running_permute =
4600       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, running);
4601   if (stopped > 0)
4602     stopped_permute =
4603       GNUNET_CRYPTO_random_permute (GNUNET_CRYPTO_QUALITY_WEAK, stopped);
4604
4605   total_running = running;
4606   total_stopped = stopped;
4607   running = 0;
4608   stopped = 0;
4609
4610   churn_ctx->num_to_start = von;
4611   churn_ctx->num_to_stop = voff;
4612   churn_ctx->cb = cb;
4613   churn_ctx->cb_cls = cb_cls;
4614
4615   for (i = 0; i < pg->total; i++)
4616     {
4617       if (pg->peers[i].daemon->running == GNUNET_YES)
4618         {
4619           GNUNET_assert ((running_arr != NULL) && (total_running > running));
4620           running_arr[running] = i;
4621           running++;
4622         }
4623       else
4624         {
4625           GNUNET_assert ((stopped_arr != NULL) && (total_stopped > stopped));
4626           stopped_arr[stopped] = i;
4627           stopped++;
4628         }
4629     }
4630
4631   GNUNET_assert (running >= voff);
4632   if (voff > 0)
4633     {
4634       shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4635       shutdown_ctx->cb = &churn_stop_callback;
4636       shutdown_ctx->cb_cls = churn_ctx;
4637       shutdown_ctx->total_peers = voff;
4638       shutdown_ctx->timeout = timeout;
4639     }
4640
4641   for (i = 0; i < voff; i++)
4642     {
4643 #if DEBUG_CHURN
4644       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n",
4645                   running_permute[i]);
4646 #endif
4647       GNUNET_assert (running_arr != NULL);
4648       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4649       peer_shutdown_ctx->daemon =
4650         pg->peers[running_arr[running_permute[i]]].daemon;
4651       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4652       GNUNET_SCHEDULER_add_now (&schedule_churn_shutdown_task,
4653                                 peer_shutdown_ctx);
4654
4655       /*
4656          GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
4657          timeout,
4658          &churn_stop_callback, churn_ctx,
4659          GNUNET_NO, GNUNET_YES); */
4660     }
4661
4662   GNUNET_assert (stopped >= von);
4663   if (von > 0)
4664     {
4665       churn_startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4666       churn_startup_ctx->churn_ctx = churn_ctx;
4667       churn_startup_ctx->timeout = timeout;
4668     }
4669   for (i = 0; i < von; i++)
4670     {
4671 #if DEBUG_CHURN
4672       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n",
4673                   stopped_permute[i]);
4674 #endif
4675       GNUNET_assert (stopped_arr != NULL);
4676       peer_restart_ctx = GNUNET_malloc (sizeof (struct PeerRestartContext));
4677       peer_restart_ctx->churn_restart_ctx = churn_startup_ctx;
4678       peer_restart_ctx->daemon =
4679         pg->peers[stopped_arr[stopped_permute[i]]].daemon;
4680       GNUNET_SCHEDULER_add_now (&schedule_churn_restart, peer_restart_ctx);
4681       /*
4682          GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
4683          timeout, &churn_start_callback, churn_ctx); */
4684     }
4685
4686   GNUNET_free_non_null (running_arr);
4687   GNUNET_free_non_null (stopped_arr);
4688   GNUNET_free_non_null (running_permute);
4689   GNUNET_free_non_null (stopped_permute);
4690 }
4691
4692
4693 /**
4694  * Restart all peers in the given group.
4695  *
4696  * @param pg the handle to the peer group
4697  * @param callback function to call on completion (or failure)
4698  * @param callback_cls closure for the callback function
4699  */
4700 void
4701 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
4702                                 GNUNET_TESTING_NotifyCompletion callback,
4703                                 void *callback_cls)
4704 {
4705   struct RestartContext *restart_context;
4706   unsigned int off;
4707
4708   if (pg->total > 0)
4709     {
4710       restart_context = GNUNET_malloc (sizeof (struct RestartContext));
4711       restart_context->peer_group = pg;
4712       restart_context->peers_restarted = 0;
4713       restart_context->callback = callback;
4714       restart_context->callback_cls = callback_cls;
4715
4716       for (off = 0; off < pg->total; off++)
4717         {
4718           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon,
4719                                          &restart_callback, restart_context);
4720         }
4721     }
4722 }
4723
4724 /**
4725  * Start or stop an individual peer from the given group.
4726  *
4727  * @param pg handle to the peer group
4728  * @param offset which peer to start or stop
4729  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
4730  * @param timeout how long to wait for shutdown
4731  * @param cb function to call at the end
4732  * @param cb_cls closure for cb
4733  */
4734 void
4735 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
4736                              unsigned int offset,
4737                              int desired_status,
4738                              struct GNUNET_TIME_Relative timeout,
4739                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4740 {
4741   struct ShutdownContext *shutdown_ctx;
4742   struct ChurnRestartContext *startup_ctx;
4743   struct ChurnContext *churn_ctx;
4744
4745   if (GNUNET_NO == desired_status)
4746     {
4747       if (NULL != pg->peers[offset].daemon)
4748         {
4749           shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4750           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4751           churn_ctx->num_to_start = 0;
4752           churn_ctx->num_to_stop = 1;
4753           churn_ctx->cb = cb;
4754           churn_ctx->cb_cls = cb_cls;
4755           shutdown_ctx->cb_cls = churn_ctx;
4756           GNUNET_TESTING_daemon_stop (pg->peers[offset].daemon,
4757                                       timeout, &churn_stop_callback,
4758                                       shutdown_ctx, GNUNET_NO, GNUNET_YES);
4759         }
4760     }
4761   else if (GNUNET_YES == desired_status)
4762     {
4763       if (NULL == pg->peers[offset].daemon)
4764         {
4765           startup_ctx = GNUNET_malloc (sizeof (struct ChurnRestartContext));
4766           churn_ctx = GNUNET_malloc (sizeof (struct ChurnContext));
4767           churn_ctx->num_to_start = 1;
4768           churn_ctx->num_to_stop = 0;
4769           churn_ctx->cb = cb;
4770           churn_ctx->cb_cls = cb_cls;
4771           startup_ctx->churn_ctx = churn_ctx;
4772           GNUNET_TESTING_daemon_start_stopped (pg->peers[offset].daemon,
4773                                                timeout, &churn_start_callback,
4774                                                startup_ctx);
4775         }
4776     }
4777   else
4778     GNUNET_break (0);
4779 }
4780
4781
4782 /**
4783  * Callback for shutting down peers in a peer group.
4784  *
4785  * @param cls closure (struct ShutdownContext)
4786  * @param emsg NULL on success
4787  */
4788 void
4789 internal_shutdown_callback (void *cls, const char *emsg)
4790 {
4791   struct ShutdownContext *shutdown_ctx = cls;
4792
4793   shutdown_ctx->outstanding--;
4794   if (emsg == NULL)
4795     {
4796       shutdown_ctx->peers_down++;
4797     }
4798   else
4799     {
4800       shutdown_ctx->peers_failed++;
4801     }
4802
4803   if ((shutdown_ctx->cb != NULL)
4804       && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed ==
4805           shutdown_ctx->total_peers))
4806     {
4807       if (shutdown_ctx->peers_failed > 0)
4808         shutdown_ctx->cb (shutdown_ctx->cb_cls,
4809                           "Not all peers successfully shut down!");
4810       else
4811         shutdown_ctx->cb (shutdown_ctx->cb_cls, NULL);
4812       GNUNET_free (shutdown_ctx);
4813     }
4814 }
4815
4816
4817 /**
4818  * Task to rate limit the number of outstanding peer shutdown
4819  * requests.  This is necessary for making sure we don't do
4820  * too many ssh connections at once, but is generally nicer
4821  * to any system as well (graduated task starts, as opposed
4822  * to calling gnunet-arm N times all at once).
4823  */
4824 static void
4825 schedule_shutdown_task (void *cls,
4826                         const struct GNUNET_SCHEDULER_TaskContext *tc)
4827 {
4828   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4829   struct ShutdownContext *shutdown_ctx;
4830
4831   GNUNET_assert (peer_shutdown_ctx != NULL);
4832   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4833   GNUNET_assert (shutdown_ctx != NULL);
4834
4835   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4836     GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply
4837                                   (GNUNET_TIME_UNIT_MILLISECONDS, 100),
4838                                   &schedule_shutdown_task, peer_shutdown_ctx);
4839   else
4840     {
4841       shutdown_ctx->outstanding++;
4842       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon,
4843                                   shutdown_ctx->timeout,
4844                                   &internal_shutdown_callback, shutdown_ctx,
4845                                   GNUNET_YES, GNUNET_NO);
4846       GNUNET_free (peer_shutdown_ctx);
4847     }
4848 }
4849
4850 /**
4851  * Shutdown all peers started in the given group.
4852  *
4853  * @param pg handle to the peer group
4854  * @param timeout how long to wait for shutdown
4855  * @param cb callback to notify upon success or failure
4856  * @param cb_cls closure for cb
4857  */
4858 void
4859 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4860                              struct GNUNET_TIME_Relative timeout,
4861                              GNUNET_TESTING_NotifyCompletion cb, void *cb_cls)
4862 {
4863   unsigned int off;
4864   struct ShutdownContext *shutdown_ctx;
4865   struct PeerShutdownContext *peer_shutdown_ctx;
4866
4867   GNUNET_assert (pg->total > 0);
4868
4869   shutdown_ctx = GNUNET_malloc (sizeof (struct ShutdownContext));
4870   shutdown_ctx->cb = cb;
4871   shutdown_ctx->cb_cls = cb_cls;
4872   shutdown_ctx->total_peers = pg->total;
4873   shutdown_ctx->timeout = timeout;
4874   /* shtudown_ctx->outstanding = 0; */
4875
4876   for (off = 0; off < pg->total; off++)
4877     {
4878       GNUNET_assert (NULL != pg->peers[off].daemon);
4879       peer_shutdown_ctx = GNUNET_malloc (sizeof (struct PeerShutdownContext));
4880       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4881       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4882       GNUNET_SCHEDULER_add_now (&schedule_shutdown_task, peer_shutdown_ctx);
4883       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4884       if (NULL != pg->peers[off].cfg)
4885         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4886       if (pg->peers[off].allowed_peers != NULL)
4887         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].allowed_peers);
4888       if (pg->peers[off].connect_peers != NULL)
4889         GNUNET_CONTAINER_multihashmap_destroy (pg->peers[off].connect_peers);
4890       if (pg->peers[off].blacklisted_peers != NULL)
4891         GNUNET_CONTAINER_multihashmap_destroy (pg->
4892                                                peers[off].blacklisted_peers);
4893     }
4894   GNUNET_free (pg->peers);
4895   for (off = 0; off < pg->num_hosts; off++)
4896     {
4897       GNUNET_free (pg->hosts[off].hostname);
4898       GNUNET_free_non_null (pg->hosts[off].username);
4899     }
4900   GNUNET_free_non_null (pg->hosts);
4901   GNUNET_free (pg);
4902 }
4903
4904
4905 /* end of testing_group.c */