src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define DEBUG_CHURN GNUNET_NO
  36
  37 /**
  38  * Lowest port used for GNUnet testing.  Should be high enough to not
  39  * conflict with other applications running on the hosts but be low
  40  * enough to not conflict with client-ports (typically starting around
  41  * 32k).
  42  */
  43 #define LOW_PORT 10000
  44
  45 /**
  46  * Highest port used for GNUnet testing.  Should be low enough to not
  47  * conflict with the port range for "local" ports (client apps; see
  48  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  49  */
  50 #define HIGH_PORT 56000
  51
  52 #define MAX_OUTSTANDING_CONNECTIONS 10
  53
  54 #define MAX_CONCURRENT_HOSTKEYS 10
  55
  56 #define MAX_CONCURRENT_STARTING 10
  57
  58 #define MAX_CONCURRENT_SHUTDOWN 10
  59
  60 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
  61
  62 #define CONNECT_ATTEMPTS 8
  63
  64 /**
  65  * Prototype of a function called whenever two peers would be connected
  66  * in a certain topology.
  67  */
  68 typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg,
  69                                                   unsigned int first,
  70                                                   unsigned int second);
  71
  72
  73 /**
  74  * Context for handling churning a peer group
  75  */
  76 struct ChurnContext
  77 {
  78   /**
  79    * Callback used to notify of churning finished
  80    */
  81   GNUNET_TESTING_NotifyCompletion cb;
  82
  83   /**
  84    * Closure for callback
  85    */
  86   void *cb_cls;
  87
  88   /**
  89    * Number of peers that still need to be started
  90    */
  91   unsigned int num_to_start;
  92
  93   /**
  94    * Number of peers that still need to be stopped
  95    */
  96   unsigned int num_to_stop;
  97
  98   /**
  99    * Number of peers that failed to start
 100    */
 101   unsigned int num_failed_start;
 102
 103   /**
 104    * Number of peers that failed to stop
 105    */
 106   unsigned int num_failed_stop;
 107 };
 108
 109 struct RestartContext
 110 {
 111   /**
 112    * The group of peers being restarted
 113    */
 114   struct GNUNET_TESTING_PeerGroup *peer_group;
 115
 116   /**
 117    * How many peers have been restarted thus far
 118    */
 119   unsigned int peers_restarted;
 120
 121   /**
 122    * How many peers got an error when restarting
 123    */
 124   unsigned int peers_restart_failed;
 125
 126   /**
 127    * The function to call once all peers have been restarted
 128    */
 129   GNUNET_TESTING_NotifyCompletion callback;
 130
 131   /**
 132    * Closure for callback function
 133    */
 134   void *callback_cls;
 135
 136 };
 137
 138
 139 struct ShutdownContext
 140 {
 141   /**
 142    * Total peers to wait for
 143    */
 144   unsigned int total_peers;
 145
 146   /**
 147    * Number of peers successfully shut down
 148    */
 149   unsigned int peers_down;
 150
 151   /**
 152    * Number of peers failed to shut down
 153    */
 154   unsigned int peers_failed;
 155
 156   /**
 157    * Number of peers we have started shutting
 158    * down.  If too many, wait on them.
 159    */
 160   unsigned int outstanding;
 161
 162   /**
 163    * Timeout for shutdown.
 164    */
 165   struct GNUNET_TIME_Relative timeout;
 166
 167   /**
 168    * Callback to call when all peers either
 169    * shutdown or failed to shutdown
 170    */
 171   GNUNET_TESTING_NotifyCompletion cb;
 172
 173   /**
 174    * Closure for cb
 175    */
 176   void *cb_cls;
 177 };
 178
 179 struct CreateTopologyContext
 180 {
 181
 182   /**
 183    * Function to call with number of connections
 184    */
 185   GNUNET_TESTING_NotifyConnections cont;
 186
 187   /**
 188    * Closure for connection notification
 189    */
 190   void *cls;
 191 };
 192
 193 #if OLD
 194 struct PeerConnection
 195 {
 196   /*
 197    * Linked list
 198    */
 199   struct PeerConnection *next;
 200
 201   /*
 202    * Pointer to daemon handle
 203    */
 204   struct GNUNET_TESTING_Daemon *daemon;
 205
 206 };
 207 #endif
 208
 209 struct InternalStartContext
 210 {
 211   /**
 212    * Pointer to peerdata
 213    */
 214   struct PeerData *peer;
 215
 216   /**
 217    * Timeout for peer startup
 218    */
 219   struct GNUNET_TIME_Relative timeout;
 220
 221   /**
 222    * Client callback for hostkey notification
 223    */
 224   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 225
 226   /**
 227    * Closure for hostkey_callback
 228    */
 229   void *hostkey_cls;
 230
 231   /**
 232    * Client callback for peer start notification
 233    */
 234   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 235
 236   /**
 237    * Closure for cb
 238    */
 239   void *start_cb_cls;
 240
 241   /**
 242    * Hostname, where to start the peer
 243    */
 244   const char *hostname;
 245 };
 246
 247 /**
 248  * Data we keep per peer.
 249  */
 250 struct PeerData
 251 {
 252   /**
 253    * (Initial) configuration of the host.
 254    * (initial because clients could change
 255    *  it and we would not know about those
 256    *  updates).
 257    */
 258   struct GNUNET_CONFIGURATION_Handle *cfg;
 259
 260   /**
 261    * Handle for controlling the daemon.
 262    */
 263   struct GNUNET_TESTING_Daemon *daemon;
 264
 265   /**
 266    * The peergroup this peer belongs to.
 267    */
 268   struct GNUNET_TESTING_PeerGroup *pg;
 269
 270   /**
 271    * Hash map of allowed peer connections (F2F created topology)
 272    */
 273   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 274
 275   /**
 276    * Hash map of blacklisted peers
 277    */
 278   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 279
 280   /**
 281    * Hash map of peer connections
 282    */
 283   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 284
 285   /**
 286    * Temporary hash map of peer connections
 287    */
 288   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 289
 290   /**
 291    * Temporary variable for topology creation, should be reset before
 292    * creating any topology so the count is valid once finished.
 293    */
 294   int num_connections;
 295
 296   /**
 297    * Context to keep track of peers being started, to
 298    * stagger hostkey generation and peer startup.
 299    */
 300   struct InternalStartContext internal_context;
 301 };
 302
 303
 304 /**
 305  * Data we keep per host.
 306  */
 307 struct HostData
 308 {
 309   /**
 310    * Name of the host.
 311    */
 312   char *hostname;
 313
 314   /**
 315    * Lowest port that we have not yet used
 316    * for GNUnet.
 317    */
 318   uint16_t minport;
 319 };
 320
 321 struct TopologyIterateContext
 322 {
 323   /**
 324    * Callback for notifying of two connected peers.
 325    */
 326   GNUNET_TESTING_NotifyTopology topology_cb;
 327
 328   /**
 329    * Closure for topology_cb
 330    */
 331   void *cls;
 332
 333   /**
 334    * Number of peers currently connected to.
 335    */
 336   unsigned int connected;
 337
 338   /**
 339    * Number of peers we have finished iterating.
 340    */
 341   unsigned int completed;
 342
 343   /**
 344    * Number of peers total.
 345    */
 346   unsigned int total;
 347 };
 348
 349 struct StatsIterateContext
 350 {
 351   /**
 352    * Continuation to call once all stats information has been retrieved.
 353    */
 354   GNUNET_STATISTICS_Callback cont;
 355
 356   /**
 357    * Proc function to call on each value received.
 358    */
 359   GNUNET_TESTING_STATISTICS_Iterator proc;
 360
 361   /**
 362    * Closure for topology_cb
 363    */
 364   void *cls;
 365
 366   /**
 367    * Number of peers currently connected to.
 368    */
 369   unsigned int connected;
 370
 371   /**
 372    * Number of peers we have finished iterating.
 373    */
 374   unsigned int completed;
 375
 376   /**
 377    * Number of peers total.
 378    */
 379   unsigned int total;
 380 };
 381
 382 struct CoreContext
 383 {
 384   void *iter_context;
 385   struct GNUNET_TESTING_Daemon *daemon;
 386 };
 387
 388 struct StatsCoreContext
 389 {
 390   void *iter_context;
 391   struct GNUNET_TESTING_Daemon *daemon;
 392   /**
 393    * Handle to the statistics service.
 394    */
 395   struct GNUNET_STATISTICS_Handle *stats_handle;
 396
 397   /**
 398    * Handle for getting statistics.
 399    */
 400   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 401 };
 402
 403 /**
 404  * Handle to a group of GNUnet peers.
 405  */
 406 struct GNUNET_TESTING_PeerGroup
 407 {
 408   /**
 409    * Our scheduler.
 410    */
 411   struct GNUNET_SCHEDULER_Handle *sched;
 412
 413   /**
 414    * Configuration template.
 415    */
 416   const struct GNUNET_CONFIGURATION_Handle *cfg;
 417
 418   /**
 419    * Function to call on each started daemon.
 420    */
 421   //GNUNET_TESTING_NotifyDaemonRunning cb;
 422
 423   /**
 424    * Closure for cb.
 425    */
 426   //void *cb_cls;
 427
 428   /*
 429    * Function to call on each topology connection created
 430    */
 431   GNUNET_TESTING_NotifyConnection notify_connection;
 432
 433   /*
 434    * Callback for notify_connection
 435    */
 436   void *notify_connection_cls;
 437
 438   /**
 439    * NULL-terminated array of information about
 440    * hosts.
 441    */
 442   struct HostData *hosts;
 443
 444   /**
 445    * Array of "total" peers.
 446    */
 447   struct PeerData *peers;
 448
 449   /**
 450    * Number of peers in this group.
 451    */
 452   unsigned int total;
 453
 454   /**
 455    * At what time should we fail the peer startup process?
 456    */
 457   struct GNUNET_TIME_Absolute max_timeout;
 458
 459   /**
 460    * How many peers are being started right now?
 461    */
 462   unsigned int starting;
 463
 464   /**
 465    * How many peers have already been started?
 466    */
 467   unsigned int started;
 468 };
 469
 470 struct UpdateContext
 471 {
 472   struct GNUNET_CONFIGURATION_Handle *ret;
 473   const struct GNUNET_CONFIGURATION_Handle *orig;
 474   const char *hostname;
 475   unsigned int nport;
 476   unsigned int upnum;
 477   unsigned int fdnum;
 478 };
 479
 480
 481 struct ConnectContext
 482 {
 483   struct GNUNET_TESTING_Daemon *first;
 484
 485   struct GNUNET_TESTING_Daemon *second;
 486
 487   struct GNUNET_TESTING_PeerGroup *pg;
 488 };
 489
 490 /**
 491  * Convert unique ID to hash code.
 492  *
 493  * @param uid unique ID to convert
 494  * @param hash set to uid (extended with zeros)
 495  */
 496 static void
 497 hash_from_uid (uint32_t uid,
 498                GNUNET_HashCode *hash)
 499 {
 500   memset (hash, 0, sizeof(GNUNET_HashCode));
 501   *((uint32_t*)hash) = uid;
 502 }
 503
 504 /**
 505  * Convert hash code to unique ID.
 506  *
 507  * @param uid unique ID to convert
 508  * @param hash set to uid (extended with zeros)
 509  */
 510 static void
 511 uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
 512 {
 513   memcpy (uid, hash, sizeof(uint32_t));
 514 }
 515
 516 /**
 517  * Number of connects we are waiting on, allows us to rate limit
 518  * connect attempts.
 519  */
 520 static int outstanding_connects;
 521
 522 /**
 523  * Get a topology from a string input.
 524  *
 525  * @param topology where to write the retrieved topology
 526  * @param topology_string The string to attempt to
 527  *        get a configuration value from
 528  * @return GNUNET_YES if topology string matched a
 529  *         known topology, GNUNET_NO if not
 530  */
 531 int
 532 GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
 533 {
 534   /**
 535    * Strings representing topologies in enum
 536    */
 537   static const char * topology_strings[] =
 538     {
 539       /**
 540        * A clique (everyone connected to everyone else).
 541        */
 542       "CLIQUE",
 543
 544       /**
 545        * Small-world network (2d torus plus random links).
 546        */
 547       "SMALL_WORLD",
 548
 549       /**
 550        * Small-world network (ring plus random links).
 551        */
 552       "SMALL_WORLD_RING",
 553
 554       /**
 555        * Ring topology.
 556        */
 557       "RING",
 558
 559       /**
 560        * 2-d torus.
 561        */
 562       "2D_TORUS",
 563
 564       /**
 565        * Random graph.
 566        */
 567       "ERDOS_RENYI",
 568
 569       /**
 570        * Certain percentage of peers are unable to communicate directly
 571        * replicating NAT conditions
 572        */
 573       "INTERNAT",
 574
 575       /**
 576        * Scale free topology.
 577        */
 578       "SCALE_FREE",
 579
 580       /**
 581        * Straight line topology.
 582        */
 583       "LINE",
 584
 585       /**
 586        * All peers are disconnected.
 587        */
 588       "NONE",
 589
 590       NULL
 591     };
 592
 593   int curr = 0;
 594   if (topology_string == NULL)
 595     return GNUNET_NO;
 596   while (topology_strings[curr] != NULL)
 597     {
 598       if (strcasecmp(topology_strings[curr], topology_string) == 0)
 599         {
 600           *topology = curr;
 601           return GNUNET_YES;
 602         }
 603       curr++;
 604     }
 605   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 606   return GNUNET_NO;
 607 }
 608
 609
 610 /**
 611  * Get connect topology option from string input.
 612  *
 613  * @param topology_option where to write the retrieved topology
 614  * @param topology_string The string to attempt to
 615  *        get a configuration value from
 616  * @return GNUNET_YES if string matched a known
 617  *         topology option, GNUNET_NO if not
 618  */
 619 int
 620 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
 621                                     char * topology_string)
 622 {
 623   /**
 624    * Options for connecting a topology as strings.
 625    */
 626   static const char * topology_option_strings[] =
 627     {
 628       /**
 629        * Try to connect all peers specified in the topology.
 630        */
 631       "CONNECT_ALL",
 632
 633       /**
 634        * Choose a random subset of connections to create.
 635        */
 636       "CONNECT_RANDOM_SUBSET",
 637
 638       /**
 639        * Create at least X connections for each peer.
 640        */
 641       "CONNECT_MINIMUM",
 642
 643       /**
 644        * Using a depth first search, create one connection
 645        * per peer.  If any are missed (graph disconnected)
 646        * start over at those peers until all have at least one
 647        * connection.
 648        */
 649       "CONNECT_DFS",
 650
 651       /**
 652        * No options specified.
 653        */
 654       "CONNECT_NONE",
 655
 656       NULL
 657     };
 658   int curr = 0;
 659
 660   if (topology_string == NULL)
 661     return GNUNET_NO;
 662   while (NULL != topology_option_strings[curr])
 663     {
 664       if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
 665         {
 666           *topology_option = curr;
 667           return GNUNET_YES;
 668         }
 669       curr++;
 670     }
 671   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 672   return GNUNET_NO;
 673 }
 674
 675 /**
 676  * Function to iterate over options.  Copies
 677  * the options to the target configuration,
 678  * updating PORT values as needed.
 679  *
 680  * @param cls closure
 681  * @param section name of the section
 682  * @param option name of the option
 683  * @param value value of the option
 684  */
 685 static void
 686 update_config (void *cls,
 687                const char *section, const char *option, const char *value)
 688 {
 689   struct UpdateContext *ctx = cls;
 690   unsigned int ival;
 691   char cval[12];
 692   char uval[128];
 693   char *single_variable;
 694   char *per_host_variable;
 695   unsigned long long num_per_host;
 696
 697   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 698     {
 699       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 700       if ((ival != 0) && (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable)))
 701         {
 702           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 703           value = cval;
 704         }
 705
 706       GNUNET_free(single_variable);
 707     }
 708
 709   if (0 == strcmp (option, "UNIXPATH"))
 710     {
 711       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 712       GNUNET_asprintf(&per_host_variable, "num_%s_per_host", section);
 713       if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable))
 714         {
 715           GNUNET_snprintf (uval,
 716                            sizeof (uval),
 717                            "/tmp/test-service-%s-%u",
 718                            section,
 719                            ctx->upnum++);
 720           value = uval;
 721         }
 722       else if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number(ctx->orig, "testing", per_host_variable, &num_per_host))
 723         {
 724           GNUNET_snprintf (uval,
 725                            sizeof (uval),
 726                            "/tmp/test-service-%s-%u",
 727                            section,
 728                            ctx->fdnum % num_per_host);
 729           value = uval;
 730         }
 731       GNUNET_free(single_variable);
 732
 733     }
 734
 735   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 736     {
 737       value = ctx->hostname;
 738     }
 739
 740   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 741 }
 742
 743
 744 /**
 745  * Create a new configuration using the given configuration
 746  * as a template; however, each PORT in the existing cfg
 747  * must be renumbered by incrementing "*port".  If we run
 748  * out of "*port" numbers, return NULL.
 749  *
 750  * @param cfg template configuration
 751  * @param port port numbers to use, update to reflect
 752  *             port numbers that were used
 753  * @param upnum number to make unix domain socket names unique
 754  * @param hostname hostname of the controlling host, to allow control connections from
 755  * @param fdnum number used to offset the unix domain socket for grouped processes
 756  *              (such as statistics or peerinfo, which can be shared among others)
 757  *
 758  * @return new configuration, NULL on error
 759  */
 760 static struct GNUNET_CONFIGURATION_Handle *
 761 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 762              uint16_t * port,
 763              uint32_t * upnum,
 764              const char *hostname, uint32_t * fdnum)
 765 {
 766   struct UpdateContext uc;
 767   uint16_t orig;
 768   char *control_host;
 769   char *allowed_hosts;
 770
 771   orig = *port;
 772   uc.nport = *port;
 773   uc.upnum = *upnum;
 774   uc.fdnum = *fdnum;
 775   uc.ret = GNUNET_CONFIGURATION_create ();
 776   uc.hostname = hostname;
 777   uc.orig = cfg;
 778
 779   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 780   if (uc.nport >= HIGH_PORT)
 781     {
 782       *port = orig;
 783       GNUNET_CONFIGURATION_destroy (uc.ret);
 784       return NULL;
 785     }
 786
 787   if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 788     {
 789       if (hostname != NULL)
 790         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1; %s;", control_host, hostname);
 791       else
 792         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
 793
 794       GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
 795       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport", "ACCEPT_FROM", allowed_hosts);
 796       GNUNET_CONFIGURATION_set_value_string(uc.ret, "dht", "ACCEPT_FROM", allowed_hosts);
 797       GNUNET_CONFIGURATION_set_value_string(uc.ret, "statistics", "ACCEPT_FROM", allowed_hosts);
 798       GNUNET_free_non_null(control_host);
 799       GNUNET_free(allowed_hosts);
 800     }
 801
 802
 803   /* arm needs to know to allow connections from the host on which it is running,
 804    * otherwise gnunet-arm is unable to connect to it in some instances */
 805   if (hostname != NULL)
 806     {
 807       GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
 808       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", hostname);
 809       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", hostname);
 810       GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
 811       GNUNET_free(allowed_hosts);
 812     }
 813   else
 814     {
 815       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", "127.0.0.1");
 816       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", "127.0.0.1");
 817     }
 818
 819   *port = (uint16_t) uc.nport;
 820   *upnum = uc.upnum;
 821   uc.fdnum++;
 822   *fdnum = uc.fdnum;
 823   return uc.ret;
 824 }
 825
 826
 827 /*
 828  * Add entries to the peers connect list
 829  *
 830  * @param pg the peer group we are working with
 831  * @param first index of the first peer
 832  * @param second index of the second peer
 833  *
 834  * @return the number of connections added
 835  *         technically should only be 0 or 2
 836  *
 837  */
 838 static int
 839 add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 840 {
 841   int added;
 842   int add_first;
 843   int add_second;
 844
 845   GNUNET_HashCode hash_first;
 846   GNUNET_HashCode hash_second;
 847
 848   hash_from_uid(first, &hash_first);
 849   hash_from_uid(second, &hash_second);
 850
 851   add_first = GNUNET_NO;
 852   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
 853     {
 854       add_first = GNUNET_YES;
 855     }
 856
 857   add_second = GNUNET_NO;
 858   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
 859     {
 860       add_second = GNUNET_YES;
 861     }
 862
 863   added = 0;
 864   if (add_first)
 865     {
 866       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 867       pg->peers[first].num_connections++;
 868       added++;
 869     }
 870
 871   if (add_second)
 872     {
 873       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 874       pg->peers[second].num_connections++;
 875       added++;
 876     }
 877
 878   return added;
 879 }
 880
 881
 882 /*
 883  * Add entries to the peers allowed connections list
 884  *
 885  * @param pg the peer group we are working with
 886  * @param first index of the first peer
 887  * @param second index of the second peer
 888  *
 889  * @return the number of connections added (can be 0, 1 or 2)
 890  *         technically should only be 0 or 2, but the small price
 891  *         of iterating over the lists (hashmaps in the future)
 892  *         for being sure doesn't bother me!
 893  *
 894  */
 895 static int
 896 add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 897 {
 898   int added;
 899 #if OLD
 900   struct PeerConnection *first_iter;
 901   struct PeerConnection *second_iter;
 902   struct PeerConnection *new_first;
 903   struct PeerConnection *new_second;
 904 #endif
 905   int add_first;
 906   int add_second;
 907
 908   GNUNET_HashCode hash_first;
 909   GNUNET_HashCode hash_second;
 910
 911   hash_from_uid(first, &hash_first);
 912   hash_from_uid(second, &hash_second);
 913
 914   add_first = GNUNET_NO;
 915   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
 916     {
 917       add_first = GNUNET_YES;
 918     }
 919
 920   add_second = GNUNET_NO;
 921   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
 922     {
 923       add_second = GNUNET_YES;
 924     }
 925 #if OLD
 926   first_iter = pg->peers[first].connected_peers;
 927   while (first_iter != NULL)
 928     {
 929       if (first_iter->daemon == pg->peers[second].daemon)
 930         add_first = GNUNET_NO;
 931       first_iter = first_iter->next;
 932     }
 933
 934   second_iter = pg->peers[second].connected_peers;
 935   add_second = GNUNET_YES;
 936   while (second_iter != NULL)
 937     {
 938       if (second_iter->daemon == pg->peers[first].daemon)
 939         add_second = GNUNET_NO;
 940       second_iter = second_iter->next;
 941     }
 942 #endif
 943
 944   added = 0;
 945   if (add_first)
 946     {
 947       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 948 #if OLD
 949       new_first = GNUNET_malloc(sizeof(struct PeerConnection));
 950       new_first->daemon = pg->peers[second].daemon;
 951       new_first->next = pg->peers[first].connected_peers;
 952       pg->peers[first].connected_peers = new_first;
 953 #endif
 954       pg->peers[first].num_connections++;
 955       added++;
 956     }
 957
 958   if (add_second)
 959     {
 960       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 961 #if OLD
 962       new_second = GNUNET_malloc(sizeof(struct PeerConnection));
 963       new_second->daemon = pg->peers[first].daemon;
 964       new_second->next = pg->peers[second].connected_peers;
 965       pg->peers[second].connected_peers = new_second;
 966       pg->peers[first].num_connections++;
 967 #endif
 968       pg->peers[second].num_connections++;
 969       added++;
 970     }
 971
 972   return added;
 973 }
 974
 975 /*
 976  * Add entries to the peers blacklisted list
 977  *
 978  * @param pg the peer group we are working with
 979  * @param first index of the first peer
 980  * @param second index of the second peer
 981  *
 982  * @return the number of connections added (can be 0, 1 or 2)
 983  *
 984  */
 985 static int
 986 blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 987 {
 988   int added;
 989   int add_first;
 990   int add_second;
 991   GNUNET_HashCode hash_first;
 992   GNUNET_HashCode hash_second;
 993
 994   hash_from_uid(first, &hash_first);
 995   hash_from_uid(second, &hash_second);
 996
 997   add_first = GNUNET_NO;
 998   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
 999     {
1000       add_first = GNUNET_YES;
1001     }
1002
1003   add_second = GNUNET_NO;
1004   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
1005     {
1006       add_second = GNUNET_YES;
1007     }
1008
1009   added = 0;
1010   if (add_first)
1011     {
1012       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1013       pg->peers[first].num_connections++;
1014       added++;
1015     }
1016
1017   if (add_second)
1018     {
1019       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1020       pg->peers[second].num_connections++;
1021       added++;
1022     }
1023
1024   return added;
1025 }
1026
1027 /*
1028  * Remove entries from the peers blacklisted list
1029  *
1030  * @param pg the peer group we are working with
1031  * @param first index of the first peer
1032  * @param second index of the second peer
1033  *
1034  * @return the number of connections removed (can be 0, 1 or 2)
1035  *
1036  */
1037 static int
1038 unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1039 {
1040   int removed;
1041   int remove_first;
1042   int remove_second;
1043   GNUNET_HashCode hash_first;
1044   GNUNET_HashCode hash_second;
1045
1046   hash_from_uid(first, &hash_first);
1047   hash_from_uid(second, &hash_second);
1048
1049   remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
1050   remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
1051
1052   removed = 0;
1053   if (remove_first)
1054     {
1055       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
1056       removed++;
1057     }
1058
1059   if (remove_second)
1060     {
1061       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
1062       removed++;
1063     }
1064
1065   return removed;
1066 }
1067
1068 /**
1069  * Scale free network construction as described in:
1070  *
1071  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1072  *
1073  * Start with a network of "one" peer, then progressively add
1074  * peers up to the total number.  At each step, iterate over
1075  * all possible peers and connect new peer based on number of
1076  * existing connections of the target peer.
1077  *
1078  * @param pg the peer group we are dealing with
1079  * @param proc the connection processor to use
1080  *
1081  * @return the number of connections created
1082  */
1083 static int
1084 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1085 {
1086
1087   unsigned int total_connections;
1088   unsigned int outer_count;
1089   unsigned int i;
1090   unsigned int previous_total_connections;
1091   double random;
1092   double probability;
1093
1094   GNUNET_assert(pg->total > 1);
1095
1096   /* Add a connection between the first two nodes */
1097   total_connections = proc(pg, 0, 1);
1098
1099   for (outer_count = 1; outer_count < pg->total; outer_count++)
1100     {
1101       previous_total_connections = total_connections;
1102       for (i = 0; i < outer_count; i++)
1103         {
1104           probability = pg->peers[i].num_connections / (double)previous_total_connections;
1105           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1106                                                       UINT64_MAX)) / ( (double) UINT64_MAX);
1107 #if VERBOSE_TESTING
1108           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1109                       "Considering connecting peer %d to peer %d\n",
1110                       outer_count, i);
1111 #endif
1112           if (random < probability)
1113             {
1114 #if VERBOSE_TESTING
1115               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1116                           "Connecting peer %d to peer %d\n",
1117                           outer_count, i);
1118 #endif
1119               total_connections += proc(pg, outer_count, i);
1120             }
1121         }
1122     }
1123
1124   return total_connections;
1125 }
1126
1127 /**
1128  * Create a topology given a peer group (set of running peers)
1129  * and a connection processor.
1130  *
1131  * @param pg the peergroup to create the topology on
1132  * @param proc the connection processor to call to actually set
1133  *        up connections between two peers
1134  *
1135  * @return the number of connections that were set up
1136  *
1137  */
1138 int
1139 create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1140 {
1141   unsigned int i, j;
1142   int nodeToConnect;
1143   unsigned int natLog;
1144   unsigned int randomPeer;
1145   double random, logNModifier, percentage;
1146   unsigned int smallWorldConnections;
1147   int connsPerPeer;
1148   char *p_string;
1149   int max;
1150   int min;
1151   unsigned int useAnd;
1152   int connect_attempts;
1153
1154   logNModifier = 0.5; /* FIXME: default value? */
1155   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1156                                                          "TESTING",
1157                                                          "LOGNMODIFIER",
1158                                                          &p_string))
1159     {
1160       if (sscanf(p_string, "%lf", &logNModifier) != 1)
1161         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1162                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1163                     p_string,
1164                     "LOGNMODIFIER",
1165                     "TESTING");
1166       GNUNET_free (p_string);
1167     }
1168   percentage = 0.5; /* FIXME: default percentage? */
1169   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1170                                                          "TESTING",
1171                                                          "PERCENTAGE",
1172                                                          &p_string))
1173     {
1174       if (sscanf(p_string, "%lf", &percentage) != 1)
1175         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1176                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1177                     p_string,
1178                     "PERCENTAGE",
1179                     "TESTING");
1180       GNUNET_free (p_string);
1181     }
1182   natLog = log (pg->total);
1183   connsPerPeer = ceil (natLog * logNModifier);
1184
1185   if (connsPerPeer % 2 == 1)
1186     connsPerPeer += 1;
1187
1188   smallWorldConnections = 0;
1189   connect_attempts = 0;
1190   for (i = 0; i < pg->total; i++)
1191     {
1192       useAnd = 0;
1193       max = i + connsPerPeer / 2;
1194       min = i - connsPerPeer / 2;
1195
1196       if (max > pg->total - 1)
1197         {
1198           max = max - pg->total;
1199           useAnd = 1;
1200         }
1201
1202       if (min < 0)
1203         {
1204           min = pg->total - 1 + min;
1205           useAnd = 1;
1206         }
1207
1208       for (j = 0; j < connsPerPeer / 2; j++)
1209         {
1210           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1211                                                       UINT64_MAX) / ( (double) UINT64_MAX));
1212           if (random < percentage)
1213             {
1214               /* Connect to uniformly selected random peer */
1215               randomPeer =
1216                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1217                                    pg->total);
1218               while ((((randomPeer < max) && (randomPeer > min))
1219                       && (useAnd == 0)) || (((randomPeer > min)
1220                                              || (randomPeer < max))
1221                                             && (useAnd == 1)))
1222                 {
1223                   randomPeer =
1224                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1225                                                          pg->total);
1226                 }
1227               smallWorldConnections +=
1228                 proc (pg, i, randomPeer);
1229             }
1230           else
1231             {
1232               nodeToConnect = i + j + 1;
1233               if (nodeToConnect > pg->total - 1)
1234                 {
1235                   nodeToConnect = nodeToConnect - pg->total;
1236                 }
1237               connect_attempts +=
1238                 proc (pg, i, nodeToConnect);
1239             }
1240         }
1241
1242     }
1243
1244   connect_attempts += smallWorldConnections;
1245
1246   return connect_attempts;
1247 }
1248
1249 /**
1250  * Create a topology given a peer group (set of running peers)
1251  * and a connection processor.
1252  *
1253  * @param pg the peergroup to create the topology on
1254  * @param proc the connection processor to call to actually set
1255  *        up connections between two peers
1256  *
1257  * @return the number of connections that were set up
1258  *
1259  */
1260 static int
1261 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1262 {
1263   unsigned int outer_count, inner_count;
1264   unsigned int cutoff;
1265   int connect_attempts;
1266   double nat_percentage;
1267   char *p_string;
1268
1269   nat_percentage = 0.6; /* FIXME: default percentage? */
1270   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1271                                                          "TESTING",
1272                                                          "NATPERCENTAGE",
1273                                                          &p_string))
1274     {
1275       if (sscanf(p_string, "%lf", &nat_percentage) != 1)
1276         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1277                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1278                     p_string,
1279                     "NATPERCENTAGE",
1280                     "TESTING");
1281       GNUNET_free (p_string);
1282     }
1283
1284
1285
1286   cutoff = (unsigned int) (nat_percentage * pg->total);
1287
1288   connect_attempts = 0;
1289
1290   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1291     {
1292       for (inner_count = outer_count + 1; inner_count < pg->total;
1293            inner_count++)
1294         {
1295           if ((outer_count > cutoff) || (inner_count > cutoff))
1296             {
1297 #if VERBOSE_TESTING
1298               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1299                           "Connecting peer %d to peer %d\n",
1300                           outer_count, inner_count);
1301 #endif
1302               connect_attempts += proc(pg, outer_count, inner_count);
1303             }
1304         }
1305     }
1306
1307   return connect_attempts;
1308
1309 }
1310
1311 /**
1312  * Create a topology given a peer group (set of running peers)
1313  * and a connection processor.
1314  *
1315  * @param pg the peergroup to create the topology on
1316  * @param proc the connection processor to call to actually set
1317  *        up connections between two peers
1318  *
1319  * @return the number of connections that were set up
1320  *
1321  */
1322 static int
1323 create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1324 {
1325   unsigned int i, j, k;
1326   unsigned int square;
1327   unsigned int rows;
1328   unsigned int cols;
1329   unsigned int toggle = 1;
1330   unsigned int nodeToConnect;
1331   unsigned int natLog;
1332   unsigned int node1Row;
1333   unsigned int node1Col;
1334   unsigned int node2Row;
1335   unsigned int node2Col;
1336   unsigned int distance;
1337   double probability, random, percentage;
1338   unsigned int smallWorldConnections;
1339   unsigned int small_world_it;
1340   char *p_string;
1341   int connect_attempts;
1342   square = floor (sqrt (pg->total));
1343   rows = square;
1344   cols = square;
1345
1346   percentage = 0.5; /* FIXME: default percentage? */
1347   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1348                                                          "TESTING",
1349                                                          "PERCENTAGE",
1350                                                          &p_string))
1351     {
1352       if (sscanf(p_string, "%lf", &percentage) != 1)
1353         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1354                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1355                     p_string,
1356                     "PERCENTAGE",
1357                     "TESTING");
1358       GNUNET_free (p_string);
1359     }
1360   if (percentage < 0.0)
1361     {
1362       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1363                   _("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1364                   "PERCENTAGE", "TESTING", percentage);
1365       percentage = 0.5;
1366     }
1367   probability = 0.5; /* FIXME: default percentage? */
1368   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1369                                                          "TESTING",
1370                                                          "PROBABILITY",
1371                                                          &p_string))
1372     {
1373       if (sscanf(p_string, "%lf", &probability) != 1)
1374         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1375                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1376                     p_string,
1377                     "PROBABILITY",
1378                     "TESTING");
1379       GNUNET_free (p_string);
1380     }
1381   if (square * square != pg->total)
1382     {
1383       while (rows * cols < pg->total)
1384         {
1385           if (toggle % 2 == 0)
1386             rows++;
1387           else
1388             cols++;
1389
1390           toggle++;
1391         }
1392     }
1393 #if VERBOSE_TESTING
1394       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1395                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1396                   rows, cols);
1397 #endif
1398
1399   connect_attempts = 0;
1400   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1401    * to the node to its right and above.  Once this is over, we'll have our torus!
1402    * Special case for the last node (if the rows and columns are not equal), connect
1403    * to the first in the row to maintain topology.
1404    */
1405   for (i = 0; i < pg->total; i++)
1406     {
1407       /* First connect to the node to the right */
1408       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1409         nodeToConnect = i + 1;
1410       else if (i + 1 == pg->total)
1411         nodeToConnect = rows * cols - cols;
1412       else
1413         nodeToConnect = i - cols + 1;
1414
1415       connect_attempts += proc (pg, i, nodeToConnect);
1416
1417       if (i < cols)
1418         nodeToConnect = (rows * cols) - cols + i;
1419       else
1420         nodeToConnect = i - cols;
1421
1422       if (nodeToConnect < pg->total)
1423         connect_attempts += proc (pg, i, nodeToConnect);
1424     }
1425   natLog = log (pg->total);
1426 #if VERBOSE_TESTING > 2
1427   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1428               _("natural log of %d is %d, will run %d iterations\n"),
1429              pg->total, natLog, (int) (natLog * percentage));
1430   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
1431 #endif
1432   smallWorldConnections = 0;
1433   small_world_it = (unsigned int)(natLog * percentage);
1434   GNUNET_assert(small_world_it > 0 && small_world_it < (unsigned int)-1);
1435   for (i = 0; i < small_world_it; i++)
1436     {
1437       for (j = 0; j < pg->total; j++)
1438         {
1439           /* Determine the row and column of node at position j on the 2d torus */
1440           node1Row = j / cols;
1441           node1Col = j - (node1Row * cols);
1442           for (k = 0; k < pg->total; k++)
1443             {
1444               /* Determine the row and column of node at position k on the 2d torus */
1445               node2Row = k / cols;
1446               node2Col = k - (node2Row * cols);
1447               /* Simple Cartesian distance */
1448               distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
1449               if (distance > 1)
1450                 {
1451                   /* Calculate probability as 1 over the square of the distance */
1452                   probability = 1.0 / (distance * distance);
1453                   /* Choose a random value between 0 and 1 */
1454                   random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1455                                                               UINT64_MAX)) / ( (double) UINT64_MAX);
1456                   /* If random < probability, then connect the two nodes */
1457                   if (random < probability)
1458                     smallWorldConnections += proc (pg, j, k);
1459
1460                 }
1461             }
1462         }
1463     }
1464   connect_attempts += smallWorldConnections;
1465 #if VERBOSE_TESTING > 2
1466           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1467                       _("Total connections added for small world: %d!\n"),
1468                       smallWorldConnections);
1469 #endif
1470   return connect_attempts;
1471 }
1472
1473 /**
1474  * Create a topology given a peer group (set of running peers)
1475  * and a connection processor.
1476  *
1477  * @param pg the peergroup to create the topology on
1478  * @param proc the connection processor to call to actually set
1479  *        up connections between two peers
1480  *
1481  * @return the number of connections that were set up
1482  *
1483  */
1484 static int
1485 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1486 {
1487   double temp_rand;
1488   unsigned int outer_count;
1489   unsigned int inner_count;
1490   int connect_attempts;
1491   double probability;
1492   char *p_string;
1493
1494   probability = 0.5; /* FIXME: default percentage? */
1495   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1496                                                          "TESTING",
1497                                                          "PROBABILITY",
1498                                                          &p_string))
1499     {
1500       if (sscanf(p_string, "%lf", &probability) != 1)
1501         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1502                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1503                     p_string,
1504                     "PROBABILITY",
1505                     "TESTING");
1506       GNUNET_free (p_string);
1507     }
1508   connect_attempts = 0;
1509   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1510     {
1511       for (inner_count = outer_count + 1; inner_count < pg->total;
1512            inner_count++)
1513         {
1514           temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1515                                                          UINT64_MAX)) / ( (double) UINT64_MAX);
1516 #if VERBOSE_TESTING
1517           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1518                       _("rand is %f probability is %f\n"), temp_rand,
1519                       probability);
1520 #endif
1521           if (temp_rand < probability)
1522             {
1523               connect_attempts += proc (pg, outer_count, inner_count);
1524             }
1525         }
1526     }
1527
1528   return connect_attempts;
1529 }
1530
1531 /**
1532  * Create a topology given a peer group (set of running peers)
1533  * and a connection processor.
1534  *
1535  * @param pg the peergroup to create the topology on
1536  * @param proc the connection processor to call to actually set
1537  *        up connections between two peers
1538  *
1539  * @return the number of connections that were set up
1540  *
1541  */
1542 static int
1543 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1544 {
1545   unsigned int i;
1546   unsigned int square;
1547   unsigned int rows;
1548   unsigned int cols;
1549   unsigned int toggle = 1;
1550   unsigned int nodeToConnect;
1551   int connect_attempts;
1552
1553   connect_attempts = 0;
1554
1555   square = floor (sqrt (pg->total));
1556   rows = square;
1557   cols = square;
1558
1559   if (square * square != pg->total)
1560     {
1561       while (rows * cols < pg->total)
1562         {
1563           if (toggle % 2 == 0)
1564             rows++;
1565           else
1566             cols++;
1567
1568           toggle++;
1569         }
1570     }
1571 #if VERBOSE_TESTING
1572       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1573                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1574                   rows, cols);
1575 #endif
1576   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1577    * to the node to its right and above.  Once this is over, we'll have our torus!
1578    * Special case for the last node (if the rows and columns are not equal), connect
1579    * to the first in the row to maintain topology.
1580    */
1581   for (i = 0; i < pg->total; i++)
1582     {
1583       /* First connect to the node to the right */
1584       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1585         nodeToConnect = i + 1;
1586       else if (i + 1 == pg->total)
1587         nodeToConnect = rows * cols - cols;
1588       else
1589         nodeToConnect = i - cols + 1;
1590 #if VERBOSE_TESTING
1591           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1592                       "Connecting peer %d to peer %d\n",
1593                       i, nodeToConnect);
1594 #endif
1595       connect_attempts += proc(pg, i, nodeToConnect);
1596
1597       /* Second connect to the node immediately above */
1598       if (i < cols)
1599         nodeToConnect = (rows * cols) - cols + i;
1600       else
1601         nodeToConnect = i - cols;
1602
1603       if (nodeToConnect < pg->total)
1604         {
1605 #if VERBOSE_TESTING
1606           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1607                       "Connecting peer %d to peer %d\n",
1608                       i, nodeToConnect);
1609 #endif
1610           connect_attempts += proc(pg, i, nodeToConnect);
1611         }
1612
1613     }
1614
1615   return connect_attempts;
1616 }
1617
1618
1619 /**
1620  * Create a topology given a peer group (set of running peers)
1621  * and a connection processor.
1622  *
1623  * @param pg the peergroup to create the topology on
1624  * @param proc the connection processor to call to actually set
1625  *        up connections between two peers
1626  *
1627  * @return the number of connections that were set up
1628  *
1629  */
1630 static int
1631 create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1632 {
1633   unsigned int outer_count;
1634   unsigned int inner_count;
1635   int connect_attempts;
1636
1637   connect_attempts = 0;
1638
1639   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1640     {
1641       for (inner_count = outer_count + 1; inner_count < pg->total;
1642            inner_count++)
1643         {
1644 #if VERBOSE_TESTING
1645           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1646                       "Connecting peer %d to peer %d\n",
1647                       outer_count, inner_count);
1648 #endif
1649           connect_attempts += proc(pg, outer_count, inner_count);
1650         }
1651     }
1652
1653   return connect_attempts;
1654 }
1655
1656 /**
1657  * Create a topology given a peer group (set of running peers)
1658  * and a connection processor.
1659  *
1660  * @param pg the peergroup to create the topology on
1661  * @param proc the connection processor to call to actually set
1662  *        up connections between two peers
1663  *
1664  * @return the number of connections that were set up
1665  *
1666  */
1667 static int
1668 create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1669 {
1670   unsigned int count;
1671   int connect_attempts;
1672
1673   connect_attempts = 0;
1674
1675   /* Connect each peer to the next highest numbered peer */
1676   for (count = 0; count < pg->total - 1; count++)
1677     {
1678 #if VERBOSE_TESTING
1679           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1680                       "Connecting peer %d to peer %d\n",
1681                       count, count + 1);
1682 #endif
1683       connect_attempts += proc(pg, count, count + 1);
1684     }
1685
1686   return connect_attempts;
1687 }
1688
1689 /**
1690  * Create a topology given a peer group (set of running peers)
1691  * and a connection processor.
1692  *
1693  * @param pg the peergroup to create the topology on
1694  * @param proc the connection processor to call to actually set
1695  *        up connections between two peers
1696  *
1697  * @return the number of connections that were set up
1698  *
1699  */
1700 static int
1701 create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1702 {
1703   unsigned int count;
1704   int connect_attempts;
1705
1706   connect_attempts = 0;
1707
1708   /* Connect each peer to the next highest numbered peer */
1709   for (count = 0; count < pg->total - 1; count++)
1710     {
1711 #if VERBOSE_TESTING
1712           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1713                       "Connecting peer %d to peer %d\n",
1714                       count, count + 1);
1715 #endif
1716       connect_attempts += proc(pg, count, count + 1);
1717     }
1718
1719   /* Connect the last peer to the first peer */
1720   connect_attempts += proc(pg, pg->total - 1, 0);
1721
1722   return connect_attempts;
1723 }
1724
1725
1726 /**
1727  * Iterator for writing friends of a peer to a file.
1728  *
1729  * @param cls closure, an open writable file handle
1730  * @param key the key the daemon was stored under
1731  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1732  *
1733  * @return GNUNET_YES to continue iteration
1734  *
1735  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1736  *       with a single file_iterator that takes a closure which contains
1737  *       the prefix to write before the peer.  Then this could be used
1738  *       for blacklisting multiple transports and writing the friend
1739  *       file.  I'm sure *someone* will complain loudly about other
1740  *       things that negate these functions even existing so no point in
1741  *       "fixing" now.
1742  */
1743 static int
1744 friend_file_iterator (void *cls,
1745                   const GNUNET_HashCode * key,
1746                   void *value)
1747 {
1748   FILE *temp_friend_handle = cls;
1749   struct GNUNET_TESTING_Daemon *peer = value;
1750   struct GNUNET_PeerIdentity *temppeer;
1751   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1752
1753   temppeer = &peer->id;
1754   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1755   fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
1756
1757   return GNUNET_YES;
1758 }
1759
1760 struct BlacklistContext
1761 {
1762   /*
1763    * The (open) file handle to write to
1764    */
1765   FILE *temp_file_handle;
1766
1767   /*
1768    * The transport that this peer will be blacklisted on.
1769    */
1770   char *transport;
1771 };
1772
1773 /**
1774  * Iterator for writing blacklist data to appropriate files.
1775  *
1776  * @param cls closure, an open writable file handle
1777  * @param key the key the daemon was stored under
1778  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1779  *
1780  * @return GNUNET_YES to continue iteration
1781  */
1782 static int
1783 blacklist_file_iterator (void *cls,
1784                          const GNUNET_HashCode * key,
1785                          void *value)
1786 {
1787   struct BlacklistContext *blacklist_ctx = cls;
1788   //FILE *temp_blacklist_handle = cls;
1789   struct GNUNET_TESTING_Daemon *peer = value;
1790   struct GNUNET_PeerIdentity *temppeer;
1791   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1792
1793   temppeer = &peer->id;
1794   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1795   fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
1796
1797   return GNUNET_YES;
1798 }
1799
1800 /*
1801  * Create the friend files based on the PeerConnection's
1802  * of each peer in the peer group, and copy the files
1803  * to the appropriate place
1804  *
1805  * @param pg the peer group we are dealing with
1806  */
1807 static int
1808 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
1809 {
1810   FILE *temp_friend_handle;
1811   unsigned int pg_iter;
1812   char *temp_service_path;
1813   pid_t *pidarr;
1814   char *arg;
1815   char * mytemp;
1816   enum GNUNET_OS_ProcessStatusType type;
1817   unsigned long return_code;
1818   int count;
1819   int ret;
1820   int max_wait = 10;
1821
1822   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1823   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1824     {
1825       mytemp = GNUNET_DISK_mktemp("friends");
1826       GNUNET_assert(mytemp != NULL);
1827       temp_friend_handle = fopen (mytemp, "wt");
1828       GNUNET_assert(temp_friend_handle != NULL);
1829       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
1830       fclose(temp_friend_handle);
1831
1832       if (GNUNET_OK !=
1833           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1834         {
1835           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1836                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1837                       "SERVICEHOME",
1838                       "PATHS");
1839           if (UNLINK (mytemp) != 0)
1840             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1841           GNUNET_free (mytemp);
1842           break;
1843         }
1844
1845       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1846         {
1847           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
1848           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1849                                          "mv", mytemp, arg, NULL);
1850 #if VERBOSE_TESTING
1851           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1852                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1853 #endif
1854
1855           GNUNET_free(arg);
1856         }
1857       else /* Remote, scp the file to the correct place */
1858         {
1859           if (NULL != pg->peers[pg_iter].daemon->username)
1860             GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1861           else
1862             GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1863           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1864                                          "scp", mytemp, arg, NULL);
1865
1866 #if VERBOSE_TESTING
1867           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1868                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1869 #endif
1870           GNUNET_free(arg);
1871         }
1872       GNUNET_free (temp_service_path);
1873       GNUNET_free (mytemp);
1874     }
1875
1876   count = 0;
1877   ret = GNUNET_SYSERR;
1878   while ((count < max_wait) && (ret != GNUNET_OK))
1879     {
1880       ret = GNUNET_OK;
1881       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1882         {
1883 #if VERBOSE_TESTING
1884           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1885                       _("Checking copy status of file %d\n"), pg_iter);
1886 #endif
1887           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1888             {
1889               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1890                 {
1891                   ret = GNUNET_SYSERR;
1892                 }
1893               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1894                 {
1895                   ret = GNUNET_SYSERR;
1896                 }
1897               else
1898                 {
1899                   pidarr[pg_iter] = 0;
1900 #if VERBOSE_TESTING
1901             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1902                       _("File %d copied\n"), pg_iter);
1903 #endif
1904                 }
1905             }
1906         }
1907       count++;
1908       if (ret == GNUNET_SYSERR)
1909         {
1910           /* FIXME: why sleep here? -CG */
1911           sleep(1);
1912         }
1913     }
1914
1915 #if VERBOSE_TESTING
1916     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1917                 _("Finished copying all friend files!\n"));
1918 #endif
1919   GNUNET_free(pidarr);
1920   return ret;
1921 }
1922
1923
1924 /*
1925  * Create the blacklist files based on the PeerConnection's
1926  * of each peer in the peer group, and copy the files
1927  * to the appropriate place.
1928  *
1929  * @param pg the peer group we are dealing with
1930  * @param transports space delimited list of transports to blacklist
1931  */
1932 static int
1933 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
1934 {
1935   FILE *temp_file_handle;
1936   static struct BlacklistContext blacklist_ctx;
1937   unsigned int pg_iter;
1938   char *temp_service_path;
1939   pid_t *pidarr;
1940   char *arg;
1941   char *mytemp;
1942   enum GNUNET_OS_ProcessStatusType type;
1943   unsigned long return_code;
1944   int count;
1945   int ret;
1946   int max_wait = 10;
1947   int transport_len;
1948   unsigned int i;
1949   char *pos;
1950   char *temp_transports;
1951
1952   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1953   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1954     {
1955       mytemp = GNUNET_DISK_mktemp("blacklist");
1956       GNUNET_assert(mytemp != NULL);
1957       temp_file_handle = fopen (mytemp, "wt");
1958       GNUNET_assert(temp_file_handle != NULL);
1959       temp_transports = GNUNET_strdup(transports);
1960       blacklist_ctx.temp_file_handle = temp_file_handle;
1961       transport_len = strlen(temp_transports) + 1;
1962       pos = NULL;
1963
1964       for (i = 0; i < transport_len; i++)
1965       {
1966         if ((temp_transports[i] == ' ') && (pos == NULL))
1967           continue; /* At start of string (whitespace) */
1968         else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
1969         {
1970           temp_transports[i] = '\0';
1971           blacklist_ctx.transport = pos;
1972           GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
1973           pos = NULL;
1974         } /* At beginning of actual string */
1975         else if (pos == NULL)
1976         {
1977           pos = &temp_transports[i];
1978         }
1979       }
1980
1981       GNUNET_free (temp_transports);
1982       fclose(temp_file_handle);
1983
1984       if (GNUNET_OK !=
1985           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1986         {
1987           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1988                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1989                       "SERVICEHOME",
1990                       "PATHS");
1991           if (UNLINK (mytemp) != 0)
1992             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1993           GNUNET_free (mytemp);
1994           break;
1995         }
1996
1997       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1998         {
1999           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
2000           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2001                                          "mv", mytemp, arg, NULL);
2002 #if VERBOSE_TESTING
2003           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2004                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2005 #endif
2006
2007           GNUNET_free(arg);
2008         }
2009       else /* Remote, scp the file to the correct place */
2010         {
2011           if (NULL != pg->peers[pg_iter].daemon->username)
2012             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
2013           else
2014             GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
2015           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
2016                                          "scp", mytemp, arg, NULL);
2017
2018 #if VERBOSE_TESTING
2019           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2020                       _("Copying file with command scp %s %s\n"), mytemp, arg);
2021 #endif
2022           GNUNET_free(arg);
2023         }
2024       GNUNET_free (temp_service_path);
2025       GNUNET_free (mytemp);
2026     }
2027
2028   count = 0;
2029   ret = GNUNET_SYSERR;
2030   while ((count < max_wait) && (ret != GNUNET_OK))
2031     {
2032       ret = GNUNET_OK;
2033       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2034         {
2035 #if VERBOSE_TESTING
2036           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2037                       _("Checking copy status of file %d\n"), pg_iter);
2038 #endif
2039           if (pidarr[pg_iter] != 0) /* Check for already completed! */
2040             {
2041               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
2042                 {
2043                   ret = GNUNET_SYSERR;
2044                 }
2045               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
2046                 {
2047                   ret = GNUNET_SYSERR;
2048                 }
2049               else
2050                 {
2051                   pidarr[pg_iter] = 0;
2052 #if VERBOSE_TESTING
2053             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2054                       _("File %d copied\n"), pg_iter);
2055 #endif
2056                 }
2057             }
2058         }
2059       count++;
2060       if (ret == GNUNET_SYSERR)
2061         {
2062           /* FIXME: why sleep here? -CG */
2063           sleep(1);
2064         }
2065     }
2066
2067 #if VERBOSE_TESTING
2068     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2069                 _("Finished copying all blacklist files!\n"));
2070 #endif
2071   GNUNET_free(pidarr);
2072   return ret;
2073 }
2074
2075
2076 /**
2077  * Internal notification of a connection, kept so that we can ensure some connections
2078  * happen instead of flooding all testing daemons with requests to connect.
2079  */
2080 static void internal_connect_notify (void *cls,
2081                                      const struct GNUNET_PeerIdentity *first,
2082                                      const struct GNUNET_PeerIdentity *second,
2083                                      uint32_t distance,
2084                                      const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2085                                      const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2086                                      struct GNUNET_TESTING_Daemon *first_daemon,
2087                                      struct GNUNET_TESTING_Daemon *second_daemon,
2088                                      const char *emsg)
2089 {
2090   struct GNUNET_TESTING_PeerGroup *pg = cls;
2091   outstanding_connects--;
2092
2093   pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
2094 }
2095
2096
2097 /**
2098  * Either delay a connection (because there are too many outstanding)
2099  * or schedule it for right now.
2100  *
2101  * @param cls a connection context
2102  * @param tc the task runtime context
2103  */
2104 static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2105 {
2106   struct ConnectContext *connect_context = cls;
2107
2108   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2109     return;
2110
2111   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2112     {
2113 #if VERBOSE_TESTING > 2
2114           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2115                       _("Delaying connect, we have too many outstanding connections!\n"));
2116 #endif
2117       GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
2118     }
2119   else
2120     {
2121 #if VERBOSE_TESTING > 2
2122           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2123                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2124 #endif
2125       outstanding_connects++;
2126       GNUNET_TESTING_daemons_connect (connect_context->first,
2127                                       connect_context->second,
2128                                       CONNECT_TIMEOUT,
2129                                       CONNECT_ATTEMPTS,
2130                                       &internal_connect_notify,
2131                                       connect_context->pg);
2132       GNUNET_free(connect_context);
2133     }
2134 }
2135
2136
2137 /**
2138  * Iterator for actually scheduling connections to be created
2139  * between two peers.
2140  *
2141  * @param cls closure, a GNUNET_TESTING_Daemon
2142  * @param key the key the second Daemon was stored under
2143  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2144  *
2145  * @return GNUNET_YES to continue iteration
2146  */
2147 static int
2148 connect_iterator (void *cls,
2149                   const GNUNET_HashCode * key,
2150                   void *value)
2151 {
2152   struct PeerData *first = cls;
2153   struct GNUNET_TESTING_Daemon *second = value;
2154   struct ConnectContext *connect_context;
2155
2156   connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2157   connect_context->pg = first->pg;
2158   connect_context->first = first->daemon;
2159   connect_context->second = second;
2160   GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
2161
2162   return GNUNET_YES;
2163 }
2164
2165
2166 /**
2167  * Iterator for copying all entries in the allowed hashmap to the
2168  * connect hashmap.
2169  *
2170  * @param cls closure, a GNUNET_TESTING_Daemon
2171  * @param key the key the second Daemon was stored under
2172  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2173  *
2174  * @return GNUNET_YES to continue iteration
2175  */
2176 static int
2177 copy_topology_iterator (void *cls,
2178                   const GNUNET_HashCode * key,
2179                   void *value)
2180 {
2181   struct PeerData *first = cls;
2182
2183   GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2184
2185   return GNUNET_YES;
2186 }
2187
2188 /**
2189  * Make the peers to connect the same as those that are allowed to be
2190  * connected.
2191  *
2192  * @param pg the peer group
2193  */
2194 static int
2195 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2196 {
2197   unsigned int pg_iter;
2198   int ret;
2199   int total;
2200
2201   total = 0;
2202   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2203     {
2204       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
2205       if (GNUNET_SYSERR == ret)
2206         return GNUNET_SYSERR;
2207
2208       total = total + ret;
2209     }
2210
2211   return total;
2212 }
2213
2214
2215 /**
2216  * Connect the topology as specified by the PeerConnection's
2217  * of each peer in the peer group
2218  *
2219  * @param pg the peer group we are dealing with
2220  * @return the number of connections that will be attempted
2221  */
2222 static int
2223 connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
2224 {
2225   unsigned int pg_iter;
2226   int ret;
2227   int total;
2228 #if OLD
2229   struct PeerConnection *connection_iter;
2230   struct ConnectContext *connect_context;
2231 #endif
2232
2233   total = 0;
2234   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2235     {
2236       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
2237       if (GNUNET_SYSERR == ret)
2238         return GNUNET_SYSERR;
2239
2240       total = total + ret;
2241
2242 #if OLD
2243       connection_iter = ;
2244       while (connection_iter != NULL)
2245         {
2246           connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2247           connect_context->pg = pg;
2248           connect_context->first = ;
2249           connect_context->second = connection_iter->daemon;
2250           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
2251           connection_iter = connection_iter->next;
2252         }
2253 #endif
2254     }
2255   return total;
2256 }
2257
2258
2259 /**
2260  * Takes a peer group and creates a topology based on the
2261  * one specified.  Creates a topology means generates friend
2262  * files for the peers so they can only connect to those allowed
2263  * by the topology.  This will only have an effect once peers
2264  * are started if the FRIENDS_ONLY option is set in the base
2265  * config.  Also takes an optional restrict topology which
2266  * disallows connections based on a particular transport
2267  * UNLESS they are specified in the restricted topology.
2268  *
2269  * @param pg the peer group struct representing the running peers
2270  * @param topology which topology to connect the peers in
2271  * @param restrict_topology allow only direct TCP connections in this topology
2272  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2273  * @param restrict_transports space delimited list of transports to blacklist
2274  *                            to create restricted topology
2275  *
2276  * @return the maximum number of connections were all allowed peers
2277  *         connected to each other
2278  */
2279 int
2280 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2281                                 enum GNUNET_TESTING_Topology topology,
2282                                 enum GNUNET_TESTING_Topology restrict_topology,
2283                                 char *restrict_transports)
2284 {
2285   int ret;
2286   int num_connections;
2287   int unblacklisted_connections;
2288
2289   GNUNET_assert (pg->notify_connection != NULL);
2290
2291   switch (topology)
2292     {
2293     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2294 #if VERBOSE_TESTING
2295       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2296                   _("Creating clique topology\n"));
2297 #endif
2298       num_connections = create_clique (pg, &add_allowed_connections);
2299       break;
2300     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2301 #if VERBOSE_TESTING
2302       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2303                   _("Creating small world (ring) topology\n"));
2304 #endif
2305       num_connections = create_small_world_ring (pg, &add_allowed_connections);
2306       break;
2307     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2308 #if VERBOSE_TESTING
2309       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2310                   _("Creating small world (2d-torus) topology\n"));
2311 #endif
2312       num_connections = create_small_world (pg, &add_allowed_connections);
2313       break;
2314     case GNUNET_TESTING_TOPOLOGY_RING:
2315 #if VERBOSE_TESTING
2316       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2317                   _("Creating ring topology\n"));
2318 #endif
2319       num_connections = create_ring (pg, &add_allowed_connections);
2320       break;
2321     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2322 #if VERBOSE_TESTING
2323       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2324                   _("Creating 2d torus topology\n"));
2325 #endif
2326       num_connections = create_2d_torus (pg, &add_allowed_connections);
2327       break;
2328     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2329 #if VERBOSE_TESTING
2330       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2331                   _("Creating Erdos-Renyi topology\n"));
2332 #endif
2333       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2334       break;
2335     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2336 #if VERBOSE_TESTING
2337       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2338                   _("Creating InterNAT topology\n"));
2339 #endif
2340       num_connections = create_nated_internet (pg, &add_allowed_connections);
2341       break;
2342     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2343 #if VERBOSE_TESTING
2344       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2345                   _("Creating Scale Free topology\n"));
2346 #endif
2347       num_connections = create_scale_free (pg, &add_allowed_connections);
2348       break;
2349     case GNUNET_TESTING_TOPOLOGY_LINE:
2350 #if VERBOSE_TESTING
2351       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2352                   _("Creating straight line topology\n"));
2353 #endif
2354       num_connections = create_line (pg, &add_allowed_connections);
2355       break;
2356     case GNUNET_TESTING_TOPOLOGY_NONE:
2357 #if VERBOSE_TESTING
2358       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2359                   _("Creating no allowed topology (all peers can connect at core level)\n"));
2360 #endif
2361       num_connections = 0;
2362       break;
2363     default:
2364       num_connections = 0;
2365       break;
2366     }
2367
2368   if (num_connections < 0)
2369     return GNUNET_SYSERR;
2370
2371   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2372     {
2373       ret = create_and_copy_friend_files(pg);
2374       if (ret != GNUNET_OK)
2375         {
2376 #if VERBOSE_TESTING
2377           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2378                       _("Failed during friend file copying!\n"));
2379 #endif
2380           return GNUNET_SYSERR;
2381         }
2382       else
2383         {
2384 #if VERBOSE_TESTING
2385               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2386                           _("Friend files created/copied successfully!\n"));
2387 #endif
2388         }
2389     }
2390
2391   /* Use the create clique method to initially set all connections as blacklisted. */
2392   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2393     create_clique (pg, &blacklist_connections);
2394   unblacklisted_connections = 0;
2395   /* Un-blacklist connections as per the topology specified */
2396   switch (restrict_topology)
2397     {
2398     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2399 #if VERBOSE_TESTING
2400       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2401                   _("Blacklisting all but clique topology\n"));
2402 #endif
2403       unblacklisted_connections = create_clique (pg, &unblacklist_connections);
2404       break;
2405     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2406 #if VERBOSE_TESTING
2407       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2408                   _("Blacklisting all but small world (ring) topology\n"));
2409 #endif
2410       unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
2411       break;
2412     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2413 #if VERBOSE_TESTING
2414       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2415                   _("Blacklisting all but small world (2d-torus) topology\n"));
2416 #endif
2417       unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
2418       break;
2419     case GNUNET_TESTING_TOPOLOGY_RING:
2420 #if VERBOSE_TESTING
2421       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2422                   _("Blacklisting all but ring topology\n"));
2423 #endif
2424       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2425       break;
2426     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2427 #if VERBOSE_TESTING
2428       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2429                   _("Blacklisting all but 2d torus topology\n"));
2430 #endif
2431       unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
2432       break;
2433     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2434 #if VERBOSE_TESTING
2435       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2436                   _("Blacklisting all but Erdos-Renyi topology\n"));
2437 #endif
2438       unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
2439       break;
2440     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2441 #if VERBOSE_TESTING
2442       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2443                   _("Blacklisting all but InterNAT topology\n"));
2444 #endif
2445       unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
2446       break;
2447     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2448 #if VERBOSE_TESTING
2449       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2450                   _("Blacklisting all but Scale Free topology\n"));
2451 #endif
2452       unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
2453       break;
2454     case GNUNET_TESTING_TOPOLOGY_LINE:
2455 #if VERBOSE_TESTING
2456       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2457                   _("Blacklisting all but straight line topology\n"));
2458 #endif
2459       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2460       break;
2461     case GNUNET_TESTING_TOPOLOGY_NONE:
2462 #if VERBOSE_TESTING
2463       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2464                   _("Creating no blacklist topology (all peers can connect at transport level)\n"));
2465 #endif
2466     default:
2467       break;
2468     }
2469
2470   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2471   {
2472     ret = create_and_copy_blacklist_files(pg, restrict_transports);
2473     if (ret != GNUNET_OK)
2474       {
2475 #if VERBOSE_TESTING
2476         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2477                     _("Failed during blacklist file copying!\n"));
2478 #endif
2479         return GNUNET_SYSERR;
2480       }
2481     else
2482       {
2483 #if VERBOSE_TESTING
2484         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2485                     _("Blacklist files created/copied successfully!\n"));
2486 #endif
2487       }
2488   }
2489   return num_connections;
2490 }
2491
2492 struct RandomContext
2493 {
2494   /**
2495    * The peergroup
2496    */
2497   struct GNUNET_TESTING_PeerGroup *pg;
2498
2499   /**
2500    * uid of the first peer
2501    */
2502   uint32_t first_uid;
2503
2504   /**
2505    * Peer data for first peer.
2506    */
2507   struct PeerData *first;
2508
2509   /**
2510    * Random percentage to use
2511    */
2512   double percentage;
2513 };
2514
2515 struct MinimumContext
2516 {
2517   /**
2518    * The peergroup
2519    */
2520   struct GNUNET_TESTING_PeerGroup *pg;
2521
2522   /**
2523    * uid of the first peer
2524    */
2525   uint32_t first_uid;
2526
2527   /**
2528    * Peer data for first peer.
2529    */
2530   struct PeerData *first;
2531
2532   /**
2533    * Number of conns per peer
2534    */
2535   unsigned int num_to_add;
2536
2537   /**
2538    * Permuted array of all possible connections.  Only add the Nth
2539    * peer if it's in the Nth position.
2540    */
2541   unsigned int *pg_array;
2542
2543   /**
2544    * What number is the current element we are iterating over?
2545    */
2546   unsigned int current;
2547 };
2548
2549 struct DFSContext
2550 {
2551   /**
2552    * The peergroup
2553    */
2554   struct GNUNET_TESTING_PeerGroup *pg;
2555
2556   /**
2557    * uid of the first peer
2558    */
2559   uint32_t first_uid;
2560
2561   /**
2562    * uid of the second peer
2563    */
2564   uint32_t second_uid;
2565
2566   /**
2567    * Peer data for first peer.
2568    */
2569   struct PeerData *first;
2570
2571   /**
2572    * Which peer has been chosen as the one to add?
2573    */
2574   unsigned int chosen;
2575
2576   /**
2577    * What number is the current element we are iterating over?
2578    */
2579   unsigned int current;
2580 };
2581
2582 /**
2583  * Iterator for choosing random peers to connect.
2584  *
2585  * @param cls closure, a RandomContext
2586  * @param key the key the second Daemon was stored under
2587  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2588  *
2589  * @return GNUNET_YES to continue iteration
2590  */
2591 static int
2592 random_connect_iterator (void *cls,
2593                   const GNUNET_HashCode * key,
2594                   void *value)
2595 {
2596   struct RandomContext *random_ctx = cls;
2597   double random_number;
2598   uint32_t second_pos;
2599   GNUNET_HashCode first_hash;
2600   random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
2601                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
2602   if (random_number < random_ctx->percentage)
2603   {
2604     GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2605   }
2606   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2607   uid_from_hash(key, &second_pos);
2608   hash_from_uid(random_ctx->first_uid, &first_hash);
2609   GNUNET_assert(random_ctx->pg->total > second_pos);
2610   GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
2611
2612   return GNUNET_YES;
2613 }
2614
2615 /**
2616  * Iterator for adding at least X peers to a peers connection set.
2617  *
2618  * @param cls closure, MinimumContext
2619  * @param key the key the second Daemon was stored under
2620  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2621  *
2622  * @return GNUNET_YES to continue iteration
2623  */
2624 static int
2625 minimum_connect_iterator (void *cls,
2626                   const GNUNET_HashCode * key,
2627                   void *value)
2628 {
2629   struct MinimumContext *min_ctx = cls;
2630   uint32_t second_pos;
2631   GNUNET_HashCode first_hash;
2632   unsigned int i;
2633
2634   if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2635   {
2636     for (i = 0; i < min_ctx->num_to_add; i++)
2637     {
2638       if (min_ctx->pg_array[i] == min_ctx->current)
2639       {
2640         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2641         uid_from_hash(key, &second_pos);
2642         hash_from_uid(min_ctx->first_uid, &first_hash);
2643         GNUNET_assert(min_ctx->pg->total > second_pos);
2644         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2645         /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2646         GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
2647       }
2648     }
2649     min_ctx->current++;
2650     return GNUNET_YES;
2651   }
2652   else
2653     return GNUNET_NO; /* We can stop iterating, we have enough peers! */
2654
2655 }
2656
2657
2658 /**
2659  * Iterator for adding peers to a connection set based on a depth first search.
2660  *
2661  * @param cls closure, MinimumContext
2662  * @param key the key the second daemon was stored under
2663  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2664  *
2665  * @return GNUNET_YES to continue iteration
2666  */
2667 static int
2668 dfs_connect_iterator (void *cls,
2669                   const GNUNET_HashCode * key,
2670                   void *value)
2671 {
2672   struct DFSContext *dfs_ctx = cls;
2673   GNUNET_HashCode first_hash;
2674
2675   if (dfs_ctx->current == dfs_ctx->chosen)
2676     {
2677       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2678       uid_from_hash(key, &dfs_ctx->second_uid);
2679       hash_from_uid(dfs_ctx->first_uid, &first_hash);
2680       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2681       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
2682       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
2683       return GNUNET_NO; /* We have found our peer, don't iterate more */
2684     }
2685
2686   dfs_ctx->current++;
2687   return GNUNET_YES;
2688 }
2689
2690
2691 /**
2692  * From the set of connections possible, choose percentage percent of connections
2693  * to actually connect.
2694  *
2695  * @param pg the peergroup we are dealing with
2696  * @param percentage what percent of total connections to make
2697  */
2698 void
2699 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
2700 {
2701   struct RandomContext random_ctx;
2702   uint32_t pg_iter;
2703
2704   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2705     {
2706       random_ctx.first_uid = pg_iter;
2707       random_ctx.first = &pg->peers[pg_iter];
2708       random_ctx.percentage = percentage;
2709       random_ctx.pg = pg;
2710       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
2711       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
2712       /* Now remove the old connections */
2713       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2714       /* And replace with the random set */
2715       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2716     }
2717 }
2718
2719 /**
2720  * From the set of connections possible, choose at least num connections per
2721  * peer.
2722  *
2723  * @param pg the peergroup we are dealing with
2724  * @param num how many connections at least should each peer have (if possible)?
2725  */
2726 static void
2727 choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2728 {
2729   struct MinimumContext minimum_ctx;
2730   uint32_t pg_iter;
2731
2732   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2733    {
2734      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2735    }
2736
2737   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2738     {
2739       minimum_ctx.first_uid = pg_iter;
2740       minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK,
2741                                                           GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
2742       minimum_ctx.first = &pg->peers[pg_iter];
2743       minimum_ctx.pg = pg;
2744       minimum_ctx.num_to_add = num;
2745       minimum_ctx.current = 0;
2746       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
2747                                             &minimum_connect_iterator,
2748                                             &minimum_ctx);
2749     }
2750
2751   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2752     {
2753       /* Remove the "old" connections */
2754       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2755       /* And replace with the working set */
2756       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2757     }
2758
2759 }
2760
2761
2762 static unsigned int
2763 count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
2764 {
2765   unsigned int count;
2766   unsigned int pg_iter;
2767
2768   count = 0;
2769
2770   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2771     {
2772       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2773     }
2774
2775   return count;
2776 }
2777
2778
2779 static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
2780 {
2781   unsigned int count;
2782   unsigned int pg_iter;
2783
2784   count = 0;
2785
2786   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2787     {
2788       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
2789     }
2790
2791   return count;
2792 }
2793
2794 /**
2795  * From the set of connections possible, choose at least num connections per
2796  * peer based on depth first traversal of peer connections.  If DFS leaves
2797  * peers unconnected, ensure those peers get connections.
2798  *
2799  * @param pg the peergroup we are dealing with
2800  * @param num how many connections at least should each peer have (if possible)?
2801  */
2802 void
2803 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2804 {
2805   struct DFSContext dfs_ctx;
2806   uint32_t pg_iter;
2807   uint32_t dfs_count;
2808   uint32_t starting_peer;
2809   uint32_t least_connections;
2810   GNUNET_HashCode second_hash;
2811
2812   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2813     {
2814       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2815     }
2816
2817   starting_peer = 0;
2818   dfs_count = 0;
2819   while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
2820     {
2821       if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
2822         {
2823           least_connections = -1; /* Set to very high number */
2824           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2825             {
2826               if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
2827                 {
2828                   starting_peer = pg_iter;
2829                   least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2830                 }
2831             }
2832         }
2833
2834       if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
2835         {
2836           dfs_count = 0;
2837           continue;
2838         }
2839
2840       /* Choose a random peer from the chosen peers set of connections to add */
2841       dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
2842       dfs_ctx.first_uid = starting_peer;
2843       dfs_ctx.first = &pg->peers[starting_peer];
2844       dfs_ctx.pg = pg;
2845       dfs_ctx.current = 0;
2846
2847       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
2848       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
2849       hash_from_uid(dfs_ctx.second_uid, &second_hash);
2850       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
2851       starting_peer = dfs_ctx.second_uid;
2852     }
2853
2854   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2855     {
2856       /* Remove the "old" connections */
2857       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2858       /* And replace with the working set */
2859       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2860     }
2861 }
2862
2863 /**
2864  * Internal callback for topology information for a particular peer.
2865  */
2866 static void
2867 internal_topology_callback(void *cls,
2868                            const struct GNUNET_PeerIdentity *peer,
2869                            struct GNUNET_TIME_Relative latency, uint32_t distance)
2870 {
2871   struct CoreContext *core_ctx = cls;
2872   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
2873
2874   if (peer == NULL) /* Either finished, or something went wrong */
2875     {
2876       iter_ctx->completed++;
2877       iter_ctx->connected--;
2878       /* One core context allocated per iteration, must free! */
2879       GNUNET_free(core_ctx);
2880     }
2881   else
2882     {
2883       iter_ctx->topology_cb(iter_ctx->cls, &core_ctx->daemon->id, peer, latency, distance, NULL);
2884     }
2885
2886   if (iter_ctx->completed == iter_ctx->total)
2887     {
2888       iter_ctx->topology_cb(iter_ctx->cls, NULL, NULL, GNUNET_TIME_relative_get_zero(), 0, NULL);
2889       /* Once all are done, free the iteration context */
2890       GNUNET_free(iter_ctx);
2891     }
2892 }
2893
2894
2895 /**
2896  * Check running topology iteration tasks, if below max start a new one, otherwise
2897  * schedule for some time in the future.
2898  */
2899 static void
2900 schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2901 {
2902   struct CoreContext *core_context = cls;
2903   struct TopologyIterateContext *topology_context = (struct TopologyIterateContext *)core_context->iter_context;
2904   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2905     return;
2906
2907   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2908     {
2909 #if VERBOSE_TESTING > 2
2910           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2911                       _("Delaying connect, we have too many outstanding connections!\n"));
2912 #endif
2913       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_topology, core_context);
2914     }
2915   else
2916     {
2917 #if VERBOSE_TESTING > 2
2918           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2919                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2920 #endif
2921       topology_context->connected++;
2922       if (GNUNET_OK != GNUNET_CORE_iterate_peers (core_context->daemon->sched, core_context->daemon->cfg, &internal_topology_callback, core_context))
2923         internal_topology_callback(core_context, NULL, GNUNET_TIME_relative_get_zero(), 0);
2924
2925     }
2926 }
2927
2928 /**
2929  * Iterate over all (running) peers in the peer group, retrieve
2930  * all connections that each currently has.
2931  */
2932 void
2933 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyTopology cb, void *cls)
2934 {
2935   struct TopologyIterateContext *topology_context;
2936   struct CoreContext *core_ctx;
2937   unsigned int i;
2938   unsigned int total_count;
2939
2940   /* Allocate a single topology iteration context */
2941   topology_context = GNUNET_malloc(sizeof(struct TopologyIterateContext));
2942   topology_context->topology_cb = cb;
2943   topology_context->cls = cls;
2944   total_count = 0;
2945   for (i = 0; i < pg->total; i++)
2946     {
2947       if (pg->peers[i].daemon->running == GNUNET_YES)
2948         {
2949           /* Allocate one core context per core we need to connect to */
2950           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
2951           core_ctx->daemon = pg->peers[i].daemon;
2952           /* Set back pointer to topology iteration context */
2953           core_ctx->iter_context = topology_context;
2954           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_topology, core_ctx);
2955           total_count++;
2956         }
2957     }
2958   topology_context->total = total_count;
2959   return;
2960 }
2961
2962 /**
2963  * Callback function to process statistic values.
2964  * This handler is here only really to insert a peer
2965  * identity (or daemon) so the statistics can be uniquely
2966  * tied to a single running peer.
2967  *
2968  * @param cls closure
2969  * @param subsystem name of subsystem that created the statistic
2970  * @param name the name of the datum
2971  * @param value the current value
2972  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
2973  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
2974  */
2975 static int internal_stats_callback (void *cls,
2976                                     const char *subsystem,
2977                                     const char *name,
2978                                     uint64_t value,
2979                                     int is_persistent)
2980 {
2981   struct StatsCoreContext *core_context = cls;
2982   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2983
2984   return stats_context->proc(stats_context->cls, &core_context->daemon->id, subsystem, name, value, is_persistent);
2985 }
2986
2987 /**
2988  * Internal continuation call for statistics iteration.
2989  *
2990  * @param cls closure, the CoreContext for this iteration
2991  * @param success whether or not the statistics iterations
2992  *        was canceled or not (we don't care)
2993  */
2994 static void internal_stats_cont (void *cls, int success)
2995 {
2996   struct StatsCoreContext *core_context = cls;
2997   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2998
2999   stats_context->connected--;
3000   stats_context->completed++;
3001
3002   if (stats_context->completed == stats_context->total)
3003     {
3004       stats_context->cont(stats_context->cls, GNUNET_YES);
3005       GNUNET_free(stats_context);
3006     }
3007
3008   if (core_context->stats_handle != NULL)
3009     GNUNET_STATISTICS_destroy(core_context->stats_handle, GNUNET_NO);
3010
3011   GNUNET_free(core_context);
3012 }
3013
3014 /**
3015  * Check running topology iteration tasks, if below max start a new one, otherwise
3016  * schedule for some time in the future.
3017  */
3018 static void
3019 schedule_get_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
3020 {
3021   struct StatsCoreContext *core_context = cls;
3022   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
3023
3024   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3025     return;
3026
3027   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3028     {
3029 #if VERBOSE_TESTING > 2
3030           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3031                       _("Delaying connect, we have too many outstanding connections!\n"));
3032 #endif
3033       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_statistics, core_context);
3034     }
3035   else
3036     {
3037 #if VERBOSE_TESTING > 2
3038           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3039                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
3040 #endif
3041
3042       stats_context->connected++;
3043       core_context->stats_handle = GNUNET_STATISTICS_create(core_context->daemon->sched, "testing", core_context->daemon->cfg);
3044       if (core_context->stats_handle == NULL)
3045         {
3046           internal_stats_cont (core_context, GNUNET_NO);
3047           return;
3048         }
3049
3050       core_context->stats_get_handle = GNUNET_STATISTICS_get(core_context->stats_handle, NULL, NULL, GNUNET_TIME_relative_get_forever(), &internal_stats_cont, &internal_stats_callback, core_context);
3051       if (core_context->stats_get_handle == NULL)
3052         internal_stats_cont (core_context, GNUNET_NO);
3053
3054     }
3055 }
3056
3057
3058 /**
3059  * Iterate over all (running) peers in the peer group, retrieve
3060  * all statistics from each.
3061  */
3062 void
3063 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3064                                GNUNET_STATISTICS_Callback cont,
3065                                GNUNET_TESTING_STATISTICS_Iterator proc, void *cls)
3066 {
3067   struct StatsIterateContext *stats_context;
3068   struct StatsCoreContext *core_ctx;
3069   unsigned int i;
3070   unsigned int total_count;
3071
3072   /* Allocate a single stats iteration context */
3073   stats_context = GNUNET_malloc(sizeof(struct StatsIterateContext));
3074   stats_context->cont = cont;
3075   stats_context->proc = proc;
3076   stats_context->cls = cls;
3077   total_count = 0;
3078   for (i = 0; i < pg->total; i++)
3079     {
3080       if (pg->peers[i].daemon->running == GNUNET_YES)
3081         {
3082           /* Allocate one core context per core we need to connect to */
3083           core_ctx = GNUNET_malloc(sizeof(struct StatsCoreContext));
3084           core_ctx->daemon = pg->peers[i].daemon;
3085           /* Set back pointer to topology iteration context */
3086           core_ctx->iter_context = stats_context;
3087           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_statistics, core_ctx);
3088           total_count++;
3089         }
3090     }
3091   stats_context->total = total_count;
3092   return;
3093 }
3094
3095 /**
3096  * There are many ways to connect peers that are supported by this function.
3097  * To connect peers in the same topology that was created via the
3098  * GNUNET_TESTING_create_topology, the topology variable must be set to
3099  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3100  * a new instance of that topology will be generated and attempted to be
3101  * connected.  This could result in some connections being impossible,
3102  * because some topologies are non-deterministic.
3103  *
3104  * @param pg the peer group struct representing the running peers
3105  * @param topology which topology to connect the peers in
3106  * @param options options for connecting the topology
3107  * @param option_modifier modifier for options that take a parameter
3108  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3109  */
3110 int
3111 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3112                                  enum GNUNET_TESTING_Topology topology,
3113                                  enum GNUNET_TESTING_TopologyOption options,
3114                                  double option_modifier)
3115 {
3116   switch (topology)
3117       {
3118       case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3119 #if VERBOSE_TOPOLOGY
3120       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3121                   _("Creating clique CONNECT topology\n"));
3122 #endif
3123         create_clique (pg, &add_actual_connections);
3124         break;
3125       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3126 #if VERBOSE_TOPOLOGY
3127       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3128                   _("Creating small world (ring) CONNECT topology\n"));
3129 #endif
3130         create_small_world_ring (pg, &add_actual_connections);
3131         break;
3132       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3133 #if VERBOSE_TOPOLOGY
3134       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3135                   _("Creating small world (2d-torus) CONNECT topology\n"));
3136 #endif
3137         create_small_world (pg, &add_actual_connections);
3138         break;
3139       case GNUNET_TESTING_TOPOLOGY_RING:
3140 #if VERBOSE_TOPOLOGY
3141       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3142                   _("Creating ring CONNECT topology\n"));
3143 #endif
3144         create_ring (pg, &add_actual_connections);
3145         break;
3146       case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3147 #if VERBOSE_TOPOLOGY
3148       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3149                   _("Creating 2d torus CONNECT topology\n"));
3150 #endif
3151         create_2d_torus (pg, &add_actual_connections);
3152         break;
3153       case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3154 #if VERBOSE_TOPOLOGY
3155       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3156                   _("Creating Erdos-Renyi CONNECT topology\n"));
3157 #endif
3158         create_erdos_renyi (pg, &add_actual_connections);
3159         break;
3160       case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3161 #if VERBOSE_TOPOLOGY
3162       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3163                   _("Creating InterNAT CONNECT topology\n"));
3164 #endif
3165         create_nated_internet (pg, &add_actual_connections);
3166         break;
3167       case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3168 #if VERBOSE_TOPOLOGY
3169       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3170                   _("Creating Scale Free CONNECT topology\n"));
3171 #endif
3172         create_scale_free (pg, &add_actual_connections);
3173         break;
3174       case GNUNET_TESTING_TOPOLOGY_LINE:
3175 #if VERBOSE_TOPOLOGY
3176       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3177                   _("Creating straight line CONNECT topology\n"));
3178 #endif
3179         create_line (pg, &add_actual_connections);
3180         break;
3181       case GNUNET_TESTING_TOPOLOGY_NONE:
3182 #if VERBOSE_TOPOLOGY
3183         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3184                   _("Creating no CONNECT topology\n"));
3185 #endif
3186         copy_allowed_topology(pg);
3187         break;
3188       default:
3189         GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3190                    _("Unknown topology specification, can't connect peers!\n"));
3191         return GNUNET_SYSERR;
3192       }
3193
3194   switch (options)
3195     {
3196     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3197 #if VERBOSE_TOPOLOGY
3198       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3199                   _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
3200 #endif
3201       choose_random_connections(pg, option_modifier);
3202       break;
3203     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3204 #if VERBOSE_TOPOLOGY
3205       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3206                   _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3207 #endif
3208       choose_minimum(pg, (unsigned int)option_modifier);
3209       break;
3210     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3211 #if VERBOSE_TOPOLOGY
3212       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3213                   _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3214 #endif
3215       perform_dfs(pg, (int)option_modifier);
3216       break;
3217     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3218       break;
3219     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3220       break;
3221     default:
3222       break;
3223     }
3224
3225   return connect_topology(pg);
3226 }
3227
3228 /**
3229  * Callback that is called whenever a hostkey is generated
3230  * for a peer.  Call the real callback and decrement the
3231  * starting counter for the peergroup.
3232  *
3233  * @param cls closure
3234  * @param id identifier for the daemon, NULL on error
3235  * @param d handle for the daemon
3236  * @param emsg error message (NULL on success)
3237  */
3238 static void internal_hostkey_callback (void *cls,
3239                                        const struct GNUNET_PeerIdentity *id,
3240                                        struct GNUNET_TESTING_Daemon *d,
3241                                        const char *emsg)
3242 {
3243   struct InternalStartContext *internal_context = cls;
3244   internal_context->peer->pg->starting--;
3245   internal_context->peer->pg->started++;
3246   if (internal_context->hostkey_callback != NULL)
3247     internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
3248   else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
3249     {
3250       internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
3251       GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
3252     }
3253 }
3254
3255 /**
3256  * Callback that is called whenever a peer has finished starting.
3257  * Call the real callback and decrement the starting counter
3258  * for the peergroup.
3259  *
3260  * @param cls closure
3261  * @param id identifier for the daemon, NULL on error
3262  * @param d handle for the daemon
3263  * @param emsg error message (NULL on success)
3264  */
3265 static void internal_startup_callback (void *cls,
3266                                        const struct GNUNET_PeerIdentity *id,
3267                                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3268                                        struct GNUNET_TESTING_Daemon *d,
3269                                        const char *emsg)
3270 {
3271   struct InternalStartContext *internal_context = cls;
3272   internal_context->peer->pg->starting--;
3273   if (internal_context->start_cb != NULL)
3274     internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
3275 }
3276
3277 static void
3278 internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3279 {
3280   struct InternalStartContext *internal_context = cls;
3281
3282   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3283     {
3284       return;
3285     }
3286
3287   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3288     {
3289       internal_context->peer->pg->starting++;
3290       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3291     }
3292   else
3293     {
3294       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
3295     }
3296 }
3297
3298 static void
3299 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3300 {
3301   struct InternalStartContext *internal_context = cls;
3302
3303   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3304     {
3305       return;
3306     }
3307
3308   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
3309     {
3310       internal_context->peer->pg->starting++;
3311       internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
3312                                                                     internal_context->peer->cfg,
3313                                                                     internal_context->timeout,
3314                                                                     internal_context->hostname,
3315                                                                     &internal_hostkey_callback,
3316                                                                     internal_context,
3317                                                                     &internal_startup_callback,
3318                                                                     internal_context);
3319     }
3320   else
3321     {
3322       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
3323     }
3324 }
3325
3326 /**
3327  * Function which continues a peer group starting up
3328  * after successfully generating hostkeys for each peer.
3329  *
3330  * @param pg the peer group to continue starting
3331  *
3332  */
3333 void
3334 GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
3335 {
3336   unsigned int i;
3337
3338   pg->starting = 0;
3339   for (i = 0; i < pg->total; i++)
3340     {
3341       GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
3342       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
3343     }
3344 }
3345
3346 /**
3347  * Start count gnunet instances with the same set of transports and
3348  * applications.  The port numbers (any option called "PORT") will be
3349  * adjusted to ensure that no two peers running on the same system
3350  * have the same port(s) in their respective configurations.
3351  *
3352  * @param sched scheduler to use
3353  * @param cfg configuration template to use
3354  * @param total number of daemons to start
3355  * @param timeout total time allowed for peers to start
3356  * @param hostkey_callback function to call on each peers hostkey generation
3357  *        if NULL, peers will be started by this call, if non-null,
3358  *        GNUNET_TESTING_daemons_continue_startup must be called after
3359  *        successful hostkey generation
3360  * @param hostkey_cls closure for hostkey callback
3361  * @param cb function to call on each daemon that was started
3362  * @param cb_cls closure for cb
3363  * @param connect_callback function to call each time two hosts are connected
3364  * @param connect_callback_cls closure for connect_callback
3365  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
3366  *
3367  * @return NULL on error, otherwise handle to control peer group
3368  */
3369 struct GNUNET_TESTING_PeerGroup *
3370 GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
3371                               const struct GNUNET_CONFIGURATION_Handle *cfg,
3372                               unsigned int total,
3373                               struct GNUNET_TIME_Relative timeout,
3374                               GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
3375                               void *hostkey_cls,
3376                               GNUNET_TESTING_NotifyDaemonRunning cb,
3377                               void *cb_cls,
3378                               GNUNET_TESTING_NotifyConnection
3379                               connect_callback, void *connect_callback_cls,
3380                               const struct GNUNET_TESTING_Host *hostnames)
3381 {
3382   struct GNUNET_TESTING_PeerGroup *pg;
3383   const struct GNUNET_TESTING_Host *hostpos;
3384 #if 0
3385   char *pos;
3386   const char *rpos;
3387   char *start;
3388 #endif
3389   const char *hostname;
3390   char *baseservicehome;
3391   char *newservicehome;
3392   char *tmpdir;
3393   struct GNUNET_CONFIGURATION_Handle *pcfg;
3394   unsigned int off;
3395   unsigned int hostcnt;
3396   uint16_t minport;
3397   uint32_t upnum;
3398   uint32_t fdnum;
3399
3400   if (0 == total)
3401     {
3402       GNUNET_break (0);
3403       return NULL;
3404     }
3405   upnum = 0;
3406   fdnum = 0;
3407   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
3408   pg->sched = sched;
3409   pg->cfg = cfg;
3410   pg->notify_connection = connect_callback;
3411   pg->notify_connection_cls = connect_callback_cls;
3412   pg->total = total;
3413   pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
3414   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
3415   if (NULL != hostnames)
3416     {
3417       off = 2;
3418       hostpos = hostnames;
3419       while (hostpos != NULL)
3420         {
3421           hostpos = hostpos->next;
3422           off++;
3423         }
3424       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3425       off = 0;
3426
3427       hostpos = hostnames;
3428       while (hostpos != NULL)
3429         {
3430           pg->hosts[off].minport = LOW_PORT;
3431           pg->hosts[off++].hostname = GNUNET_strdup(hostpos->hostname);
3432           hostpos = hostpos->next;
3433         }
3434
3435       if (off == 0)
3436         {
3437           GNUNET_free (pg->hosts);
3438           pg->hosts = NULL;
3439         }
3440       hostcnt = off;
3441       minport = 0;
3442
3443 #if NO_LL
3444       off = 2;
3445       /* skip leading spaces */
3446       while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
3447         hostnames++;
3448       rpos = hostnames;
3449       while ('\0' != *rpos)
3450         {
3451           if (isspace ( (unsigned char) *rpos))
3452             off++;
3453           rpos++;
3454         }
3455       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3456       off = 0;
3457       start = GNUNET_strdup (hostnames);
3458       pos = start;
3459       while ('\0' != *pos)
3460         {
3461           if (isspace ( (unsigned char) *pos))
3462             {
3463               *pos = '\0';
3464               if (strlen (start) > 0)
3465                 {
3466                   pg->hosts[off].minport = LOW_PORT;
3467                   pg->hosts[off++].hostname = start;
3468                 }
3469               start = pos + 1;
3470             }
3471           pos++;
3472         }
3473       if (strlen (start) > 0)
3474         {
3475           pg->hosts[off].minport = LOW_PORT;
3476           pg->hosts[off++].hostname = start;
3477         }
3478       if (off == 0)
3479         {
3480           GNUNET_free (start);
3481           GNUNET_free (pg->hosts);
3482           pg->hosts = NULL;
3483         }
3484       hostcnt = off;
3485       minport = 0;              /* make gcc happy */
3486 #endif
3487     }
3488   else
3489     {
3490       hostcnt = 0;
3491       minport = LOW_PORT;
3492     }
3493   for (off = 0; off < total; off++)
3494     {
3495       if (hostcnt > 0)
3496         {
3497           hostname = pg->hosts[off % hostcnt].hostname;
3498           pcfg = make_config (cfg,
3499                               &pg->hosts[off % hostcnt].minport,
3500                               &upnum,
3501                               hostname, &fdnum);
3502         }
3503       else
3504         {
3505           hostname = NULL;
3506           pcfg = make_config (cfg,
3507                               &minport,
3508                               &upnum,
3509                               hostname, &fdnum);
3510         }
3511
3512       if (NULL == pcfg)
3513         {
3514           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3515                       _
3516                       ("Could not create configuration for peer number %u on `%s'!\n"),
3517                       off, hostname == NULL ? "localhost" : hostname);
3518           continue;
3519         }
3520
3521       if (GNUNET_YES ==
3522           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
3523                                                  &baseservicehome))
3524         {
3525           GNUNET_asprintf (&newservicehome,
3526                            "%s/%d/", baseservicehome, off);
3527           GNUNET_free (baseservicehome);
3528         }
3529       else
3530         {
3531           tmpdir = getenv ("TMPDIR");
3532           tmpdir = tmpdir ? tmpdir : "/tmp";
3533           GNUNET_asprintf (&newservicehome,
3534                            "%s/%s/%d/",
3535                            tmpdir,
3536                            "gnunet-testing-test-test", off);
3537         }
3538       GNUNET_CONFIGURATION_set_value_string (pcfg,
3539                                              "PATHS",
3540                                              "SERVICEHOME", newservicehome);
3541       GNUNET_free (newservicehome);
3542       pg->peers[off].cfg = pcfg;
3543       pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
3544       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
3545       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
3546       pg->peers[off].pg = pg;
3547
3548       pg->peers[off].internal_context.peer = &pg->peers[off];
3549       pg->peers[off].internal_context.timeout = timeout;
3550       pg->peers[off].internal_context.hostname = hostname;
3551       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
3552       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
3553       pg->peers[off].internal_context.start_cb = cb;
3554       pg->peers[off].internal_context.start_cb_cls = cb_cls;
3555
3556       GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
3557
3558     }
3559   return pg;
3560 }
3561
3562 /*
3563  * Get a daemon by number, so callers don't have to do nasty
3564  * offsetting operation.
3565  */
3566 struct GNUNET_TESTING_Daemon *
3567 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
3568 {
3569   if (position < pg->total)
3570     return pg->peers[position].daemon;
3571   else
3572     return NULL;
3573 }
3574
3575 /*
3576  * Get a daemon by peer identity, so callers can
3577  * retrieve the daemon without knowing it's offset.
3578  *
3579  * @param pg the peer group to retrieve the daemon from
3580  * @param peer_id the peer identity of the daemon to retrieve
3581  *
3582  * @return the daemon on success, or NULL if no such peer identity is found
3583  */
3584 struct GNUNET_TESTING_Daemon *
3585 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_PeerIdentity *peer_id)
3586 {
3587   unsigned int i;
3588
3589   for (i = 0; i < pg->total; i ++)
3590     {
3591       if (0 == memcmp(&pg->peers[i].daemon->id, peer_id, sizeof(struct GNUNET_PeerIdentity)))
3592         return pg->peers[i].daemon;
3593     }
3594
3595   return NULL;
3596 }
3597
3598 /**
3599  * Prototype of a function that will be called when a
3600  * particular operation was completed the testing library.
3601  *
3602  * @param cls closure (a struct RestartContext)
3603  * @param id id of the peer that was restarted
3604  * @param cfg handle to the configuration of the peer
3605  * @param d handle to the daemon that was restarted
3606  * @param emsg NULL on success
3607  */
3608 void restart_callback (void *cls,
3609                        const struct GNUNET_PeerIdentity *id,
3610                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3611                        struct GNUNET_TESTING_Daemon *d,
3612                        const char *emsg)
3613 {
3614   struct RestartContext *restart_context = cls;
3615
3616   if (emsg == NULL)
3617     {
3618       restart_context->peers_restarted++;
3619     }
3620   else
3621     {
3622       restart_context->peers_restart_failed++;
3623     }
3624
3625   if (restart_context->peers_restarted == restart_context->peer_group->total)
3626     {
3627       restart_context->callback(restart_context->callback_cls, NULL);
3628       GNUNET_free(restart_context);
3629     }
3630   else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
3631     {
3632       restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
3633       GNUNET_free(restart_context);
3634     }
3635
3636 }
3637
3638 /**
3639  * Callback for informing us about a successful
3640  * or unsuccessful churn stop call.
3641  *
3642  * @param cls a ChurnContext
3643  * @param emsg NULL on success, non-NULL on failure
3644  *
3645  */
3646 void
3647 churn_stop_callback (void *cls, const char *emsg)
3648 {
3649   struct ChurnContext *churn_ctx = cls;
3650   unsigned int total_left;
3651   char *error_message;
3652
3653   error_message = NULL;
3654   if (emsg != NULL)
3655     {
3656       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3657                  "Churn stop callback failed with error `%s'\n", emsg);
3658       churn_ctx->num_failed_stop++;
3659     }
3660   else
3661     {
3662       churn_ctx->num_to_stop--;
3663     }
3664
3665 #if DEBUG_CHURN
3666   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3667              "Stopped peer, %d left.\n",
3668              churn_ctx->num_to_stop);
3669 #endif
3670   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3671
3672   if (total_left == 0)
3673   {
3674     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3675       {
3676         GNUNET_asprintf(&error_message,
3677                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3678                         churn_ctx->num_failed_start,
3679                         churn_ctx->num_failed_stop);
3680       }
3681     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3682     GNUNET_free_non_null(error_message);
3683     GNUNET_free(churn_ctx);
3684   }
3685 }
3686
3687 /**
3688  * Callback for informing us about a successful
3689  * or unsuccessful churn start call.
3690  *
3691  * @param cls a ChurnContext
3692  * @param id the peer identity of the started peer
3693  * @param cfg the handle to the configuration of the peer
3694  * @param d handle to the daemon for the peer
3695  * @param emsg NULL on success, non-NULL on failure
3696  *
3697  */
3698 void
3699 churn_start_callback (void *cls,
3700                       const struct GNUNET_PeerIdentity *id,
3701                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3702                       struct GNUNET_TESTING_Daemon *d,
3703                       const char *emsg)
3704 {
3705   struct ChurnContext *churn_ctx = cls;
3706   unsigned int total_left;
3707   char *error_message;
3708
3709   error_message = NULL;
3710   if (emsg != NULL)
3711     {
3712       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3713                   "Churn stop callback failed with error `%s'\n",
3714                   emsg);
3715       churn_ctx->num_failed_start++;
3716     }
3717   else
3718     {
3719       churn_ctx->num_to_start--;
3720     }
3721
3722 #if DEBUG_CHURN
3723   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3724              "Started peer, %d left.\n",
3725              churn_ctx->num_to_start);
3726 #endif
3727
3728   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3729
3730   if (total_left == 0)
3731   {
3732     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3733       GNUNET_asprintf(&error_message,
3734                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3735                       churn_ctx->num_failed_start,
3736                       churn_ctx->num_failed_stop);
3737     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3738     GNUNET_free_non_null(error_message);
3739     GNUNET_free(churn_ctx);
3740   }
3741 }
3742
3743
3744 /**
3745  * Simulate churn by stopping some peers (and possibly
3746  * re-starting others if churn is called multiple times).  This
3747  * function can only be used to create leave-join churn (peers "never"
3748  * leave for good).  First "voff" random peers that are currently
3749  * online will be taken offline; then "von" random peers that are then
3750  * offline will be put back online.  No notifications will be
3751  * generated for any of these operations except for the callback upon
3752  * completion.
3753  *
3754  * @param pg handle for the peer group
3755  * @param voff number of peers that should go offline
3756  * @param von number of peers that should come back online;
3757  *            must be zero on first call (since "testbed_start"
3758  *            always starts all of the peers)
3759  * @param timeout how long to wait for operations to finish before
3760  *        giving up
3761  * @param cb function to call at the end
3762  * @param cb_cls closure for cb
3763  */
3764 void
3765 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
3766                               unsigned int voff,
3767                               unsigned int von,
3768                               struct GNUNET_TIME_Relative timeout,
3769                               GNUNET_TESTING_NotifyCompletion cb,
3770                               void *cb_cls)
3771 {
3772   struct ChurnContext *churn_ctx;
3773   unsigned int running;
3774   unsigned int stopped;
3775   unsigned int total_running;
3776   unsigned int total_stopped;
3777   unsigned int i;
3778   unsigned int *running_arr;
3779   unsigned int *stopped_arr;
3780   unsigned int *running_permute;
3781   unsigned int *stopped_permute;
3782
3783   running = 0;
3784   stopped = 0;
3785
3786   if ((von == 0) && (voff == 0)) /* No peers at all? */
3787     {
3788       cb(cb_cls, NULL);
3789       return;
3790     }
3791
3792   for (i = 0; i < pg->total; i++)
3793   {
3794     if (pg->peers[i].daemon->running == GNUNET_YES)
3795     {
3796       GNUNET_assert(running != -1);
3797       running++;
3798     }
3799     else
3800     {
3801       GNUNET_assert(stopped != -1);
3802       stopped++;
3803     }
3804   }
3805
3806   if (voff > running)
3807   {
3808     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
3809     cb(cb_cls, "Trying to stop more peers than are currently running!");
3810     return;
3811   }
3812
3813   if (von > stopped)
3814   {
3815     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
3816     cb(cb_cls, "Trying to start more peers than are currently stopped!");
3817     return;
3818   }
3819
3820   churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3821
3822   running_arr = NULL;
3823   if (running > 0)
3824     running_arr = GNUNET_malloc(running * sizeof(unsigned int));
3825
3826   stopped_arr = NULL;
3827   if (stopped > 0)
3828     stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
3829
3830   running_permute = NULL;
3831   stopped_permute = NULL;
3832
3833   if (running > 0)
3834     running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
3835   if (stopped > 0)
3836     stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
3837
3838   total_running = running;
3839   total_stopped = stopped;
3840   running = 0;
3841   stopped = 0;
3842
3843   churn_ctx->num_to_start = von;
3844   churn_ctx->num_to_stop = voff;
3845   churn_ctx->cb = cb;
3846   churn_ctx->cb_cls = cb_cls;
3847
3848   for (i = 0; i < pg->total; i++)
3849   {
3850     if (pg->peers[i].daemon->running == GNUNET_YES)
3851     {
3852       GNUNET_assert((running_arr != NULL) && (total_running > running));
3853       running_arr[running] = i;
3854       running++;
3855     }
3856     else
3857     {
3858       GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
3859       stopped_arr[stopped] = i;
3860       stopped++;
3861     }
3862   }
3863
3864   GNUNET_assert(running >= voff);
3865   for (i = 0; i < voff; i++)
3866   {
3867 #if DEBUG_CHURN
3868     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
3869 #endif
3870     GNUNET_assert(running_arr != NULL);
3871     GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
3872                                 timeout,
3873                                 &churn_stop_callback, churn_ctx,
3874                                 GNUNET_NO, GNUNET_YES);
3875   }
3876
3877   GNUNET_assert(stopped >= von);
3878   for (i = 0; i < von; i++)
3879     {
3880 #if DEBUG_CHURN
3881       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
3882 #endif
3883       GNUNET_assert(stopped_arr != NULL);
3884       GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
3885                                           timeout, &churn_start_callback, churn_ctx);
3886   }
3887
3888   GNUNET_free_non_null(running_arr);
3889   GNUNET_free_non_null(stopped_arr);
3890   GNUNET_free_non_null(running_permute);
3891   GNUNET_free_non_null(stopped_permute);
3892 }
3893
3894
3895 /**
3896  * Restart all peers in the given group.
3897  *
3898  * @param pg the handle to the peer group
3899  * @param callback function to call on completion (or failure)
3900  * @param callback_cls closure for the callback function
3901  */
3902 void
3903 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
3904                                 GNUNET_TESTING_NotifyCompletion callback,
3905                                 void *callback_cls)
3906 {
3907   struct RestartContext *restart_context;
3908   unsigned int off;
3909
3910   if (pg->total > 0)
3911     {
3912       restart_context = GNUNET_malloc(sizeof(struct RestartContext));
3913       restart_context->peer_group = pg;
3914       restart_context->peers_restarted = 0;
3915       restart_context->callback = callback;
3916       restart_context->callback_cls = callback_cls;
3917
3918       for (off = 0; off < pg->total; off++)
3919         {
3920           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
3921         }
3922     }
3923 }
3924
3925 /**
3926  * Start or stop an individual peer from the given group.
3927  *
3928  * @param pg handle to the peer group
3929  * @param offset which peer to start or stop
3930  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
3931  * @param timeout how long to wait for shutdown
3932  * @param cb function to call at the end
3933  * @param cb_cls closure for cb
3934  */
3935 void
3936 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
3937                              unsigned int offset,
3938                              int desired_status,
3939                              struct GNUNET_TIME_Relative timeout,
3940                              GNUNET_TESTING_NotifyCompletion cb,
3941                              void *cb_cls)
3942 {
3943   struct ChurnContext *churn_ctx;
3944
3945   if (GNUNET_NO == desired_status)
3946     {
3947       if (NULL != pg->peers[offset].daemon)
3948         {
3949           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3950           churn_ctx->num_to_start = 0;
3951           churn_ctx->num_to_stop = 1;
3952           churn_ctx->cb = cb;
3953           churn_ctx->cb_cls = cb_cls;
3954           GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon,
3955                                      timeout, &churn_stop_callback, churn_ctx,
3956                                      GNUNET_NO, GNUNET_YES);
3957         }
3958     }
3959   else if (GNUNET_YES == desired_status)
3960     {
3961       if (NULL == pg->peers[offset].daemon)
3962         {
3963           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3964           churn_ctx->num_to_start = 1;
3965           churn_ctx->num_to_stop = 0;
3966           churn_ctx->cb = cb;
3967           churn_ctx->cb_cls = cb_cls;
3968           GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon,
3969                                               timeout, &churn_start_callback, churn_ctx);
3970         }
3971     }
3972   else
3973     GNUNET_break (0);
3974 }
3975
3976
3977 /**
3978  * Callback for shutting down peers in a peer group.
3979  *
3980  * @param cls closure (struct ShutdownContext)
3981  * @param emsg NULL on success
3982  */
3983 void internal_shutdown_callback (void *cls,
3984                                  const char *emsg)
3985 {
3986   struct ShutdownContext *shutdown_ctx = cls;
3987
3988   shutdown_ctx->outstanding--;
3989   if (emsg == NULL)
3990     {
3991       shutdown_ctx->peers_down++;
3992     }
3993   else
3994     {
3995       shutdown_ctx->peers_failed++;
3996     }
3997
3998   if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
3999     {
4000       if (shutdown_ctx->peers_failed > 0)
4001         shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
4002       else
4003         shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
4004       GNUNET_free(shutdown_ctx);
4005     }
4006 }
4007
4008 /**
4009  * Individual shutdown context for a particular peer.
4010  */
4011 struct PeerShutdownContext
4012 {
4013   /**
4014    * Pointer to the high level shutdown context.
4015    */
4016   struct ShutdownContext *shutdown_ctx;
4017
4018   /**
4019    * The daemon handle for the peer to shut down.
4020    */
4021   struct GNUNET_TESTING_Daemon *daemon;
4022 };
4023
4024 /**
4025  * Task to rate limit the number of outstanding peer shutdown
4026  * requests.  This is necessary for making sure we don't do
4027  * too many ssh connections at once, but is generally nicer
4028  * to any system as well (graduated task starts, as opposed
4029  * to calling gnunet-arm N times all at once).
4030  */
4031 static void
4032 schedule_shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
4033 {
4034   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4035   struct ShutdownContext *shutdown_ctx;
4036
4037   GNUNET_assert(peer_shutdown_ctx != NULL);
4038   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4039   GNUNET_assert(shutdown_ctx != NULL);
4040
4041   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4042     GNUNET_SCHEDULER_add_delayed(peer_shutdown_ctx->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_shutdown_task, peer_shutdown_ctx);
4043   else
4044     {
4045       shutdown_ctx->outstanding++;
4046       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, shutdown_ctx->timeout, &internal_shutdown_callback, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4047       GNUNET_free(peer_shutdown_ctx);
4048     }
4049 }
4050 /**
4051  * Shutdown all peers started in the given group.
4052  *
4053  * @param pg handle to the peer group
4054  * @param timeout how long to wait for shutdown
4055  * @param cb callback to notify upon success or failure
4056  * @param cb_cls closure for cb
4057  */
4058 void
4059 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4060                              struct GNUNET_TIME_Relative timeout,
4061                              GNUNET_TESTING_NotifyCompletion cb,
4062                              void *cb_cls)
4063 {
4064   unsigned int off;
4065   struct ShutdownContext *shutdown_ctx;
4066   struct PeerShutdownContext *peer_shutdown_ctx;
4067
4068   GNUNET_assert(pg->total > 0);
4069
4070   shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
4071   shutdown_ctx->cb = cb;
4072   shutdown_ctx->cb_cls = cb_cls;
4073   shutdown_ctx->total_peers = pg->total;
4074   shutdown_ctx->timeout = timeout;
4075   /* shtudown_ctx->outstanding = 0; */
4076
4077   for (off = 0; off < pg->total; off++)
4078     {
4079       GNUNET_assert(NULL != pg->peers[off].daemon);
4080       peer_shutdown_ctx = GNUNET_malloc(sizeof(struct PeerShutdownContext));
4081       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4082       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4083       GNUNET_SCHEDULER_add_now(pg->peers[off].daemon->sched, &schedule_shutdown_task, peer_shutdown_ctx);
4084       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4085       if (NULL != pg->peers[off].cfg)
4086         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4087       if (pg->peers[off].allowed_peers != NULL)
4088         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
4089       if (pg->peers[off].connect_peers != NULL)
4090         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
4091       if (pg->peers[off].blacklisted_peers != NULL)
4092         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
4093     }
4094   GNUNET_free (pg->peers);
4095   if (NULL != pg->hosts)
4096     {
4097       GNUNET_free (pg->hosts[0].hostname);
4098       GNUNET_free (pg->hosts);
4099     }
4100   GNUNET_free (pg);
4101 }
4102
4103
4104 /* end of testing_group.c */