src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define DEBUG_CHURN GNUNET_NO
  36
  37 /**
  38  * Lowest port used for GNUnet testing.  Should be high enough to not
  39  * conflict with other applications running on the hosts but be low
  40  * enough to not conflict with client-ports (typically starting around
  41  * 32k).
  42  */
  43 #define LOW_PORT 10000
  44
  45 /**
  46  * Highest port used for GNUnet testing.  Should be low enough to not
  47  * conflict with the port range for "local" ports (client apps; see
  48  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  49  */
  50 #define HIGH_PORT 56000
  51
  52 #define MAX_OUTSTANDING_CONNECTIONS 10
  53
  54 #define MAX_CONCURRENT_HOSTKEYS 10
  55
  56 #define MAX_CONCURRENT_STARTING 10
  57
  58 #define MAX_CONCURRENT_SHUTDOWN 10
  59
  60 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
  61
  62 #define CONNECT_ATTEMPTS 8
  63
  64 /**
  65  * Prototype of a function called whenever two peers would be connected
  66  * in a certain topology.
  67  */
  68 typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg,
  69                                                   unsigned int first,
  70                                                   unsigned int second);
  71
  72
  73 /**
  74  * Context for handling churning a peer group
  75  */
  76 struct ChurnContext
  77 {
  78   /**
  79    * Callback used to notify of churning finished
  80    */
  81   GNUNET_TESTING_NotifyCompletion cb;
  82
  83   /**
  84    * Closure for callback
  85    */
  86   void *cb_cls;
  87
  88   /**
  89    * Number of peers that still need to be started
  90    */
  91   unsigned int num_to_start;
  92
  93   /**
  94    * Number of peers that still need to be stopped
  95    */
  96   unsigned int num_to_stop;
  97
  98   /**
  99    * Number of peers that failed to start
 100    */
 101   unsigned int num_failed_start;
 102
 103   /**
 104    * Number of peers that failed to stop
 105    */
 106   unsigned int num_failed_stop;
 107 };
 108
 109 struct RestartContext
 110 {
 111   /**
 112    * The group of peers being restarted
 113    */
 114   struct GNUNET_TESTING_PeerGroup *peer_group;
 115
 116   /**
 117    * How many peers have been restarted thus far
 118    */
 119   unsigned int peers_restarted;
 120
 121   /**
 122    * How many peers got an error when restarting
 123    */
 124   unsigned int peers_restart_failed;
 125
 126   /**
 127    * The function to call once all peers have been restarted
 128    */
 129   GNUNET_TESTING_NotifyCompletion callback;
 130
 131   /**
 132    * Closure for callback function
 133    */
 134   void *callback_cls;
 135
 136 };
 137
 138
 139 struct ShutdownContext
 140 {
 141   /**
 142    * Total peers to wait for
 143    */
 144   unsigned int total_peers;
 145
 146   /**
 147    * Number of peers successfully shut down
 148    */
 149   unsigned int peers_down;
 150
 151   /**
 152    * Number of peers failed to shut down
 153    */
 154   unsigned int peers_failed;
 155
 156   /**
 157    * Number of peers we have started shutting
 158    * down.  If too many, wait on them.
 159    */
 160   unsigned int outstanding;
 161
 162   /**
 163    * Timeout for shutdown.
 164    */
 165   struct GNUNET_TIME_Relative timeout;
 166
 167   /**
 168    * Callback to call when all peers either
 169    * shutdown or failed to shutdown
 170    */
 171   GNUNET_TESTING_NotifyCompletion cb;
 172
 173   /**
 174    * Closure for cb
 175    */
 176   void *cb_cls;
 177 };
 178
 179 struct CreateTopologyContext
 180 {
 181
 182   /**
 183    * Function to call with number of connections
 184    */
 185   GNUNET_TESTING_NotifyConnections cont;
 186
 187   /**
 188    * Closure for connection notification
 189    */
 190   void *cls;
 191 };
 192
 193 #if OLD
 194 struct PeerConnection
 195 {
 196   /*
 197    * Linked list
 198    */
 199   struct PeerConnection *next;
 200
 201   /*
 202    * Pointer to daemon handle
 203    */
 204   struct GNUNET_TESTING_Daemon *daemon;
 205
 206 };
 207 #endif
 208
 209 struct InternalStartContext
 210 {
 211   /**
 212    * Pointer to peerdata
 213    */
 214   struct PeerData *peer;
 215
 216   /**
 217    * Timeout for peer startup
 218    */
 219   struct GNUNET_TIME_Relative timeout;
 220
 221   /**
 222    * Client callback for hostkey notification
 223    */
 224   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 225
 226   /**
 227    * Closure for hostkey_callback
 228    */
 229   void *hostkey_cls;
 230
 231   /**
 232    * Client callback for peer start notification
 233    */
 234   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 235
 236   /**
 237    * Closure for cb
 238    */
 239   void *start_cb_cls;
 240
 241   /**
 242    * Hostname, where to start the peer
 243    */
 244   const char *hostname;
 245 };
 246
 247 /**
 248  * Data we keep per peer.
 249  */
 250 struct PeerData
 251 {
 252   /**
 253    * (Initial) configuration of the host.
 254    * (initial because clients could change
 255    *  it and we would not know about those
 256    *  updates).
 257    */
 258   struct GNUNET_CONFIGURATION_Handle *cfg;
 259
 260   /**
 261    * Handle for controlling the daemon.
 262    */
 263   struct GNUNET_TESTING_Daemon *daemon;
 264
 265   /**
 266    * The peergroup this peer belongs to.
 267    */
 268   struct GNUNET_TESTING_PeerGroup *pg;
 269
 270   /**
 271    * Hash map of allowed peer connections (F2F created topology)
 272    */
 273   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 274
 275   /**
 276    * Hash map of blacklisted peers
 277    */
 278   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 279
 280   /**
 281    * Hash map of peer connections
 282    */
 283   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 284
 285   /**
 286    * Temporary hash map of peer connections
 287    */
 288   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 289
 290   /**
 291    * Temporary variable for topology creation, should be reset before
 292    * creating any topology so the count is valid once finished.
 293    */
 294   int num_connections;
 295
 296   /**
 297    * Context to keep track of peers being started, to
 298    * stagger hostkey generation and peer startup.
 299    */
 300   struct InternalStartContext internal_context;
 301 };
 302
 303
 304 /**
 305  * Data we keep per host.
 306  */
 307 struct HostData
 308 {
 309   /**
 310    * Name of the host.
 311    */
 312   char *hostname;
 313
 314   /**
 315    * Lowest port that we have not yet used
 316    * for GNUnet.
 317    */
 318   uint16_t minport;
 319 };
 320
 321 struct TopologyIterateContext
 322 {
 323   /**
 324    * Callback for notifying of two connected peers.
 325    */
 326   GNUNET_TESTING_NotifyTopology topology_cb;
 327
 328   /**
 329    * Closure for topology_cb
 330    */
 331   void *cls;
 332
 333   /**
 334    * Number of peers currently connected to.
 335    */
 336   unsigned int connected;
 337
 338   /**
 339    * Number of peers we have finished iterating.
 340    */
 341   unsigned int completed;
 342
 343   /**
 344    * Number of peers total.
 345    */
 346   unsigned int total;
 347 };
 348
 349 struct StatsIterateContext
 350 {
 351   /**
 352    * Continuation to call once all stats information has been retrieved.
 353    */
 354   GNUNET_STATISTICS_Callback cont;
 355
 356   /**
 357    * Proc function to call on each value received.
 358    */
 359   GNUNET_TESTING_STATISTICS_Iterator proc;
 360
 361   /**
 362    * Closure for topology_cb
 363    */
 364   void *cls;
 365
 366   /**
 367    * Number of peers currently connected to.
 368    */
 369   unsigned int connected;
 370
 371   /**
 372    * Number of peers we have finished iterating.
 373    */
 374   unsigned int completed;
 375
 376   /**
 377    * Number of peers total.
 378    */
 379   unsigned int total;
 380 };
 381
 382 struct CoreContext
 383 {
 384   void *iter_context;
 385   struct GNUNET_TESTING_Daemon *daemon;
 386 };
 387
 388 struct StatsCoreContext
 389 {
 390   void *iter_context;
 391   struct GNUNET_TESTING_Daemon *daemon;
 392   /**
 393    * Handle to the statistics service.
 394    */
 395   struct GNUNET_STATISTICS_Handle *stats_handle;
 396
 397   /**
 398    * Handle for getting statistics.
 399    */
 400   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 401 };
 402
 403 /**
 404  * Handle to a group of GNUnet peers.
 405  */
 406 struct GNUNET_TESTING_PeerGroup
 407 {
 408   /**
 409    * Our scheduler.
 410    */
 411   struct GNUNET_SCHEDULER_Handle *sched;
 412
 413   /**
 414    * Configuration template.
 415    */
 416   const struct GNUNET_CONFIGURATION_Handle *cfg;
 417
 418   /**
 419    * Function to call on each started daemon.
 420    */
 421   //GNUNET_TESTING_NotifyDaemonRunning cb;
 422
 423   /**
 424    * Closure for cb.
 425    */
 426   //void *cb_cls;
 427
 428   /*
 429    * Function to call on each topology connection created
 430    */
 431   GNUNET_TESTING_NotifyConnection notify_connection;
 432
 433   /*
 434    * Callback for notify_connection
 435    */
 436   void *notify_connection_cls;
 437
 438   /**
 439    * NULL-terminated array of information about
 440    * hosts.
 441    */
 442   struct HostData *hosts;
 443
 444   /**
 445    * Array of "total" peers.
 446    */
 447   struct PeerData *peers;
 448
 449   /**
 450    * Number of peers in this group.
 451    */
 452   unsigned int total;
 453
 454   /**
 455    * At what time should we fail the peer startup process?
 456    */
 457   struct GNUNET_TIME_Absolute max_timeout;
 458
 459   /**
 460    * How many peers are being started right now?
 461    */
 462   unsigned int starting;
 463
 464   /**
 465    * How many peers have already been started?
 466    */
 467   unsigned int started;
 468 };
 469
 470 struct UpdateContext
 471 {
 472   struct GNUNET_CONFIGURATION_Handle *ret;
 473   const struct GNUNET_CONFIGURATION_Handle *orig;
 474   const char *hostname;
 475   unsigned int nport;
 476   unsigned int upnum;
 477 };
 478
 479
 480 struct ConnectContext
 481 {
 482   struct GNUNET_TESTING_Daemon *first;
 483
 484   struct GNUNET_TESTING_Daemon *second;
 485
 486   struct GNUNET_TESTING_PeerGroup *pg;
 487 };
 488
 489 /**
 490  * Convert unique ID to hash code.
 491  *
 492  * @param uid unique ID to convert
 493  * @param hash set to uid (extended with zeros)
 494  */
 495 static void
 496 hash_from_uid (uint32_t uid,
 497                GNUNET_HashCode *hash)
 498 {
 499   memset (hash, 0, sizeof(GNUNET_HashCode));
 500   *((uint32_t*)hash) = uid;
 501 }
 502
 503 /**
 504  * Convert hash code to unique ID.
 505  *
 506  * @param uid unique ID to convert
 507  * @param hash set to uid (extended with zeros)
 508  */
 509 static void
 510 uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
 511 {
 512   memcpy (uid, hash, sizeof(uint32_t));
 513 }
 514
 515 /**
 516  * Number of connects we are waiting on, allows us to rate limit
 517  * connect attempts.
 518  */
 519 static int outstanding_connects;
 520
 521 /**
 522  * Get a topology from a string input.
 523  *
 524  * @param topology where to write the retrieved topology
 525  * @param topology_string The string to attempt to
 526  *        get a configuration value from
 527  * @return GNUNET_YES if topology string matched a
 528  *         known topology, GNUNET_NO if not
 529  */
 530 int
 531 GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
 532 {
 533   /**
 534    * Strings representing topologies in enum
 535    */
 536   static const char * topology_strings[] =
 537     {
 538       /**
 539        * A clique (everyone connected to everyone else).
 540        */
 541       "CLIQUE",
 542
 543       /**
 544        * Small-world network (2d torus plus random links).
 545        */
 546       "SMALL_WORLD",
 547
 548       /**
 549        * Small-world network (ring plus random links).
 550        */
 551       "SMALL_WORLD_RING",
 552
 553       /**
 554        * Ring topology.
 555        */
 556       "RING",
 557
 558       /**
 559        * 2-d torus.
 560        */
 561       "2D_TORUS",
 562
 563       /**
 564        * Random graph.
 565        */
 566       "ERDOS_RENYI",
 567
 568       /**
 569        * Certain percentage of peers are unable to communicate directly
 570        * replicating NAT conditions
 571        */
 572       "INTERNAT",
 573
 574       /**
 575        * Scale free topology.
 576        */
 577       "SCALE_FREE",
 578
 579       /**
 580        * Straight line topology.
 581        */
 582       "LINE",
 583
 584       /**
 585        * All peers are disconnected.
 586        */
 587       "NONE",
 588
 589       NULL
 590     };
 591
 592   int curr = 0;
 593   if (topology_string == NULL)
 594     return GNUNET_NO;
 595   while (topology_strings[curr] != NULL)
 596     {
 597       if (strcasecmp(topology_strings[curr], topology_string) == 0)
 598         {
 599           *topology = curr;
 600           return GNUNET_YES;
 601         }
 602       curr++;
 603     }
 604   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 605   return GNUNET_NO;
 606 }
 607
 608
 609 /**
 610  * Get connect topology option from string input.
 611  *
 612  * @param topology_option where to write the retrieved topology
 613  * @param topology_string The string to attempt to
 614  *        get a configuration value from
 615  * @return GNUNET_YES if string matched a known
 616  *         topology option, GNUNET_NO if not
 617  */
 618 int
 619 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
 620                                     char * topology_string)
 621 {
 622   /**
 623    * Options for connecting a topology as strings.
 624    */
 625   static const char * topology_option_strings[] =
 626     {
 627       /**
 628        * Try to connect all peers specified in the topology.
 629        */
 630       "CONNECT_ALL",
 631
 632       /**
 633        * Choose a random subset of connections to create.
 634        */
 635       "CONNECT_RANDOM_SUBSET",
 636
 637       /**
 638        * Create at least X connections for each peer.
 639        */
 640       "CONNECT_MINIMUM",
 641
 642       /**
 643        * Using a depth first search, create one connection
 644        * per peer.  If any are missed (graph disconnected)
 645        * start over at those peers until all have at least one
 646        * connection.
 647        */
 648       "CONNECT_DFS",
 649
 650       /**
 651        * No options specified.
 652        */
 653       "CONNECT_NONE",
 654
 655       NULL
 656     };
 657   int curr = 0;
 658
 659   if (topology_string == NULL)
 660     return GNUNET_NO;
 661   while (NULL != topology_option_strings[curr])
 662     {
 663       if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
 664         {
 665           *topology_option = curr;
 666           return GNUNET_YES;
 667         }
 668       curr++;
 669     }
 670   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 671   return GNUNET_NO;
 672 }
 673
 674 /**
 675  * Function to iterate over options.  Copies
 676  * the options to the target configuration,
 677  * updating PORT values as needed.
 678  *
 679  * @param cls closure
 680  * @param section name of the section
 681  * @param option name of the option
 682  * @param value value of the option
 683  */
 684 static void
 685 update_config (void *cls,
 686                const char *section, const char *option, const char *value)
 687 {
 688   struct UpdateContext *ctx = cls;
 689   unsigned int ival;
 690   char cval[12];
 691   char uval[128];
 692   char *single_variable;
 693
 694   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 695     {
 696       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 697       if ((ival != 0) && (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable)))
 698         {
 699           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 700           value = cval;
 701         }
 702
 703       GNUNET_free(single_variable);
 704     }
 705
 706   if (0 == strcmp (option, "UNIXPATH"))
 707     {
 708       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 709       if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable))
 710         {
 711           GNUNET_snprintf (uval,
 712                            sizeof (uval),
 713                            "/tmp/test-service-%s-%u",
 714                            section,
 715                            ctx->upnum++);
 716           value = uval;
 717         }
 718       GNUNET_free(single_variable);
 719
 720     }
 721
 722   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 723     {
 724       value = ctx->hostname;
 725     }
 726
 727   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 728 }
 729
 730
 731 /**
 732  * Create a new configuration using the given configuration
 733  * as a template; however, each PORT in the existing cfg
 734  * must be renumbered by incrementing "*port".  If we run
 735  * out of "*port" numbers, return NULL.
 736  *
 737  * @param cfg template configuration
 738  * @param port port numbers to use, update to reflect
 739  *             port numbers that were used
 740  * @param upnum number to make unix domain socket names unique
 741  * @param hostname hostname of the controlling host, to allow control connections from
 742  *
 743  * @return new configuration, NULL on error
 744  */
 745 static struct GNUNET_CONFIGURATION_Handle *
 746 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 747              uint16_t * port,
 748              uint32_t * upnum,
 749              const char *hostname)
 750 {
 751   struct UpdateContext uc;
 752   uint16_t orig;
 753   char *control_host;
 754   char *allowed_hosts;
 755
 756   orig = *port;
 757   uc.nport = *port;
 758   uc.upnum = *upnum;
 759   uc.ret = GNUNET_CONFIGURATION_create ();
 760   uc.hostname = hostname;
 761   uc.orig = cfg;
 762
 763   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 764   if (uc.nport >= HIGH_PORT)
 765     {
 766       *port = orig;
 767       GNUNET_CONFIGURATION_destroy (uc.ret);
 768       return NULL;
 769     }
 770
 771   if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 772     {
 773       if (hostname != NULL)
 774         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1; %s;", control_host, hostname);
 775       else
 776         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
 777
 778       GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
 779       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport", "ACCEPT_FROM", allowed_hosts);
 780       GNUNET_CONFIGURATION_set_value_string(uc.ret, "dht", "ACCEPT_FROM", allowed_hosts);
 781       GNUNET_CONFIGURATION_set_value_string(uc.ret, "statistics", "ACCEPT_FROM", allowed_hosts);
 782       GNUNET_free_non_null(control_host);
 783       GNUNET_free(allowed_hosts);
 784     }
 785
 786
 787   /* arm needs to know to allow connections from the host on which it is running,
 788    * otherwise gnunet-arm is unable to connect to it in some instances */
 789   if (hostname != NULL)
 790     {
 791       GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
 792       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", hostname);
 793       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", hostname);
 794       GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
 795       GNUNET_free(allowed_hosts);
 796     }
 797   else
 798     {
 799       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", "127.0.0.1");
 800       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", "127.0.0.1");
 801     }
 802
 803   *port = (uint16_t) uc.nport;
 804   *upnum = uc.upnum;
 805   return uc.ret;
 806 }
 807
 808
 809 /*
 810  * Add entries to the peers connect list
 811  *
 812  * @param pg the peer group we are working with
 813  * @param first index of the first peer
 814  * @param second index of the second peer
 815  *
 816  * @return the number of connections added
 817  *         technically should only be 0 or 2
 818  *
 819  */
 820 static int
 821 add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 822 {
 823   int added;
 824   int add_first;
 825   int add_second;
 826
 827   GNUNET_HashCode hash_first;
 828   GNUNET_HashCode hash_second;
 829
 830   hash_from_uid(first, &hash_first);
 831   hash_from_uid(second, &hash_second);
 832
 833   add_first = GNUNET_NO;
 834   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
 835     {
 836       add_first = GNUNET_YES;
 837     }
 838
 839   add_second = GNUNET_NO;
 840   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
 841     {
 842       add_second = GNUNET_YES;
 843     }
 844
 845   added = 0;
 846   if (add_first)
 847     {
 848       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 849       pg->peers[first].num_connections++;
 850       added++;
 851     }
 852
 853   if (add_second)
 854     {
 855       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 856       pg->peers[second].num_connections++;
 857       added++;
 858     }
 859
 860   return added;
 861 }
 862
 863
 864 /*
 865  * Add entries to the peers allowed connections list
 866  *
 867  * @param pg the peer group we are working with
 868  * @param first index of the first peer
 869  * @param second index of the second peer
 870  *
 871  * @return the number of connections added (can be 0, 1 or 2)
 872  *         technically should only be 0 or 2, but the small price
 873  *         of iterating over the lists (hashmaps in the future)
 874  *         for being sure doesn't bother me!
 875  *
 876  */
 877 static int
 878 add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 879 {
 880   int added;
 881 #if OLD
 882   struct PeerConnection *first_iter;
 883   struct PeerConnection *second_iter;
 884   struct PeerConnection *new_first;
 885   struct PeerConnection *new_second;
 886 #endif
 887   int add_first;
 888   int add_second;
 889
 890   GNUNET_HashCode hash_first;
 891   GNUNET_HashCode hash_second;
 892
 893   hash_from_uid(first, &hash_first);
 894   hash_from_uid(second, &hash_second);
 895
 896   add_first = GNUNET_NO;
 897   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
 898     {
 899       add_first = GNUNET_YES;
 900     }
 901
 902   add_second = GNUNET_NO;
 903   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
 904     {
 905       add_second = GNUNET_YES;
 906     }
 907 #if OLD
 908   first_iter = pg->peers[first].connected_peers;
 909   while (first_iter != NULL)
 910     {
 911       if (first_iter->daemon == pg->peers[second].daemon)
 912         add_first = GNUNET_NO;
 913       first_iter = first_iter->next;
 914     }
 915
 916   second_iter = pg->peers[second].connected_peers;
 917   add_second = GNUNET_YES;
 918   while (second_iter != NULL)
 919     {
 920       if (second_iter->daemon == pg->peers[first].daemon)
 921         add_second = GNUNET_NO;
 922       second_iter = second_iter->next;
 923     }
 924 #endif
 925
 926   added = 0;
 927   if (add_first)
 928     {
 929       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 930 #if OLD
 931       new_first = GNUNET_malloc(sizeof(struct PeerConnection));
 932       new_first->daemon = pg->peers[second].daemon;
 933       new_first->next = pg->peers[first].connected_peers;
 934       pg->peers[first].connected_peers = new_first;
 935 #endif
 936       pg->peers[first].num_connections++;
 937       added++;
 938     }
 939
 940   if (add_second)
 941     {
 942       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 943 #if OLD
 944       new_second = GNUNET_malloc(sizeof(struct PeerConnection));
 945       new_second->daemon = pg->peers[first].daemon;
 946       new_second->next = pg->peers[second].connected_peers;
 947       pg->peers[second].connected_peers = new_second;
 948       pg->peers[first].num_connections++;
 949 #endif
 950       pg->peers[second].num_connections++;
 951       added++;
 952     }
 953
 954   return added;
 955 }
 956
 957 /*
 958  * Add entries to the peers blacklisted list
 959  *
 960  * @param pg the peer group we are working with
 961  * @param first index of the first peer
 962  * @param second index of the second peer
 963  *
 964  * @return the number of connections added (can be 0, 1 or 2)
 965  *
 966  */
 967 static int
 968 blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 969 {
 970   int added;
 971   int add_first;
 972   int add_second;
 973   GNUNET_HashCode hash_first;
 974   GNUNET_HashCode hash_second;
 975
 976   hash_from_uid(first, &hash_first);
 977   hash_from_uid(second, &hash_second);
 978
 979   add_first = GNUNET_NO;
 980   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
 981     {
 982       add_first = GNUNET_YES;
 983     }
 984
 985   add_second = GNUNET_NO;
 986   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
 987     {
 988       add_second = GNUNET_YES;
 989     }
 990
 991   added = 0;
 992   if (add_first)
 993     {
 994       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 995       pg->peers[first].num_connections++;
 996       added++;
 997     }
 998
 999   if (add_second)
1000     {
1001       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1002       pg->peers[second].num_connections++;
1003       added++;
1004     }
1005
1006   return added;
1007 }
1008
1009 /*
1010  * Remove entries from the peers blacklisted list
1011  *
1012  * @param pg the peer group we are working with
1013  * @param first index of the first peer
1014  * @param second index of the second peer
1015  *
1016  * @return the number of connections removed (can be 0, 1 or 2)
1017  *
1018  */
1019 static int
1020 unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1021 {
1022   int removed;
1023   int remove_first;
1024   int remove_second;
1025   GNUNET_HashCode hash_first;
1026   GNUNET_HashCode hash_second;
1027
1028   hash_from_uid(first, &hash_first);
1029   hash_from_uid(second, &hash_second);
1030
1031   remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
1032   remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
1033
1034   removed = 0;
1035   if (remove_first)
1036     {
1037       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
1038       removed++;
1039     }
1040
1041   if (remove_second)
1042     {
1043       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
1044       removed++;
1045     }
1046
1047   return removed;
1048 }
1049
1050 /**
1051  * Scale free network construction as described in:
1052  *
1053  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1054  *
1055  * Start with a network of "one" peer, then progressively add
1056  * peers up to the total number.  At each step, iterate over
1057  * all possible peers and connect new peer based on number of
1058  * existing connections of the target peer.
1059  *
1060  * @param pg the peer group we are dealing with
1061  * @param proc the connection processor to use
1062  *
1063  * @return the number of connections created
1064  */
1065 static int
1066 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1067 {
1068
1069   unsigned int total_connections;
1070   unsigned int outer_count;
1071   unsigned int i;
1072   unsigned int previous_total_connections;
1073   double random;
1074   double probability;
1075
1076   GNUNET_assert(pg->total > 1);
1077
1078   /* Add a connection between the first two nodes */
1079   total_connections = proc(pg, 0, 1);
1080
1081   for (outer_count = 1; outer_count < pg->total; outer_count++)
1082     {
1083       previous_total_connections = total_connections;
1084       for (i = 0; i < outer_count; i++)
1085         {
1086           probability = pg->peers[i].num_connections / (double)previous_total_connections;
1087           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1088                                                       UINT64_MAX)) / ( (double) UINT64_MAX);
1089 #if VERBOSE_TESTING
1090           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1091                       "Considering connecting peer %d to peer %d\n",
1092                       outer_count, i);
1093 #endif
1094           if (random < probability)
1095             {
1096 #if VERBOSE_TESTING
1097               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1098                           "Connecting peer %d to peer %d\n",
1099                           outer_count, i);
1100 #endif
1101               total_connections += proc(pg, outer_count, i);
1102             }
1103         }
1104     }
1105
1106   return total_connections;
1107 }
1108
1109 /**
1110  * Create a topology given a peer group (set of running peers)
1111  * and a connection processor.
1112  *
1113  * @param pg the peergroup to create the topology on
1114  * @param proc the connection processor to call to actually set
1115  *        up connections between two peers
1116  *
1117  * @return the number of connections that were set up
1118  *
1119  */
1120 int
1121 create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1122 {
1123   unsigned int i, j;
1124   int nodeToConnect;
1125   unsigned int natLog;
1126   unsigned int randomPeer;
1127   double random, logNModifier, percentage;
1128   unsigned int smallWorldConnections;
1129   int connsPerPeer;
1130   char *p_string;
1131   int max;
1132   int min;
1133   unsigned int useAnd;
1134   int connect_attempts;
1135
1136   logNModifier = 0.5; /* FIXME: default value? */
1137   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1138                                                          "TESTING",
1139                                                          "LOGNMODIFIER",
1140                                                          &p_string))
1141     {
1142       if (sscanf(p_string, "%lf", &logNModifier) != 1)
1143         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1144                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1145                     p_string,
1146                     "LOGNMODIFIER",
1147                     "TESTING");
1148       GNUNET_free (p_string);
1149     }
1150   percentage = 0.5; /* FIXME: default percentage? */
1151   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1152                                                          "TESTING",
1153                                                          "PERCENTAGE",
1154                                                          &p_string))
1155     {
1156       if (sscanf(p_string, "%lf", &percentage) != 1)
1157         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1158                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1159                     p_string,
1160                     "PERCENTAGE",
1161                     "TESTING");
1162       GNUNET_free (p_string);
1163     }
1164   natLog = log (pg->total);
1165   connsPerPeer = ceil (natLog * logNModifier);
1166
1167   if (connsPerPeer % 2 == 1)
1168     connsPerPeer += 1;
1169
1170   smallWorldConnections = 0;
1171   connect_attempts = 0;
1172   for (i = 0; i < pg->total; i++)
1173     {
1174       useAnd = 0;
1175       max = i + connsPerPeer / 2;
1176       min = i - connsPerPeer / 2;
1177
1178       if (max > pg->total - 1)
1179         {
1180           max = max - pg->total;
1181           useAnd = 1;
1182         }
1183
1184       if (min < 0)
1185         {
1186           min = pg->total - 1 + min;
1187           useAnd = 1;
1188         }
1189
1190       for (j = 0; j < connsPerPeer / 2; j++)
1191         {
1192           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1193                                                       UINT64_MAX) / ( (double) UINT64_MAX));
1194           if (random < percentage)
1195             {
1196               /* Connect to uniformly selected random peer */
1197               randomPeer =
1198                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1199                                    pg->total);
1200               while ((((randomPeer < max) && (randomPeer > min))
1201                       && (useAnd == 0)) || (((randomPeer > min)
1202                                              || (randomPeer < max))
1203                                             && (useAnd == 1)))
1204                 {
1205                   randomPeer =
1206                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1207                                                          pg->total);
1208                 }
1209               smallWorldConnections +=
1210                 proc (pg, i, randomPeer);
1211             }
1212           else
1213             {
1214               nodeToConnect = i + j + 1;
1215               if (nodeToConnect > pg->total - 1)
1216                 {
1217                   nodeToConnect = nodeToConnect - pg->total;
1218                 }
1219               connect_attempts +=
1220                 proc (pg, i, nodeToConnect);
1221             }
1222         }
1223
1224     }
1225
1226   connect_attempts += smallWorldConnections;
1227
1228   return connect_attempts;
1229 }
1230
1231 /**
1232  * Create a topology given a peer group (set of running peers)
1233  * and a connection processor.
1234  *
1235  * @param pg the peergroup to create the topology on
1236  * @param proc the connection processor to call to actually set
1237  *        up connections between two peers
1238  *
1239  * @return the number of connections that were set up
1240  *
1241  */
1242 static int
1243 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1244 {
1245   unsigned int outer_count, inner_count;
1246   unsigned int cutoff;
1247   int connect_attempts;
1248   double nat_percentage;
1249   char *p_string;
1250
1251   nat_percentage = 0.6; /* FIXME: default percentage? */
1252   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1253                                                          "TESTING",
1254                                                          "NATPERCENTAGE",
1255                                                          &p_string))
1256     {
1257       if (sscanf(p_string, "%lf", &nat_percentage) != 1)
1258         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1259                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1260                     p_string,
1261                     "NATPERCENTAGE",
1262                     "TESTING");
1263       GNUNET_free (p_string);
1264     }
1265
1266
1267
1268   cutoff = (unsigned int) (nat_percentage * pg->total);
1269
1270   connect_attempts = 0;
1271
1272   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1273     {
1274       for (inner_count = outer_count + 1; inner_count < pg->total;
1275            inner_count++)
1276         {
1277           if ((outer_count > cutoff) || (inner_count > cutoff))
1278             {
1279 #if VERBOSE_TESTING
1280               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1281                           "Connecting peer %d to peer %d\n",
1282                           outer_count, inner_count);
1283 #endif
1284               connect_attempts += proc(pg, outer_count, inner_count);
1285             }
1286         }
1287     }
1288
1289   return connect_attempts;
1290
1291 }
1292
1293 /**
1294  * Create a topology given a peer group (set of running peers)
1295  * and a connection processor.
1296  *
1297  * @param pg the peergroup to create the topology on
1298  * @param proc the connection processor to call to actually set
1299  *        up connections between two peers
1300  *
1301  * @return the number of connections that were set up
1302  *
1303  */
1304 static int
1305 create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1306 {
1307   unsigned int i, j, k;
1308   unsigned int square;
1309   unsigned int rows;
1310   unsigned int cols;
1311   unsigned int toggle = 1;
1312   unsigned int nodeToConnect;
1313   unsigned int natLog;
1314   unsigned int node1Row;
1315   unsigned int node1Col;
1316   unsigned int node2Row;
1317   unsigned int node2Col;
1318   unsigned int distance;
1319   double probability, random, percentage;
1320   unsigned int smallWorldConnections;
1321   unsigned int small_world_it;
1322   char *p_string;
1323   int connect_attempts;
1324   square = floor (sqrt (pg->total));
1325   rows = square;
1326   cols = square;
1327
1328   percentage = 0.5; /* FIXME: default percentage? */
1329   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1330                                                          "TESTING",
1331                                                          "PERCENTAGE",
1332                                                          &p_string))
1333     {
1334       if (sscanf(p_string, "%lf", &percentage) != 1)
1335         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1336                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1337                     p_string,
1338                     "PERCENTAGE",
1339                     "TESTING");
1340       GNUNET_free (p_string);
1341     }
1342   if (percentage < 0.0)
1343     {
1344       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1345                   _("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1346                   "PERCENTAGE", "TESTING", percentage);
1347       percentage = 0.5;
1348     }
1349   probability = 0.5; /* FIXME: default percentage? */
1350   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1351                                                          "TESTING",
1352                                                          "PROBABILITY",
1353                                                          &p_string))
1354     {
1355       if (sscanf(p_string, "%lf", &probability) != 1)
1356         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1357                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1358                     p_string,
1359                     "PROBABILITY",
1360                     "TESTING");
1361       GNUNET_free (p_string);
1362     }
1363   if (square * square != pg->total)
1364     {
1365       while (rows * cols < pg->total)
1366         {
1367           if (toggle % 2 == 0)
1368             rows++;
1369           else
1370             cols++;
1371
1372           toggle++;
1373         }
1374     }
1375 #if VERBOSE_TESTING
1376       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1377                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1378                   rows, cols);
1379 #endif
1380
1381   connect_attempts = 0;
1382   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1383    * to the node to its right and above.  Once this is over, we'll have our torus!
1384    * Special case for the last node (if the rows and columns are not equal), connect
1385    * to the first in the row to maintain topology.
1386    */
1387   for (i = 0; i < pg->total; i++)
1388     {
1389       /* First connect to the node to the right */
1390       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1391         nodeToConnect = i + 1;
1392       else if (i + 1 == pg->total)
1393         nodeToConnect = rows * cols - cols;
1394       else
1395         nodeToConnect = i - cols + 1;
1396
1397       connect_attempts += proc (pg, i, nodeToConnect);
1398
1399       if (i < cols)
1400         nodeToConnect = (rows * cols) - cols + i;
1401       else
1402         nodeToConnect = i - cols;
1403
1404       if (nodeToConnect < pg->total)
1405         connect_attempts += proc (pg, i, nodeToConnect);
1406     }
1407   natLog = log (pg->total);
1408 #if VERBOSE_TESTING > 2
1409   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1410               _("natural log of %d is %d, will run %d iterations\n"),
1411              pg->total, natLog, (int) (natLog * percentage));
1412   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
1413 #endif
1414   smallWorldConnections = 0;
1415   small_world_it = (unsigned int)(natLog * percentage);
1416   GNUNET_assert(small_world_it > 0 && small_world_it < (unsigned int)-1);
1417   for (i = 0; i < small_world_it; i++)
1418     {
1419       for (j = 0; j < pg->total; j++)
1420         {
1421           /* Determine the row and column of node at position j on the 2d torus */
1422           node1Row = j / cols;
1423           node1Col = j - (node1Row * cols);
1424           for (k = 0; k < pg->total; k++)
1425             {
1426               /* Determine the row and column of node at position k on the 2d torus */
1427               node2Row = k / cols;
1428               node2Col = k - (node2Row * cols);
1429               /* Simple Cartesian distance */
1430               distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
1431               if (distance > 1)
1432                 {
1433                   /* Calculate probability as 1 over the square of the distance */
1434                   probability = 1.0 / (distance * distance);
1435                   /* Choose a random value between 0 and 1 */
1436                   random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1437                                                               UINT64_MAX)) / ( (double) UINT64_MAX);
1438                   /* If random < probability, then connect the two nodes */
1439                   if (random < probability)
1440                     smallWorldConnections += proc (pg, j, k);
1441
1442                 }
1443             }
1444         }
1445     }
1446   connect_attempts += smallWorldConnections;
1447 #if VERBOSE_TESTING > 2
1448           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1449                       _("Total connections added for small world: %d!\n"),
1450                       smallWorldConnections);
1451 #endif
1452   return connect_attempts;
1453 }
1454
1455 /**
1456  * Create a topology given a peer group (set of running peers)
1457  * and a connection processor.
1458  *
1459  * @param pg the peergroup to create the topology on
1460  * @param proc the connection processor to call to actually set
1461  *        up connections between two peers
1462  *
1463  * @return the number of connections that were set up
1464  *
1465  */
1466 static int
1467 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1468 {
1469   double temp_rand;
1470   unsigned int outer_count;
1471   unsigned int inner_count;
1472   int connect_attempts;
1473   double probability;
1474   char *p_string;
1475
1476   probability = 0.5; /* FIXME: default percentage? */
1477   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1478                                                          "TESTING",
1479                                                          "PROBABILITY",
1480                                                          &p_string))
1481     {
1482       if (sscanf(p_string, "%lf", &probability) != 1)
1483         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1484                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1485                     p_string,
1486                     "PROBABILITY",
1487                     "TESTING");
1488       GNUNET_free (p_string);
1489     }
1490   connect_attempts = 0;
1491   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1492     {
1493       for (inner_count = outer_count + 1; inner_count < pg->total;
1494            inner_count++)
1495         {
1496           temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1497                                                          UINT64_MAX)) / ( (double) UINT64_MAX);
1498 #if VERBOSE_TESTING
1499           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1500                       _("rand is %f probability is %f\n"), temp_rand,
1501                       probability);
1502 #endif
1503           if (temp_rand < probability)
1504             {
1505               connect_attempts += proc (pg, outer_count, inner_count);
1506             }
1507         }
1508     }
1509
1510   return connect_attempts;
1511 }
1512
1513 /**
1514  * Create a topology given a peer group (set of running peers)
1515  * and a connection processor.
1516  *
1517  * @param pg the peergroup to create the topology on
1518  * @param proc the connection processor to call to actually set
1519  *        up connections between two peers
1520  *
1521  * @return the number of connections that were set up
1522  *
1523  */
1524 static int
1525 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1526 {
1527   unsigned int i;
1528   unsigned int square;
1529   unsigned int rows;
1530   unsigned int cols;
1531   unsigned int toggle = 1;
1532   unsigned int nodeToConnect;
1533   int connect_attempts;
1534
1535   connect_attempts = 0;
1536
1537   square = floor (sqrt (pg->total));
1538   rows = square;
1539   cols = square;
1540
1541   if (square * square != pg->total)
1542     {
1543       while (rows * cols < pg->total)
1544         {
1545           if (toggle % 2 == 0)
1546             rows++;
1547           else
1548             cols++;
1549
1550           toggle++;
1551         }
1552     }
1553 #if VERBOSE_TESTING
1554       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1555                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1556                   rows, cols);
1557 #endif
1558   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1559    * to the node to its right and above.  Once this is over, we'll have our torus!
1560    * Special case for the last node (if the rows and columns are not equal), connect
1561    * to the first in the row to maintain topology.
1562    */
1563   for (i = 0; i < pg->total; i++)
1564     {
1565       /* First connect to the node to the right */
1566       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1567         nodeToConnect = i + 1;
1568       else if (i + 1 == pg->total)
1569         nodeToConnect = rows * cols - cols;
1570       else
1571         nodeToConnect = i - cols + 1;
1572 #if VERBOSE_TESTING
1573           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1574                       "Connecting peer %d to peer %d\n",
1575                       i, nodeToConnect);
1576 #endif
1577       connect_attempts += proc(pg, i, nodeToConnect);
1578
1579       /* Second connect to the node immediately above */
1580       if (i < cols)
1581         nodeToConnect = (rows * cols) - cols + i;
1582       else
1583         nodeToConnect = i - cols;
1584
1585       if (nodeToConnect < pg->total)
1586         {
1587 #if VERBOSE_TESTING
1588           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1589                       "Connecting peer %d to peer %d\n",
1590                       i, nodeToConnect);
1591 #endif
1592           connect_attempts += proc(pg, i, nodeToConnect);
1593         }
1594
1595     }
1596
1597   return connect_attempts;
1598 }
1599
1600
1601 /**
1602  * Create a topology given a peer group (set of running peers)
1603  * and a connection processor.
1604  *
1605  * @param pg the peergroup to create the topology on
1606  * @param proc the connection processor to call to actually set
1607  *        up connections between two peers
1608  *
1609  * @return the number of connections that were set up
1610  *
1611  */
1612 static int
1613 create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1614 {
1615   unsigned int outer_count;
1616   unsigned int inner_count;
1617   int connect_attempts;
1618
1619   connect_attempts = 0;
1620
1621   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1622     {
1623       for (inner_count = outer_count + 1; inner_count < pg->total;
1624            inner_count++)
1625         {
1626 #if VERBOSE_TESTING
1627           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1628                       "Connecting peer %d to peer %d\n",
1629                       outer_count, inner_count);
1630 #endif
1631           connect_attempts += proc(pg, outer_count, inner_count);
1632         }
1633     }
1634
1635   return connect_attempts;
1636 }
1637
1638 /**
1639  * Create a topology given a peer group (set of running peers)
1640  * and a connection processor.
1641  *
1642  * @param pg the peergroup to create the topology on
1643  * @param proc the connection processor to call to actually set
1644  *        up connections between two peers
1645  *
1646  * @return the number of connections that were set up
1647  *
1648  */
1649 static int
1650 create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1651 {
1652   unsigned int count;
1653   int connect_attempts;
1654
1655   connect_attempts = 0;
1656
1657   /* Connect each peer to the next highest numbered peer */
1658   for (count = 0; count < pg->total - 1; count++)
1659     {
1660 #if VERBOSE_TESTING
1661           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1662                       "Connecting peer %d to peer %d\n",
1663                       count, count + 1);
1664 #endif
1665       connect_attempts += proc(pg, count, count + 1);
1666     }
1667
1668   return connect_attempts;
1669 }
1670
1671 /**
1672  * Create a topology given a peer group (set of running peers)
1673  * and a connection processor.
1674  *
1675  * @param pg the peergroup to create the topology on
1676  * @param proc the connection processor to call to actually set
1677  *        up connections between two peers
1678  *
1679  * @return the number of connections that were set up
1680  *
1681  */
1682 static int
1683 create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1684 {
1685   unsigned int count;
1686   int connect_attempts;
1687
1688   connect_attempts = 0;
1689
1690   /* Connect each peer to the next highest numbered peer */
1691   for (count = 0; count < pg->total - 1; count++)
1692     {
1693 #if VERBOSE_TESTING
1694           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1695                       "Connecting peer %d to peer %d\n",
1696                       count, count + 1);
1697 #endif
1698       connect_attempts += proc(pg, count, count + 1);
1699     }
1700
1701   /* Connect the last peer to the first peer */
1702   connect_attempts += proc(pg, pg->total - 1, 0);
1703
1704   return connect_attempts;
1705 }
1706
1707
1708 /**
1709  * Iterator for writing friends of a peer to a file.
1710  *
1711  * @param cls closure, an open writable file handle
1712  * @param key the key the daemon was stored under
1713  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1714  *
1715  * @return GNUNET_YES to continue iteration
1716  *
1717  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1718  *       with a single file_iterator that takes a closure which contains
1719  *       the prefix to write before the peer.  Then this could be used
1720  *       for blacklisting multiple transports and writing the friend
1721  *       file.  I'm sure *someone* will complain loudly about other
1722  *       things that negate these functions even existing so no point in
1723  *       "fixing" now.
1724  */
1725 static int
1726 friend_file_iterator (void *cls,
1727                   const GNUNET_HashCode * key,
1728                   void *value)
1729 {
1730   FILE *temp_friend_handle = cls;
1731   struct GNUNET_TESTING_Daemon *peer = value;
1732   struct GNUNET_PeerIdentity *temppeer;
1733   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1734
1735   temppeer = &peer->id;
1736   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1737   fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
1738
1739   return GNUNET_YES;
1740 }
1741
1742 struct BlacklistContext
1743 {
1744   /*
1745    * The (open) file handle to write to
1746    */
1747   FILE *temp_file_handle;
1748
1749   /*
1750    * The transport that this peer will be blacklisted on.
1751    */
1752   char *transport;
1753 };
1754
1755 /**
1756  * Iterator for writing blacklist data to appropriate files.
1757  *
1758  * @param cls closure, an open writable file handle
1759  * @param key the key the daemon was stored under
1760  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1761  *
1762  * @return GNUNET_YES to continue iteration
1763  */
1764 static int
1765 blacklist_file_iterator (void *cls,
1766                          const GNUNET_HashCode * key,
1767                          void *value)
1768 {
1769   struct BlacklistContext *blacklist_ctx = cls;
1770   //FILE *temp_blacklist_handle = cls;
1771   struct GNUNET_TESTING_Daemon *peer = value;
1772   struct GNUNET_PeerIdentity *temppeer;
1773   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1774
1775   temppeer = &peer->id;
1776   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1777   fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
1778
1779   return GNUNET_YES;
1780 }
1781
1782 /*
1783  * Create the friend files based on the PeerConnection's
1784  * of each peer in the peer group, and copy the files
1785  * to the appropriate place
1786  *
1787  * @param pg the peer group we are dealing with
1788  */
1789 static int
1790 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
1791 {
1792   FILE *temp_friend_handle;
1793   unsigned int pg_iter;
1794   char *temp_service_path;
1795   pid_t *pidarr;
1796   char *arg;
1797   char * mytemp;
1798   enum GNUNET_OS_ProcessStatusType type;
1799   unsigned long return_code;
1800   int count;
1801   int ret;
1802   int max_wait = 10;
1803
1804   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1805   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1806     {
1807       mytemp = GNUNET_DISK_mktemp("friends");
1808       GNUNET_assert(mytemp != NULL);
1809       temp_friend_handle = fopen (mytemp, "wt");
1810       GNUNET_assert(temp_friend_handle != NULL);
1811       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
1812       fclose(temp_friend_handle);
1813
1814       if (GNUNET_OK !=
1815           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1816         {
1817           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1818                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1819                       "SERVICEHOME",
1820                       "PATHS");
1821           if (UNLINK (mytemp) != 0)
1822             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1823           GNUNET_free (mytemp);
1824           break;
1825         }
1826
1827       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1828         {
1829           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
1830           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1831                                          "mv", mytemp, arg, NULL);
1832 #if VERBOSE_TESTING
1833           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1834                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1835 #endif
1836
1837           GNUNET_free(arg);
1838         }
1839       else /* Remote, scp the file to the correct place */
1840         {
1841           if (NULL != pg->peers[pg_iter].daemon->username)
1842             GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1843           else
1844             GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1845           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1846                                          "scp", mytemp, arg, NULL);
1847
1848 #if VERBOSE_TESTING
1849           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1850                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1851 #endif
1852           GNUNET_free(arg);
1853         }
1854       GNUNET_free (temp_service_path);
1855       GNUNET_free (mytemp);
1856     }
1857
1858   count = 0;
1859   ret = GNUNET_SYSERR;
1860   while ((count < max_wait) && (ret != GNUNET_OK))
1861     {
1862       ret = GNUNET_OK;
1863       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1864         {
1865 #if VERBOSE_TESTING
1866           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1867                       _("Checking copy status of file %d\n"), pg_iter);
1868 #endif
1869           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1870             {
1871               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1872                 {
1873                   ret = GNUNET_SYSERR;
1874                 }
1875               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1876                 {
1877                   ret = GNUNET_SYSERR;
1878                 }
1879               else
1880                 {
1881                   pidarr[pg_iter] = 0;
1882 #if VERBOSE_TESTING
1883             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1884                       _("File %d copied\n"), pg_iter);
1885 #endif
1886                 }
1887             }
1888         }
1889       count++;
1890       if (ret == GNUNET_SYSERR)
1891         {
1892           /* FIXME: why sleep here? -CG */
1893           sleep(1);
1894         }
1895     }
1896
1897 #if VERBOSE_TESTING
1898     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1899                 _("Finished copying all friend files!\n"));
1900 #endif
1901   GNUNET_free(pidarr);
1902   return ret;
1903 }
1904
1905
1906 /*
1907  * Create the blacklist files based on the PeerConnection's
1908  * of each peer in the peer group, and copy the files
1909  * to the appropriate place.
1910  *
1911  * @param pg the peer group we are dealing with
1912  * @param transports space delimited list of transports to blacklist
1913  */
1914 static int
1915 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
1916 {
1917   FILE *temp_file_handle;
1918   static struct BlacklistContext blacklist_ctx;
1919   unsigned int pg_iter;
1920   char *temp_service_path;
1921   pid_t *pidarr;
1922   char *arg;
1923   char *mytemp;
1924   enum GNUNET_OS_ProcessStatusType type;
1925   unsigned long return_code;
1926   int count;
1927   int ret;
1928   int max_wait = 10;
1929   int transport_len;
1930   unsigned int i;
1931   char *pos;
1932   char *temp_transports;
1933
1934   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1935   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1936     {
1937       mytemp = GNUNET_DISK_mktemp("blacklist");
1938       GNUNET_assert(mytemp != NULL);
1939       temp_file_handle = fopen (mytemp, "wt");
1940       GNUNET_assert(temp_file_handle != NULL);
1941       temp_transports = GNUNET_strdup(transports);
1942       blacklist_ctx.temp_file_handle = temp_file_handle;
1943       transport_len = strlen(temp_transports) + 1;
1944       pos = NULL;
1945
1946       for (i = 0; i < transport_len; i++)
1947       {
1948         if ((temp_transports[i] == ' ') && (pos == NULL))
1949           continue; /* At start of string (whitespace) */
1950         else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
1951         {
1952           temp_transports[i] = '\0';
1953           blacklist_ctx.transport = pos;
1954           GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
1955           pos = NULL;
1956         } /* At beginning of actual string */
1957         else if (pos == NULL)
1958         {
1959           pos = &temp_transports[i];
1960         }
1961       }
1962
1963       GNUNET_free (temp_transports);
1964       fclose(temp_file_handle);
1965
1966       if (GNUNET_OK !=
1967           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1968         {
1969           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1970                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1971                       "SERVICEHOME",
1972                       "PATHS");
1973           if (UNLINK (mytemp) != 0)
1974             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1975           GNUNET_free (mytemp);
1976           break;
1977         }
1978
1979       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1980         {
1981           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
1982           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1983                                          "mv", mytemp, arg, NULL);
1984 #if VERBOSE_TESTING
1985           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1986                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1987 #endif
1988
1989           GNUNET_free(arg);
1990         }
1991       else /* Remote, scp the file to the correct place */
1992         {
1993           if (NULL != pg->peers[pg_iter].daemon->username)
1994             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1995           else
1996             GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1997           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1998                                          "scp", mytemp, arg, NULL);
1999
2000 #if VERBOSE_TESTING
2001           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2002                       _("Copying file with command scp %s %s\n"), mytemp, arg);
2003 #endif
2004           GNUNET_free(arg);
2005         }
2006       GNUNET_free (temp_service_path);
2007       GNUNET_free (mytemp);
2008     }
2009
2010   count = 0;
2011   ret = GNUNET_SYSERR;
2012   while ((count < max_wait) && (ret != GNUNET_OK))
2013     {
2014       ret = GNUNET_OK;
2015       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2016         {
2017 #if VERBOSE_TESTING
2018           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2019                       _("Checking copy status of file %d\n"), pg_iter);
2020 #endif
2021           if (pidarr[pg_iter] != 0) /* Check for already completed! */
2022             {
2023               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
2024                 {
2025                   ret = GNUNET_SYSERR;
2026                 }
2027               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
2028                 {
2029                   ret = GNUNET_SYSERR;
2030                 }
2031               else
2032                 {
2033                   pidarr[pg_iter] = 0;
2034 #if VERBOSE_TESTING
2035             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2036                       _("File %d copied\n"), pg_iter);
2037 #endif
2038                 }
2039             }
2040         }
2041       count++;
2042       if (ret == GNUNET_SYSERR)
2043         {
2044           /* FIXME: why sleep here? -CG */
2045           sleep(1);
2046         }
2047     }
2048
2049 #if VERBOSE_TESTING
2050     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2051                 _("Finished copying all blacklist files!\n"));
2052 #endif
2053   GNUNET_free(pidarr);
2054   return ret;
2055 }
2056
2057
2058 /**
2059  * Internal notification of a connection, kept so that we can ensure some connections
2060  * happen instead of flooding all testing daemons with requests to connect.
2061  */
2062 static void internal_connect_notify (void *cls,
2063                                      const struct GNUNET_PeerIdentity *first,
2064                                      const struct GNUNET_PeerIdentity *second,
2065                                      uint32_t distance,
2066                                      const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2067                                      const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2068                                      struct GNUNET_TESTING_Daemon *first_daemon,
2069                                      struct GNUNET_TESTING_Daemon *second_daemon,
2070                                      const char *emsg)
2071 {
2072   struct GNUNET_TESTING_PeerGroup *pg = cls;
2073   outstanding_connects--;
2074
2075   pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
2076 }
2077
2078
2079 /**
2080  * Either delay a connection (because there are too many outstanding)
2081  * or schedule it for right now.
2082  *
2083  * @param cls a connection context
2084  * @param tc the task runtime context
2085  */
2086 static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2087 {
2088   struct ConnectContext *connect_context = cls;
2089
2090   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2091     return;
2092
2093   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2094     {
2095 #if VERBOSE_TESTING > 2
2096           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2097                       _("Delaying connect, we have too many outstanding connections!\n"));
2098 #endif
2099       GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
2100     }
2101   else
2102     {
2103 #if VERBOSE_TESTING > 2
2104           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2105                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2106 #endif
2107       outstanding_connects++;
2108       GNUNET_TESTING_daemons_connect (connect_context->first,
2109                                       connect_context->second,
2110                                       CONNECT_TIMEOUT,
2111                                       CONNECT_ATTEMPTS,
2112                                       &internal_connect_notify,
2113                                       connect_context->pg);
2114       GNUNET_free(connect_context);
2115     }
2116 }
2117
2118
2119 /**
2120  * Iterator for actually scheduling connections to be created
2121  * between two peers.
2122  *
2123  * @param cls closure, a GNUNET_TESTING_Daemon
2124  * @param key the key the second Daemon was stored under
2125  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2126  *
2127  * @return GNUNET_YES to continue iteration
2128  */
2129 static int
2130 connect_iterator (void *cls,
2131                   const GNUNET_HashCode * key,
2132                   void *value)
2133 {
2134   struct PeerData *first = cls;
2135   struct GNUNET_TESTING_Daemon *second = value;
2136   struct ConnectContext *connect_context;
2137
2138   connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2139   connect_context->pg = first->pg;
2140   connect_context->first = first->daemon;
2141   connect_context->second = second;
2142   GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
2143
2144   return GNUNET_YES;
2145 }
2146
2147
2148 /**
2149  * Iterator for copying all entries in the allowed hashmap to the
2150  * connect hashmap.
2151  *
2152  * @param cls closure, a GNUNET_TESTING_Daemon
2153  * @param key the key the second Daemon was stored under
2154  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2155  *
2156  * @return GNUNET_YES to continue iteration
2157  */
2158 static int
2159 copy_topology_iterator (void *cls,
2160                   const GNUNET_HashCode * key,
2161                   void *value)
2162 {
2163   struct PeerData *first = cls;
2164
2165   GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2166
2167   return GNUNET_YES;
2168 }
2169
2170 /**
2171  * Make the peers to connect the same as those that are allowed to be
2172  * connected.
2173  *
2174  * @param pg the peer group
2175  */
2176 static int
2177 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2178 {
2179   unsigned int pg_iter;
2180   int ret;
2181   int total;
2182
2183   total = 0;
2184   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2185     {
2186       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
2187       if (GNUNET_SYSERR == ret)
2188         return GNUNET_SYSERR;
2189
2190       total = total + ret;
2191     }
2192
2193   return total;
2194 }
2195
2196
2197 /**
2198  * Connect the topology as specified by the PeerConnection's
2199  * of each peer in the peer group
2200  *
2201  * @param pg the peer group we are dealing with
2202  * @return the number of connections that will be attempted
2203  */
2204 static int
2205 connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
2206 {
2207   unsigned int pg_iter;
2208   int ret;
2209   int total;
2210 #if OLD
2211   struct PeerConnection *connection_iter;
2212   struct ConnectContext *connect_context;
2213 #endif
2214
2215   total = 0;
2216   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2217     {
2218       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
2219       if (GNUNET_SYSERR == ret)
2220         return GNUNET_SYSERR;
2221
2222       total = total + ret;
2223
2224 #if OLD
2225       connection_iter = ;
2226       while (connection_iter != NULL)
2227         {
2228           connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2229           connect_context->pg = pg;
2230           connect_context->first = ;
2231           connect_context->second = connection_iter->daemon;
2232           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
2233           connection_iter = connection_iter->next;
2234         }
2235 #endif
2236     }
2237   return total;
2238 }
2239
2240
2241 /**
2242  * Takes a peer group and creates a topology based on the
2243  * one specified.  Creates a topology means generates friend
2244  * files for the peers so they can only connect to those allowed
2245  * by the topology.  This will only have an effect once peers
2246  * are started if the FRIENDS_ONLY option is set in the base
2247  * config.  Also takes an optional restrict topology which
2248  * disallows connections based on a particular transport
2249  * UNLESS they are specified in the restricted topology.
2250  *
2251  * @param pg the peer group struct representing the running peers
2252  * @param topology which topology to connect the peers in
2253  * @param restrict_topology allow only direct TCP connections in this topology
2254  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2255  * @param restrict_transports space delimited list of transports to blacklist
2256  *                            to create restricted topology
2257  *
2258  * @return the maximum number of connections were all allowed peers
2259  *         connected to each other
2260  */
2261 int
2262 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2263                                 enum GNUNET_TESTING_Topology topology,
2264                                 enum GNUNET_TESTING_Topology restrict_topology,
2265                                 char *restrict_transports)
2266 {
2267   int ret;
2268   int num_connections;
2269   int unblacklisted_connections;
2270
2271   GNUNET_assert (pg->notify_connection != NULL);
2272
2273   switch (topology)
2274     {
2275     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2276 #if VERBOSE_TESTING
2277       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2278                   _("Creating clique topology\n"));
2279 #endif
2280       num_connections = create_clique (pg, &add_allowed_connections);
2281       break;
2282     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2283 #if VERBOSE_TESTING
2284       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2285                   _("Creating small world (ring) topology\n"));
2286 #endif
2287       num_connections = create_small_world_ring (pg, &add_allowed_connections);
2288       break;
2289     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2290 #if VERBOSE_TESTING
2291       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2292                   _("Creating small world (2d-torus) topology\n"));
2293 #endif
2294       num_connections = create_small_world (pg, &add_allowed_connections);
2295       break;
2296     case GNUNET_TESTING_TOPOLOGY_RING:
2297 #if VERBOSE_TESTING
2298       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2299                   _("Creating ring topology\n"));
2300 #endif
2301       num_connections = create_ring (pg, &add_allowed_connections);
2302       break;
2303     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2304 #if VERBOSE_TESTING
2305       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2306                   _("Creating 2d torus topology\n"));
2307 #endif
2308       num_connections = create_2d_torus (pg, &add_allowed_connections);
2309       break;
2310     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2311 #if VERBOSE_TESTING
2312       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2313                   _("Creating Erdos-Renyi topology\n"));
2314 #endif
2315       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2316       break;
2317     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2318 #if VERBOSE_TESTING
2319       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2320                   _("Creating InterNAT topology\n"));
2321 #endif
2322       num_connections = create_nated_internet (pg, &add_allowed_connections);
2323       break;
2324     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2325 #if VERBOSE_TESTING
2326       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2327                   _("Creating Scale Free topology\n"));
2328 #endif
2329       num_connections = create_scale_free (pg, &add_allowed_connections);
2330       break;
2331     case GNUNET_TESTING_TOPOLOGY_LINE:
2332 #if VERBOSE_TESTING
2333       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2334                   _("Creating straight line topology\n"));
2335 #endif
2336       num_connections = create_line (pg, &add_allowed_connections);
2337       break;
2338     case GNUNET_TESTING_TOPOLOGY_NONE:
2339 #if VERBOSE_TESTING
2340       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2341                   _("Creating no allowed topology (all peers can connect at core level)\n"));
2342 #endif
2343       num_connections = 0;
2344       break;
2345     default:
2346       num_connections = 0;
2347       break;
2348     }
2349
2350   if (num_connections < 0)
2351     return GNUNET_SYSERR;
2352
2353   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2354     {
2355       ret = create_and_copy_friend_files(pg);
2356       if (ret != GNUNET_OK)
2357         {
2358 #if VERBOSE_TESTING
2359           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2360                       _("Failed during friend file copying!\n"));
2361 #endif
2362           return GNUNET_SYSERR;
2363         }
2364       else
2365         {
2366 #if VERBOSE_TESTING
2367               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2368                           _("Friend files created/copied successfully!\n"));
2369 #endif
2370         }
2371     }
2372
2373   /* Use the create clique method to initially set all connections as blacklisted. */
2374   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2375     create_clique (pg, &blacklist_connections);
2376   unblacklisted_connections = 0;
2377   /* Un-blacklist connections as per the topology specified */
2378   switch (restrict_topology)
2379     {
2380     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2381 #if VERBOSE_TESTING
2382       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2383                   _("Blacklisting all but clique topology\n"));
2384 #endif
2385       unblacklisted_connections = create_clique (pg, &unblacklist_connections);
2386       break;
2387     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2388 #if VERBOSE_TESTING
2389       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2390                   _("Blacklisting all but small world (ring) topology\n"));
2391 #endif
2392       unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
2393       break;
2394     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2395 #if VERBOSE_TESTING
2396       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2397                   _("Blacklisting all but small world (2d-torus) topology\n"));
2398 #endif
2399       unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
2400       break;
2401     case GNUNET_TESTING_TOPOLOGY_RING:
2402 #if VERBOSE_TESTING
2403       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2404                   _("Blacklisting all but ring topology\n"));
2405 #endif
2406       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2407       break;
2408     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2409 #if VERBOSE_TESTING
2410       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2411                   _("Blacklisting all but 2d torus topology\n"));
2412 #endif
2413       unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
2414       break;
2415     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2416 #if VERBOSE_TESTING
2417       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2418                   _("Blacklisting all but Erdos-Renyi topology\n"));
2419 #endif
2420       unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
2421       break;
2422     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2423 #if VERBOSE_TESTING
2424       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2425                   _("Blacklisting all but InterNAT topology\n"));
2426 #endif
2427       unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
2428       break;
2429     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2430 #if VERBOSE_TESTING
2431       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2432                   _("Blacklisting all but Scale Free topology\n"));
2433 #endif
2434       unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
2435       break;
2436     case GNUNET_TESTING_TOPOLOGY_LINE:
2437 #if VERBOSE_TESTING
2438       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2439                   _("Blacklisting all but straight line topology\n"));
2440 #endif
2441       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2442       break;
2443     case GNUNET_TESTING_TOPOLOGY_NONE:
2444 #if VERBOSE_TESTING
2445       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2446                   _("Creating no blacklist topology (all peers can connect at transport level)\n"));
2447 #endif
2448     default:
2449       break;
2450     }
2451
2452   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2453   {
2454     ret = create_and_copy_blacklist_files(pg, restrict_transports);
2455     if (ret != GNUNET_OK)
2456       {
2457 #if VERBOSE_TESTING
2458         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2459                     _("Failed during blacklist file copying!\n"));
2460 #endif
2461         return GNUNET_SYSERR;
2462       }
2463     else
2464       {
2465 #if VERBOSE_TESTING
2466         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2467                     _("Blacklist files created/copied successfully!\n"));
2468 #endif
2469       }
2470   }
2471   return num_connections;
2472 }
2473
2474 struct RandomContext
2475 {
2476   /**
2477    * The peergroup
2478    */
2479   struct GNUNET_TESTING_PeerGroup *pg;
2480
2481   /**
2482    * uid of the first peer
2483    */
2484   uint32_t first_uid;
2485
2486   /**
2487    * Peer data for first peer.
2488    */
2489   struct PeerData *first;
2490
2491   /**
2492    * Random percentage to use
2493    */
2494   double percentage;
2495 };
2496
2497 struct MinimumContext
2498 {
2499   /**
2500    * The peergroup
2501    */
2502   struct GNUNET_TESTING_PeerGroup *pg;
2503
2504   /**
2505    * uid of the first peer
2506    */
2507   uint32_t first_uid;
2508
2509   /**
2510    * Peer data for first peer.
2511    */
2512   struct PeerData *first;
2513
2514   /**
2515    * Number of conns per peer
2516    */
2517   unsigned int num_to_add;
2518
2519   /**
2520    * Permuted array of all possible connections.  Only add the Nth
2521    * peer if it's in the Nth position.
2522    */
2523   unsigned int *pg_array;
2524
2525   /**
2526    * What number is the current element we are iterating over?
2527    */
2528   unsigned int current;
2529 };
2530
2531 struct DFSContext
2532 {
2533   /**
2534    * The peergroup
2535    */
2536   struct GNUNET_TESTING_PeerGroup *pg;
2537
2538   /**
2539    * uid of the first peer
2540    */
2541   uint32_t first_uid;
2542
2543   /**
2544    * uid of the second peer
2545    */
2546   uint32_t second_uid;
2547
2548   /**
2549    * Peer data for first peer.
2550    */
2551   struct PeerData *first;
2552
2553   /**
2554    * Which peer has been chosen as the one to add?
2555    */
2556   unsigned int chosen;
2557
2558   /**
2559    * What number is the current element we are iterating over?
2560    */
2561   unsigned int current;
2562 };
2563
2564 /**
2565  * Iterator for choosing random peers to connect.
2566  *
2567  * @param cls closure, a RandomContext
2568  * @param key the key the second Daemon was stored under
2569  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2570  *
2571  * @return GNUNET_YES to continue iteration
2572  */
2573 static int
2574 random_connect_iterator (void *cls,
2575                   const GNUNET_HashCode * key,
2576                   void *value)
2577 {
2578   struct RandomContext *random_ctx = cls;
2579   double random_number;
2580   uint32_t second_pos;
2581   GNUNET_HashCode first_hash;
2582   random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
2583                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
2584   if (random_number < random_ctx->percentage)
2585   {
2586     GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2587   }
2588   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2589   uid_from_hash(key, &second_pos);
2590   hash_from_uid(random_ctx->first_uid, &first_hash);
2591   GNUNET_assert(random_ctx->pg->total > second_pos);
2592   GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
2593
2594   return GNUNET_YES;
2595 }
2596
2597 /**
2598  * Iterator for adding at least X peers to a peers connection set.
2599  *
2600  * @param cls closure, MinimumContext
2601  * @param key the key the second Daemon was stored under
2602  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2603  *
2604  * @return GNUNET_YES to continue iteration
2605  */
2606 static int
2607 minimum_connect_iterator (void *cls,
2608                   const GNUNET_HashCode * key,
2609                   void *value)
2610 {
2611   struct MinimumContext *min_ctx = cls;
2612   uint32_t second_pos;
2613   GNUNET_HashCode first_hash;
2614   unsigned int i;
2615
2616   if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2617   {
2618     for (i = 0; i < min_ctx->num_to_add; i++)
2619     {
2620       if (min_ctx->pg_array[i] == min_ctx->current)
2621       {
2622         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2623         uid_from_hash(key, &second_pos);
2624         hash_from_uid(min_ctx->first_uid, &first_hash);
2625         GNUNET_assert(min_ctx->pg->total > second_pos);
2626         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2627         /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2628         GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
2629       }
2630     }
2631     min_ctx->current++;
2632     return GNUNET_YES;
2633   }
2634   else
2635     return GNUNET_NO; /* We can stop iterating, we have enough peers! */
2636
2637 }
2638
2639
2640 /**
2641  * Iterator for adding peers to a connection set based on a depth first search.
2642  *
2643  * @param cls closure, MinimumContext
2644  * @param key the key the second daemon was stored under
2645  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2646  *
2647  * @return GNUNET_YES to continue iteration
2648  */
2649 static int
2650 dfs_connect_iterator (void *cls,
2651                   const GNUNET_HashCode * key,
2652                   void *value)
2653 {
2654   struct DFSContext *dfs_ctx = cls;
2655   GNUNET_HashCode first_hash;
2656
2657   if (dfs_ctx->current == dfs_ctx->chosen)
2658     {
2659       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2660       uid_from_hash(key, &dfs_ctx->second_uid);
2661       hash_from_uid(dfs_ctx->first_uid, &first_hash);
2662       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2663       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
2664       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
2665       return GNUNET_NO; /* We have found our peer, don't iterate more */
2666     }
2667
2668   dfs_ctx->current++;
2669   return GNUNET_YES;
2670 }
2671
2672
2673 /**
2674  * From the set of connections possible, choose percentage percent of connections
2675  * to actually connect.
2676  *
2677  * @param pg the peergroup we are dealing with
2678  * @param percentage what percent of total connections to make
2679  */
2680 void
2681 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
2682 {
2683   struct RandomContext random_ctx;
2684   uint32_t pg_iter;
2685
2686   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2687     {
2688       random_ctx.first_uid = pg_iter;
2689       random_ctx.first = &pg->peers[pg_iter];
2690       random_ctx.percentage = percentage;
2691       random_ctx.pg = pg;
2692       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
2693       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
2694       /* Now remove the old connections */
2695       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2696       /* And replace with the random set */
2697       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2698     }
2699 }
2700
2701 /**
2702  * From the set of connections possible, choose at least num connections per
2703  * peer.
2704  *
2705  * @param pg the peergroup we are dealing with
2706  * @param num how many connections at least should each peer have (if possible)?
2707  */
2708 static void
2709 choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2710 {
2711   struct MinimumContext minimum_ctx;
2712   uint32_t pg_iter;
2713
2714   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2715    {
2716      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2717    }
2718
2719   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2720     {
2721       minimum_ctx.first_uid = pg_iter;
2722       minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK,
2723                                                           GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
2724       minimum_ctx.first = &pg->peers[pg_iter];
2725       minimum_ctx.pg = pg;
2726       minimum_ctx.num_to_add = num;
2727       minimum_ctx.current = 0;
2728       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
2729                                             &minimum_connect_iterator,
2730                                             &minimum_ctx);
2731     }
2732
2733   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2734     {
2735       /* Remove the "old" connections */
2736       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2737       /* And replace with the working set */
2738       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2739     }
2740
2741 }
2742
2743
2744 static unsigned int
2745 count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
2746 {
2747   unsigned int count;
2748   unsigned int pg_iter;
2749
2750   count = 0;
2751
2752   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2753     {
2754       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2755     }
2756
2757   return count;
2758 }
2759
2760
2761 static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
2762 {
2763   unsigned int count;
2764   unsigned int pg_iter;
2765
2766   count = 0;
2767
2768   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2769     {
2770       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
2771     }
2772
2773   return count;
2774 }
2775
2776 /**
2777  * From the set of connections possible, choose at least num connections per
2778  * peer based on depth first traversal of peer connections.  If DFS leaves
2779  * peers unconnected, ensure those peers get connections.
2780  *
2781  * @param pg the peergroup we are dealing with
2782  * @param num how many connections at least should each peer have (if possible)?
2783  */
2784 void
2785 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2786 {
2787   struct DFSContext dfs_ctx;
2788   uint32_t pg_iter;
2789   uint32_t dfs_count;
2790   uint32_t starting_peer;
2791   uint32_t least_connections;
2792   GNUNET_HashCode second_hash;
2793
2794   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2795     {
2796       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2797     }
2798
2799   starting_peer = 0;
2800   dfs_count = 0;
2801   while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
2802     {
2803       if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
2804         {
2805           least_connections = -1; /* Set to very high number */
2806           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2807             {
2808               if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
2809                 {
2810                   starting_peer = pg_iter;
2811                   least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2812                 }
2813             }
2814         }
2815
2816       if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
2817         {
2818           dfs_count = 0;
2819           continue;
2820         }
2821
2822       /* Choose a random peer from the chosen peers set of connections to add */
2823       dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
2824       dfs_ctx.first_uid = starting_peer;
2825       dfs_ctx.first = &pg->peers[starting_peer];
2826       dfs_ctx.pg = pg;
2827       dfs_ctx.current = 0;
2828
2829       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
2830       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
2831       hash_from_uid(dfs_ctx.second_uid, &second_hash);
2832       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
2833       starting_peer = dfs_ctx.second_uid;
2834     }
2835
2836   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2837     {
2838       /* Remove the "old" connections */
2839       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2840       /* And replace with the working set */
2841       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2842     }
2843 }
2844
2845 /**
2846  * Internal callback for topology information for a particular peer.
2847  */
2848 static void
2849 internal_topology_callback(void *cls,
2850                            const struct GNUNET_PeerIdentity *peer,
2851                            struct GNUNET_TIME_Relative latency, uint32_t distance)
2852 {
2853   struct CoreContext *core_ctx = cls;
2854   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
2855
2856   if (peer == NULL) /* Either finished, or something went wrong */
2857     {
2858       iter_ctx->completed++;
2859       iter_ctx->connected--;
2860       /* One core context allocated per iteration, must free! */
2861       GNUNET_free(core_ctx);
2862     }
2863   else
2864     {
2865       iter_ctx->topology_cb(iter_ctx->cls, &core_ctx->daemon->id, peer, latency, distance, NULL);
2866     }
2867
2868   if (iter_ctx->completed == iter_ctx->total)
2869     {
2870       iter_ctx->topology_cb(iter_ctx->cls, NULL, NULL, GNUNET_TIME_relative_get_zero(), 0, NULL);
2871       /* Once all are done, free the iteration context */
2872       GNUNET_free(iter_ctx);
2873     }
2874 }
2875
2876
2877 /**
2878  * Check running topology iteration tasks, if below max start a new one, otherwise
2879  * schedule for some time in the future.
2880  */
2881 static void
2882 schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2883 {
2884   struct CoreContext *core_context = cls;
2885   struct TopologyIterateContext *topology_context = (struct TopologyIterateContext *)core_context->iter_context;
2886   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2887     return;
2888
2889   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2890     {
2891 #if VERBOSE_TESTING > 2
2892           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2893                       _("Delaying connect, we have too many outstanding connections!\n"));
2894 #endif
2895       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_topology, core_context);
2896     }
2897   else
2898     {
2899 #if VERBOSE_TESTING > 2
2900           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2901                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2902 #endif
2903       topology_context->connected++;
2904       if (GNUNET_OK != GNUNET_CORE_iterate_peers (core_context->daemon->sched, core_context->daemon->cfg, &internal_topology_callback, core_context))
2905         internal_topology_callback(core_context, NULL, GNUNET_TIME_relative_get_zero(), 0);
2906
2907     }
2908 }
2909
2910 /**
2911  * Iterate over all (running) peers in the peer group, retrieve
2912  * all connections that each currently has.
2913  */
2914 void
2915 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyTopology cb, void *cls)
2916 {
2917   struct TopologyIterateContext *topology_context;
2918   struct CoreContext *core_ctx;
2919   unsigned int i;
2920   unsigned int total_count;
2921
2922   /* Allocate a single topology iteration context */
2923   topology_context = GNUNET_malloc(sizeof(struct TopologyIterateContext));
2924   topology_context->topology_cb = cb;
2925   topology_context->cls = cls;
2926   total_count = 0;
2927   for (i = 0; i < pg->total; i++)
2928     {
2929       if (pg->peers[i].daemon->running == GNUNET_YES)
2930         {
2931           /* Allocate one core context per core we need to connect to */
2932           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
2933           core_ctx->daemon = pg->peers[i].daemon;
2934           /* Set back pointer to topology iteration context */
2935           core_ctx->iter_context = topology_context;
2936           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_topology, core_ctx);
2937           total_count++;
2938         }
2939     }
2940   topology_context->total = total_count;
2941   return;
2942 }
2943
2944 /**
2945  * Callback function to process statistic values.
2946  * This handler is here only really to insert a peer
2947  * identity (or daemon) so the statistics can be uniquely
2948  * tied to a single running peer.
2949  *
2950  * @param cls closure
2951  * @param subsystem name of subsystem that created the statistic
2952  * @param name the name of the datum
2953  * @param value the current value
2954  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
2955  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
2956  */
2957 static int internal_stats_callback (void *cls,
2958                                     const char *subsystem,
2959                                     const char *name,
2960                                     uint64_t value,
2961                                     int is_persistent)
2962 {
2963   struct StatsCoreContext *core_context = cls;
2964   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2965
2966   return stats_context->proc(stats_context->cls, &core_context->daemon->id, subsystem, name, value, is_persistent);
2967 }
2968
2969 /**
2970  * Internal continuation call for statistics iteration.
2971  *
2972  * @param cls closure, the CoreContext for this iteration
2973  * @param success whether or not the statistics iterations
2974  *        was canceled or not (we don't care)
2975  */
2976 static void internal_stats_cont (void *cls, int success)
2977 {
2978   struct StatsCoreContext *core_context = cls;
2979   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2980
2981   stats_context->connected--;
2982   stats_context->completed++;
2983
2984   if (stats_context->completed == stats_context->total)
2985     {
2986       stats_context->cont(stats_context->cls, GNUNET_YES);
2987       GNUNET_free(stats_context);
2988     }
2989
2990   if (core_context->stats_handle != NULL)
2991     GNUNET_STATISTICS_destroy(core_context->stats_handle, GNUNET_NO);
2992
2993   GNUNET_free(core_context);
2994 }
2995
2996 /**
2997  * Check running topology iteration tasks, if below max start a new one, otherwise
2998  * schedule for some time in the future.
2999  */
3000 static void
3001 schedule_get_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
3002 {
3003   struct StatsCoreContext *core_context = cls;
3004   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
3005
3006   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3007     return;
3008
3009   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3010     {
3011 #if VERBOSE_TESTING > 2
3012           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3013                       _("Delaying connect, we have too many outstanding connections!\n"));
3014 #endif
3015       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_statistics, core_context);
3016     }
3017   else
3018     {
3019 #if VERBOSE_TESTING > 2
3020           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3021                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
3022 #endif
3023
3024       stats_context->connected++;
3025       core_context->stats_handle = GNUNET_STATISTICS_create(core_context->daemon->sched, "testing", core_context->daemon->cfg);
3026       if (core_context->stats_handle == NULL)
3027         {
3028           internal_stats_cont (core_context, GNUNET_NO);
3029           return;
3030         }
3031
3032       core_context->stats_get_handle = GNUNET_STATISTICS_get(core_context->stats_handle, NULL, NULL, GNUNET_TIME_relative_get_forever(), &internal_stats_cont, &internal_stats_callback, core_context);
3033       if (core_context->stats_get_handle == NULL)
3034         internal_stats_cont (core_context, GNUNET_NO);
3035
3036     }
3037 }
3038
3039
3040 /**
3041  * Iterate over all (running) peers in the peer group, retrieve
3042  * all statistics from each.
3043  */
3044 void
3045 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3046                                GNUNET_STATISTICS_Callback cont,
3047                                GNUNET_TESTING_STATISTICS_Iterator proc, void *cls)
3048 {
3049   struct StatsIterateContext *stats_context;
3050   struct StatsCoreContext *core_ctx;
3051   unsigned int i;
3052   unsigned int total_count;
3053
3054   /* Allocate a single stats iteration context */
3055   stats_context = GNUNET_malloc(sizeof(struct StatsIterateContext));
3056   stats_context->cont = cont;
3057   stats_context->proc = proc;
3058   stats_context->cls = cls;
3059   total_count = 0;
3060   for (i = 0; i < pg->total; i++)
3061     {
3062       if (pg->peers[i].daemon->running == GNUNET_YES)
3063         {
3064           /* Allocate one core context per core we need to connect to */
3065           core_ctx = GNUNET_malloc(sizeof(struct StatsCoreContext));
3066           core_ctx->daemon = pg->peers[i].daemon;
3067           /* Set back pointer to topology iteration context */
3068           core_ctx->iter_context = stats_context;
3069           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_statistics, core_ctx);
3070           total_count++;
3071         }
3072     }
3073   stats_context->total = total_count;
3074   return;
3075 }
3076
3077 /**
3078  * There are many ways to connect peers that are supported by this function.
3079  * To connect peers in the same topology that was created via the
3080  * GNUNET_TESTING_create_topology, the topology variable must be set to
3081  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3082  * a new instance of that topology will be generated and attempted to be
3083  * connected.  This could result in some connections being impossible,
3084  * because some topologies are non-deterministic.
3085  *
3086  * @param pg the peer group struct representing the running peers
3087  * @param topology which topology to connect the peers in
3088  * @param options options for connecting the topology
3089  * @param option_modifier modifier for options that take a parameter
3090  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3091  */
3092 int
3093 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3094                                  enum GNUNET_TESTING_Topology topology,
3095                                  enum GNUNET_TESTING_TopologyOption options,
3096                                  double option_modifier)
3097 {
3098   switch (topology)
3099       {
3100       case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3101 #if VERBOSE_TOPOLOGY
3102       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3103                   _("Creating clique CONNECT topology\n"));
3104 #endif
3105         create_clique (pg, &add_actual_connections);
3106         break;
3107       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3108 #if VERBOSE_TOPOLOGY
3109       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3110                   _("Creating small world (ring) CONNECT topology\n"));
3111 #endif
3112         create_small_world_ring (pg, &add_actual_connections);
3113         break;
3114       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3115 #if VERBOSE_TOPOLOGY
3116       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3117                   _("Creating small world (2d-torus) CONNECT topology\n"));
3118 #endif
3119         create_small_world (pg, &add_actual_connections);
3120         break;
3121       case GNUNET_TESTING_TOPOLOGY_RING:
3122 #if VERBOSE_TOPOLOGY
3123       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3124                   _("Creating ring CONNECT topology\n"));
3125 #endif
3126         create_ring (pg, &add_actual_connections);
3127         break;
3128       case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3129 #if VERBOSE_TOPOLOGY
3130       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3131                   _("Creating 2d torus CONNECT topology\n"));
3132 #endif
3133         create_2d_torus (pg, &add_actual_connections);
3134         break;
3135       case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3136 #if VERBOSE_TOPOLOGY
3137       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3138                   _("Creating Erdos-Renyi CONNECT topology\n"));
3139 #endif
3140         create_erdos_renyi (pg, &add_actual_connections);
3141         break;
3142       case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3143 #if VERBOSE_TOPOLOGY
3144       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3145                   _("Creating InterNAT CONNECT topology\n"));
3146 #endif
3147         create_nated_internet (pg, &add_actual_connections);
3148         break;
3149       case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3150 #if VERBOSE_TOPOLOGY
3151       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3152                   _("Creating Scale Free CONNECT topology\n"));
3153 #endif
3154         create_scale_free (pg, &add_actual_connections);
3155         break;
3156       case GNUNET_TESTING_TOPOLOGY_LINE:
3157 #if VERBOSE_TOPOLOGY
3158       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3159                   _("Creating straight line CONNECT topology\n"));
3160 #endif
3161         create_line (pg, &add_actual_connections);
3162         break;
3163       case GNUNET_TESTING_TOPOLOGY_NONE:
3164 #if VERBOSE_TOPOLOGY
3165         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3166                   _("Creating no CONNECT topology\n"));
3167 #endif
3168         copy_allowed_topology(pg);
3169         break;
3170       default:
3171         GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3172                    _("Unknown topology specification, can't connect peers!\n"));
3173         return GNUNET_SYSERR;
3174       }
3175
3176   switch (options)
3177     {
3178     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3179 #if VERBOSE_TOPOLOGY
3180       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3181                   _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
3182 #endif
3183       choose_random_connections(pg, option_modifier);
3184       break;
3185     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3186 #if VERBOSE_TOPOLOGY
3187       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3188                   _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3189 #endif
3190       choose_minimum(pg, (unsigned int)option_modifier);
3191       break;
3192     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3193 #if VERBOSE_TOPOLOGY
3194       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3195                   _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3196 #endif
3197       perform_dfs(pg, (int)option_modifier);
3198       break;
3199     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3200       break;
3201     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3202       break;
3203     default:
3204       break;
3205     }
3206
3207   return connect_topology(pg);
3208 }
3209
3210 /**
3211  * Callback that is called whenever a hostkey is generated
3212  * for a peer.  Call the real callback and decrement the
3213  * starting counter for the peergroup.
3214  *
3215  * @param cls closure
3216  * @param id identifier for the daemon, NULL on error
3217  * @param d handle for the daemon
3218  * @param emsg error message (NULL on success)
3219  */
3220 static void internal_hostkey_callback (void *cls,
3221                                        const struct GNUNET_PeerIdentity *id,
3222                                        struct GNUNET_TESTING_Daemon *d,
3223                                        const char *emsg)
3224 {
3225   struct InternalStartContext *internal_context = cls;
3226   internal_context->peer->pg->starting--;
3227   internal_context->peer->pg->started++;
3228   if (internal_context->hostkey_callback != NULL)
3229     internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
3230   else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
3231     {
3232       internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
3233       GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
3234     }
3235 }
3236
3237 /**
3238  * Callback that is called whenever a peer has finished starting.
3239  * Call the real callback and decrement the starting counter
3240  * for the peergroup.
3241  *
3242  * @param cls closure
3243  * @param id identifier for the daemon, NULL on error
3244  * @param d handle for the daemon
3245  * @param emsg error message (NULL on success)
3246  */
3247 static void internal_startup_callback (void *cls,
3248                                        const struct GNUNET_PeerIdentity *id,
3249                                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3250                                        struct GNUNET_TESTING_Daemon *d,
3251                                        const char *emsg)
3252 {
3253   struct InternalStartContext *internal_context = cls;
3254   internal_context->peer->pg->starting--;
3255   if (internal_context->start_cb != NULL)
3256     internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
3257 }
3258
3259 static void
3260 internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3261 {
3262   struct InternalStartContext *internal_context = cls;
3263
3264   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3265     {
3266       return;
3267     }
3268
3269   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3270     {
3271       internal_context->peer->pg->starting++;
3272       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3273     }
3274   else
3275     {
3276       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
3277     }
3278 }
3279
3280 static void
3281 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3282 {
3283   struct InternalStartContext *internal_context = cls;
3284
3285   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3286     {
3287       return;
3288     }
3289
3290   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
3291     {
3292       internal_context->peer->pg->starting++;
3293       internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
3294                                                                     internal_context->peer->cfg,
3295                                                                     internal_context->timeout,
3296                                                                     internal_context->hostname,
3297                                                                     &internal_hostkey_callback,
3298                                                                     internal_context,
3299                                                                     &internal_startup_callback,
3300                                                                     internal_context);
3301     }
3302   else
3303     {
3304       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
3305     }
3306 }
3307
3308 /**
3309  * Function which continues a peer group starting up
3310  * after successfully generating hostkeys for each peer.
3311  *
3312  * @param pg the peer group to continue starting
3313  *
3314  */
3315 void
3316 GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
3317 {
3318   unsigned int i;
3319
3320   pg->starting = 0;
3321   for (i = 0; i < pg->total; i++)
3322     {
3323       GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
3324       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
3325     }
3326 }
3327
3328 /**
3329  * Start count gnunet instances with the same set of transports and
3330  * applications.  The port numbers (any option called "PORT") will be
3331  * adjusted to ensure that no two peers running on the same system
3332  * have the same port(s) in their respective configurations.
3333  *
3334  * @param sched scheduler to use
3335  * @param cfg configuration template to use
3336  * @param total number of daemons to start
3337  * @param timeout total time allowed for peers to start
3338  * @param hostkey_callback function to call on each peers hostkey generation
3339  *        if NULL, peers will be started by this call, if non-null,
3340  *        GNUNET_TESTING_daemons_continue_startup must be called after
3341  *        successful hostkey generation
3342  * @param hostkey_cls closure for hostkey callback
3343  * @param cb function to call on each daemon that was started
3344  * @param cb_cls closure for cb
3345  * @param connect_callback function to call each time two hosts are connected
3346  * @param connect_callback_cls closure for connect_callback
3347  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
3348  *
3349  * @return NULL on error, otherwise handle to control peer group
3350  */
3351 struct GNUNET_TESTING_PeerGroup *
3352 GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
3353                               const struct GNUNET_CONFIGURATION_Handle *cfg,
3354                               unsigned int total,
3355                               struct GNUNET_TIME_Relative timeout,
3356                               GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
3357                               void *hostkey_cls,
3358                               GNUNET_TESTING_NotifyDaemonRunning cb,
3359                               void *cb_cls,
3360                               GNUNET_TESTING_NotifyConnection
3361                               connect_callback, void *connect_callback_cls,
3362                               const struct GNUNET_TESTING_Host *hostnames)
3363 {
3364   struct GNUNET_TESTING_PeerGroup *pg;
3365   const struct GNUNET_TESTING_Host *hostpos;
3366 #if 0
3367   char *pos;
3368   const char *rpos;
3369   char *start;
3370 #endif
3371   const char *hostname;
3372   char *baseservicehome;
3373   char *newservicehome;
3374   char *tmpdir;
3375   struct GNUNET_CONFIGURATION_Handle *pcfg;
3376   unsigned int off;
3377   unsigned int hostcnt;
3378   uint16_t minport;
3379   uint32_t upnum;
3380
3381   if (0 == total)
3382     {
3383       GNUNET_break (0);
3384       return NULL;
3385     }
3386   upnum = 0;
3387   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
3388   pg->sched = sched;
3389   pg->cfg = cfg;
3390   pg->notify_connection = connect_callback;
3391   pg->notify_connection_cls = connect_callback_cls;
3392   pg->total = total;
3393   pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
3394   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
3395   if (NULL != hostnames)
3396     {
3397       off = 2;
3398       hostpos = hostnames;
3399       while (hostpos != NULL)
3400         {
3401           hostpos = hostpos->next;
3402           off++;
3403         }
3404       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3405       off = 0;
3406
3407       hostpos = hostnames;
3408       while (hostpos != NULL)
3409         {
3410           pg->hosts[off].minport = LOW_PORT;
3411           pg->hosts[off++].hostname = GNUNET_strdup(hostpos->hostname);
3412           hostpos = hostpos->next;
3413         }
3414
3415       if (off == 0)
3416         {
3417           GNUNET_free (pg->hosts);
3418           pg->hosts = NULL;
3419         }
3420       hostcnt = off;
3421       minport = 0;
3422
3423 #if NO_LL
3424       off = 2;
3425       /* skip leading spaces */
3426       while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
3427         hostnames++;
3428       rpos = hostnames;
3429       while ('\0' != *rpos)
3430         {
3431           if (isspace ( (unsigned char) *rpos))
3432             off++;
3433           rpos++;
3434         }
3435       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3436       off = 0;
3437       start = GNUNET_strdup (hostnames);
3438       pos = start;
3439       while ('\0' != *pos)
3440         {
3441           if (isspace ( (unsigned char) *pos))
3442             {
3443               *pos = '\0';
3444               if (strlen (start) > 0)
3445                 {
3446                   pg->hosts[off].minport = LOW_PORT;
3447                   pg->hosts[off++].hostname = start;
3448                 }
3449               start = pos + 1;
3450             }
3451           pos++;
3452         }
3453       if (strlen (start) > 0)
3454         {
3455           pg->hosts[off].minport = LOW_PORT;
3456           pg->hosts[off++].hostname = start;
3457         }
3458       if (off == 0)
3459         {
3460           GNUNET_free (start);
3461           GNUNET_free (pg->hosts);
3462           pg->hosts = NULL;
3463         }
3464       hostcnt = off;
3465       minport = 0;              /* make gcc happy */
3466 #endif
3467     }
3468   else
3469     {
3470       hostcnt = 0;
3471       minport = LOW_PORT;
3472     }
3473   for (off = 0; off < total; off++)
3474     {
3475       if (hostcnt > 0)
3476         {
3477           hostname = pg->hosts[off % hostcnt].hostname;
3478           pcfg = make_config (cfg,
3479                               &pg->hosts[off % hostcnt].minport,
3480                               &upnum,
3481                               hostname);
3482         }
3483       else
3484         {
3485           hostname = NULL;
3486           pcfg = make_config (cfg,
3487                               &minport,
3488                               &upnum,
3489                               hostname);
3490         }
3491
3492       if (NULL == pcfg)
3493         {
3494           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3495                       _
3496                       ("Could not create configuration for peer number %u on `%s'!\n"),
3497                       off, hostname == NULL ? "localhost" : hostname);
3498           continue;
3499         }
3500
3501       if (GNUNET_YES ==
3502           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
3503                                                  &baseservicehome))
3504         {
3505           GNUNET_asprintf (&newservicehome,
3506                            "%s/%d/", baseservicehome, off);
3507           GNUNET_free (baseservicehome);
3508         }
3509       else
3510         {
3511           tmpdir = getenv ("TMPDIR");
3512           tmpdir = tmpdir ? tmpdir : "/tmp";
3513           GNUNET_asprintf (&newservicehome,
3514                            "%s/%s/%d/",
3515                            tmpdir,
3516                            "gnunet-testing-test-test", off);
3517         }
3518       GNUNET_CONFIGURATION_set_value_string (pcfg,
3519                                              "PATHS",
3520                                              "SERVICEHOME", newservicehome);
3521       GNUNET_free (newservicehome);
3522       pg->peers[off].cfg = pcfg;
3523       pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
3524       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
3525       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
3526       pg->peers[off].pg = pg;
3527
3528       pg->peers[off].internal_context.peer = &pg->peers[off];
3529       pg->peers[off].internal_context.timeout = timeout;
3530       pg->peers[off].internal_context.hostname = hostname;
3531       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
3532       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
3533       pg->peers[off].internal_context.start_cb = cb;
3534       pg->peers[off].internal_context.start_cb_cls = cb_cls;
3535
3536       GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
3537
3538     }
3539   return pg;
3540 }
3541
3542 /*
3543  * Get a daemon by number, so callers don't have to do nasty
3544  * offsetting operation.
3545  */
3546 struct GNUNET_TESTING_Daemon *
3547 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
3548 {
3549   if (position < pg->total)
3550     return pg->peers[position].daemon;
3551   else
3552     return NULL;
3553 }
3554
3555 /*
3556  * Get a daemon by peer identity, so callers can
3557  * retrieve the daemon without knowing it's offset.
3558  *
3559  * @param pg the peer group to retrieve the daemon from
3560  * @param peer_id the peer identity of the daemon to retrieve
3561  *
3562  * @return the daemon on success, or NULL if no such peer identity is found
3563  */
3564 struct GNUNET_TESTING_Daemon *
3565 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_PeerIdentity *peer_id)
3566 {
3567   unsigned int i;
3568
3569   for (i = 0; i < pg->total; i ++)
3570     {
3571       if (0 == memcmp(&pg->peers[i].daemon->id, peer_id, sizeof(struct GNUNET_PeerIdentity)))
3572         return pg->peers[i].daemon;
3573     }
3574
3575   return NULL;
3576 }
3577
3578 /**
3579  * Prototype of a function that will be called when a
3580  * particular operation was completed the testing library.
3581  *
3582  * @param cls closure (a struct RestartContext)
3583  * @param id id of the peer that was restarted
3584  * @param cfg handle to the configuration of the peer
3585  * @param d handle to the daemon that was restarted
3586  * @param emsg NULL on success
3587  */
3588 void restart_callback (void *cls,
3589                        const struct GNUNET_PeerIdentity *id,
3590                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3591                        struct GNUNET_TESTING_Daemon *d,
3592                        const char *emsg)
3593 {
3594   struct RestartContext *restart_context = cls;
3595
3596   if (emsg == NULL)
3597     {
3598       restart_context->peers_restarted++;
3599     }
3600   else
3601     {
3602       restart_context->peers_restart_failed++;
3603     }
3604
3605   if (restart_context->peers_restarted == restart_context->peer_group->total)
3606     {
3607       restart_context->callback(restart_context->callback_cls, NULL);
3608       GNUNET_free(restart_context);
3609     }
3610   else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
3611     {
3612       restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
3613       GNUNET_free(restart_context);
3614     }
3615
3616 }
3617
3618 /**
3619  * Callback for informing us about a successful
3620  * or unsuccessful churn stop call.
3621  *
3622  * @param cls a ChurnContext
3623  * @param emsg NULL on success, non-NULL on failure
3624  *
3625  */
3626 void
3627 churn_stop_callback (void *cls, const char *emsg)
3628 {
3629   struct ChurnContext *churn_ctx = cls;
3630   unsigned int total_left;
3631   char *error_message;
3632
3633   error_message = NULL;
3634   if (emsg != NULL)
3635     {
3636       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3637                  "Churn stop callback failed with error `%s'\n", emsg);
3638       churn_ctx->num_failed_stop++;
3639     }
3640   else
3641     {
3642       churn_ctx->num_to_stop--;
3643     }
3644
3645 #if DEBUG_CHURN
3646   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3647              "Stopped peer, %d left.\n",
3648              churn_ctx->num_to_stop);
3649 #endif
3650   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3651
3652   if (total_left == 0)
3653   {
3654     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3655       {
3656         GNUNET_asprintf(&error_message,
3657                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3658                         churn_ctx->num_failed_start,
3659                         churn_ctx->num_failed_stop);
3660       }
3661     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3662     GNUNET_free_non_null(error_message);
3663     GNUNET_free(churn_ctx);
3664   }
3665 }
3666
3667 /**
3668  * Callback for informing us about a successful
3669  * or unsuccessful churn start call.
3670  *
3671  * @param cls a ChurnContext
3672  * @param id the peer identity of the started peer
3673  * @param cfg the handle to the configuration of the peer
3674  * @param d handle to the daemon for the peer
3675  * @param emsg NULL on success, non-NULL on failure
3676  *
3677  */
3678 void
3679 churn_start_callback (void *cls,
3680                       const struct GNUNET_PeerIdentity *id,
3681                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3682                       struct GNUNET_TESTING_Daemon *d,
3683                       const char *emsg)
3684 {
3685   struct ChurnContext *churn_ctx = cls;
3686   unsigned int total_left;
3687   char *error_message;
3688
3689   error_message = NULL;
3690   if (emsg != NULL)
3691     {
3692       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3693                   "Churn stop callback failed with error `%s'\n",
3694                   emsg);
3695       churn_ctx->num_failed_start++;
3696     }
3697   else
3698     {
3699       churn_ctx->num_to_start--;
3700     }
3701
3702 #if DEBUG_CHURN
3703   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3704              "Started peer, %d left.\n",
3705              churn_ctx->num_to_start);
3706 #endif
3707
3708   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3709
3710   if (total_left == 0)
3711   {
3712     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3713       GNUNET_asprintf(&error_message,
3714                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3715                       churn_ctx->num_failed_start,
3716                       churn_ctx->num_failed_stop);
3717     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3718     GNUNET_free_non_null(error_message);
3719     GNUNET_free(churn_ctx);
3720   }
3721 }
3722
3723
3724 /**
3725  * Simulate churn by stopping some peers (and possibly
3726  * re-starting others if churn is called multiple times).  This
3727  * function can only be used to create leave-join churn (peers "never"
3728  * leave for good).  First "voff" random peers that are currently
3729  * online will be taken offline; then "von" random peers that are then
3730  * offline will be put back online.  No notifications will be
3731  * generated for any of these operations except for the callback upon
3732  * completion.
3733  *
3734  * @param pg handle for the peer group
3735  * @param voff number of peers that should go offline
3736  * @param von number of peers that should come back online;
3737  *            must be zero on first call (since "testbed_start"
3738  *            always starts all of the peers)
3739  * @param timeout how long to wait for operations to finish before
3740  *        giving up
3741  * @param cb function to call at the end
3742  * @param cb_cls closure for cb
3743  */
3744 void
3745 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
3746                               unsigned int voff,
3747                               unsigned int von,
3748                               struct GNUNET_TIME_Relative timeout,
3749                               GNUNET_TESTING_NotifyCompletion cb,
3750                               void *cb_cls)
3751 {
3752   struct ChurnContext *churn_ctx;
3753   unsigned int running;
3754   unsigned int stopped;
3755   unsigned int total_running;
3756   unsigned int total_stopped;
3757   unsigned int i;
3758   unsigned int *running_arr;
3759   unsigned int *stopped_arr;
3760   unsigned int *running_permute;
3761   unsigned int *stopped_permute;
3762
3763   running = 0;
3764   stopped = 0;
3765
3766   if ((von == 0) && (voff == 0)) /* No peers at all? */
3767     {
3768       cb(cb_cls, NULL);
3769       return;
3770     }
3771
3772   for (i = 0; i < pg->total; i++)
3773   {
3774     if (pg->peers[i].daemon->running == GNUNET_YES)
3775     {
3776       GNUNET_assert(running != -1);
3777       running++;
3778     }
3779     else
3780     {
3781       GNUNET_assert(stopped != -1);
3782       stopped++;
3783     }
3784   }
3785
3786   if (voff > running)
3787   {
3788     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
3789     cb(cb_cls, "Trying to stop more peers than are currently running!");
3790     return;
3791   }
3792
3793   if (von > stopped)
3794   {
3795     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
3796     cb(cb_cls, "Trying to start more peers than are currently stopped!");
3797     return;
3798   }
3799
3800   churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3801
3802   running_arr = NULL;
3803   if (running > 0)
3804     running_arr = GNUNET_malloc(running * sizeof(unsigned int));
3805
3806   stopped_arr = NULL;
3807   if (stopped > 0)
3808     stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
3809
3810   running_permute = NULL;
3811   stopped_permute = NULL;
3812
3813   if (running > 0)
3814     running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
3815   if (stopped > 0)
3816     stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
3817
3818   total_running = running;
3819   total_stopped = stopped;
3820   running = 0;
3821   stopped = 0;
3822
3823   churn_ctx->num_to_start = von;
3824   churn_ctx->num_to_stop = voff;
3825   churn_ctx->cb = cb;
3826   churn_ctx->cb_cls = cb_cls;
3827
3828   for (i = 0; i < pg->total; i++)
3829   {
3830     if (pg->peers[i].daemon->running == GNUNET_YES)
3831     {
3832       GNUNET_assert((running_arr != NULL) && (total_running > running));
3833       running_arr[running] = i;
3834       running++;
3835     }
3836     else
3837     {
3838       GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
3839       stopped_arr[stopped] = i;
3840       stopped++;
3841     }
3842   }
3843
3844   GNUNET_assert(running >= voff);
3845   for (i = 0; i < voff; i++)
3846   {
3847 #if DEBUG_CHURN
3848     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
3849 #endif
3850     GNUNET_assert(running_arr != NULL);
3851     GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
3852                                 timeout,
3853                                 &churn_stop_callback, churn_ctx,
3854                                 GNUNET_NO, GNUNET_YES);
3855   }
3856
3857   GNUNET_assert(stopped >= von);
3858   for (i = 0; i < von; i++)
3859     {
3860 #if DEBUG_CHURN
3861       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
3862 #endif
3863       GNUNET_assert(stopped_arr != NULL);
3864       GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
3865                                           timeout, &churn_start_callback, churn_ctx);
3866   }
3867
3868   GNUNET_free_non_null(running_arr);
3869   GNUNET_free_non_null(stopped_arr);
3870   GNUNET_free_non_null(running_permute);
3871   GNUNET_free_non_null(stopped_permute);
3872 }
3873
3874
3875 /**
3876  * Restart all peers in the given group.
3877  *
3878  * @param pg the handle to the peer group
3879  * @param callback function to call on completion (or failure)
3880  * @param callback_cls closure for the callback function
3881  */
3882 void
3883 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
3884                                 GNUNET_TESTING_NotifyCompletion callback,
3885                                 void *callback_cls)
3886 {
3887   struct RestartContext *restart_context;
3888   unsigned int off;
3889
3890   if (pg->total > 0)
3891     {
3892       restart_context = GNUNET_malloc(sizeof(struct RestartContext));
3893       restart_context->peer_group = pg;
3894       restart_context->peers_restarted = 0;
3895       restart_context->callback = callback;
3896       restart_context->callback_cls = callback_cls;
3897
3898       for (off = 0; off < pg->total; off++)
3899         {
3900           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
3901         }
3902     }
3903 }
3904
3905 /**
3906  * Start or stop an individual peer from the given group.
3907  *
3908  * @param pg handle to the peer group
3909  * @param offset which peer to start or stop
3910  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
3911  * @param timeout how long to wait for shutdown
3912  * @param cb function to call at the end
3913  * @param cb_cls closure for cb
3914  */
3915 void
3916 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
3917                              unsigned int offset,
3918                              int desired_status,
3919                              struct GNUNET_TIME_Relative timeout,
3920                              GNUNET_TESTING_NotifyCompletion cb,
3921                              void *cb_cls)
3922 {
3923   struct ChurnContext *churn_ctx;
3924
3925   if (GNUNET_NO == desired_status)
3926     {
3927       if (NULL != pg->peers[offset].daemon)
3928         {
3929           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3930           churn_ctx->num_to_start = 0;
3931           churn_ctx->num_to_stop = 1;
3932           churn_ctx->cb = cb;
3933           churn_ctx->cb_cls = cb_cls;
3934           GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon,
3935                                      timeout, &churn_stop_callback, churn_ctx,
3936                                      GNUNET_NO, GNUNET_YES);
3937         }
3938     }
3939   else if (GNUNET_YES == desired_status)
3940     {
3941       if (NULL == pg->peers[offset].daemon)
3942         {
3943           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3944           churn_ctx->num_to_start = 1;
3945           churn_ctx->num_to_stop = 0;
3946           churn_ctx->cb = cb;
3947           churn_ctx->cb_cls = cb_cls;
3948           GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon,
3949                                               timeout, &churn_start_callback, churn_ctx);
3950         }
3951     }
3952   else
3953     GNUNET_break (0);
3954 }
3955
3956
3957 /**
3958  * Callback for shutting down peers in a peer group.
3959  *
3960  * @param cls closure (struct ShutdownContext)
3961  * @param emsg NULL on success
3962  */
3963 void internal_shutdown_callback (void *cls,
3964                                  const char *emsg)
3965 {
3966   struct ShutdownContext *shutdown_ctx = cls;
3967
3968   shutdown_ctx->outstanding--;
3969   if (emsg == NULL)
3970     {
3971       shutdown_ctx->peers_down++;
3972     }
3973   else
3974     {
3975       shutdown_ctx->peers_failed++;
3976     }
3977
3978   if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
3979     {
3980       if (shutdown_ctx->peers_failed > 0)
3981         shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
3982       else
3983         shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
3984       GNUNET_free(shutdown_ctx);
3985     }
3986 }
3987
3988 /**
3989  * Individual shutdown context for a particular peer.
3990  */
3991 struct PeerShutdownContext
3992 {
3993   /**
3994    * Pointer to the high level shutdown context.
3995    */
3996   struct ShutdownContext *shutdown_ctx;
3997
3998   /**
3999    * The daemon handle for the peer to shut down.
4000    */
4001   struct GNUNET_TESTING_Daemon *daemon;
4002 };
4003
4004 /**
4005  * Task to rate limit the number of outstanding peer shutdown
4006  * requests.  This is necessary for making sure we don't do
4007  * too many ssh connections at once, but is generally nicer
4008  * to any system as well (graduated task starts, as opposed
4009  * to calling gnunet-arm N times all at once).
4010  */
4011 static void
4012 schedule_shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
4013 {
4014   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4015   struct ShutdownContext *shutdown_ctx;
4016
4017   GNUNET_assert(peer_shutdown_ctx != NULL);
4018   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4019   GNUNET_assert(shutdown_ctx != NULL);
4020
4021   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4022     GNUNET_SCHEDULER_add_delayed(peer_shutdown_ctx->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_shutdown_task, peer_shutdown_ctx);
4023   else
4024     {
4025       shutdown_ctx->outstanding++;
4026       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, shutdown_ctx->timeout, &internal_shutdown_callback, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4027       GNUNET_free(peer_shutdown_ctx);
4028     }
4029 }
4030 /**
4031  * Shutdown all peers started in the given group.
4032  *
4033  * @param pg handle to the peer group
4034  * @param timeout how long to wait for shutdown
4035  * @param cb callback to notify upon success or failure
4036  * @param cb_cls closure for cb
4037  */
4038 void
4039 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4040                              struct GNUNET_TIME_Relative timeout,
4041                              GNUNET_TESTING_NotifyCompletion cb,
4042                              void *cb_cls)
4043 {
4044   unsigned int off;
4045   struct ShutdownContext *shutdown_ctx;
4046   struct PeerShutdownContext *peer_shutdown_ctx;
4047
4048   GNUNET_assert(pg->total > 0);
4049
4050   shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
4051   shutdown_ctx->cb = cb;
4052   shutdown_ctx->cb_cls = cb_cls;
4053   shutdown_ctx->total_peers = pg->total;
4054   shutdown_ctx->timeout = timeout;
4055   /* shtudown_ctx->outstanding = 0; */
4056
4057   for (off = 0; off < pg->total; off++)
4058     {
4059       GNUNET_assert(NULL != pg->peers[off].daemon);
4060       peer_shutdown_ctx = GNUNET_malloc(sizeof(struct PeerShutdownContext));
4061       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4062       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4063       GNUNET_SCHEDULER_add_now(pg->peers[off].daemon->sched, &schedule_shutdown_task, peer_shutdown_ctx);
4064       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4065       if (NULL != pg->peers[off].cfg)
4066         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4067       if (pg->peers[off].allowed_peers != NULL)
4068         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
4069       if (pg->peers[off].connect_peers != NULL)
4070         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
4071       if (pg->peers[off].blacklisted_peers != NULL)
4072         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
4073     }
4074   GNUNET_free (pg->peers);
4075   if (NULL != pg->hosts)
4076     {
4077       GNUNET_free (pg->hosts[0].hostname);
4078       GNUNET_free (pg->hosts);
4079     }
4080   GNUNET_free (pg);
4081 }
4082
4083
4084 /* end of testing_group.c */