src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define DEBUG_CHURN GNUNET_NO
  36
  37 /**
  38  * Lowest port used for GNUnet testing.  Should be high enough to not
  39  * conflict with other applications running on the hosts but be low
  40  * enough to not conflict with client-ports (typically starting around
  41  * 32k).
  42  */
  43 #define LOW_PORT 10000
  44
  45 /**
  46  * Highest port used for GNUnet testing.  Should be low enough to not
  47  * conflict with the port range for "local" ports (client apps; see
  48  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  49  */
  50 #define HIGH_PORT 56000
  51
  52 #define MAX_OUTSTANDING_CONNECTIONS 10
  53
  54 #define MAX_CONCURRENT_HOSTKEYS 10
  55
  56 #define MAX_CONCURRENT_STARTING 10
  57
  58 #define MAX_CONCURRENT_SHUTDOWN 10
  59
  60 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
  61
  62 #define CONNECT_ATTEMPTS 8
  63
  64 /**
  65  * Prototype of a function called whenever two peers would be connected
  66  * in a certain topology.
  67  */
  68 typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg,
  69                                                   unsigned int first,
  70                                                   unsigned int second);
  71
  72
  73 /**
  74  * Context for handling churning a peer group
  75  */
  76 struct ChurnContext
  77 {
  78   /**
  79    * Callback used to notify of churning finished
  80    */
  81   GNUNET_TESTING_NotifyCompletion cb;
  82
  83   /**
  84    * Closure for callback
  85    */
  86   void *cb_cls;
  87
  88   /**
  89    * Number of peers that still need to be started
  90    */
  91   unsigned int num_to_start;
  92
  93   /**
  94    * Number of peers that still need to be stopped
  95    */
  96   unsigned int num_to_stop;
  97
  98   /**
  99    * Number of peers that failed to start
 100    */
 101   unsigned int num_failed_start;
 102
 103   /**
 104    * Number of peers that failed to stop
 105    */
 106   unsigned int num_failed_stop;
 107 };
 108
 109 struct RestartContext
 110 {
 111   /**
 112    * The group of peers being restarted
 113    */
 114   struct GNUNET_TESTING_PeerGroup *peer_group;
 115
 116   /**
 117    * How many peers have been restarted thus far
 118    */
 119   unsigned int peers_restarted;
 120
 121   /**
 122    * How many peers got an error when restarting
 123    */
 124   unsigned int peers_restart_failed;
 125
 126   /**
 127    * The function to call once all peers have been restarted
 128    */
 129   GNUNET_TESTING_NotifyCompletion callback;
 130
 131   /**
 132    * Closure for callback function
 133    */
 134   void *callback_cls;
 135
 136 };
 137
 138
 139 struct ShutdownContext
 140 {
 141   /**
 142    * Total peers to wait for
 143    */
 144   unsigned int total_peers;
 145
 146   /**
 147    * Number of peers successfully shut down
 148    */
 149   unsigned int peers_down;
 150
 151   /**
 152    * Number of peers failed to shut down
 153    */
 154   unsigned int peers_failed;
 155
 156   /**
 157    * Number of peers we have started shutting
 158    * down.  If too many, wait on them.
 159    */
 160   unsigned int outstanding;
 161
 162   /**
 163    * Timeout for shutdown.
 164    */
 165   struct GNUNET_TIME_Relative timeout;
 166
 167   /**
 168    * Callback to call when all peers either
 169    * shutdown or failed to shutdown
 170    */
 171   GNUNET_TESTING_NotifyCompletion cb;
 172
 173   /**
 174    * Closure for cb
 175    */
 176   void *cb_cls;
 177 };
 178
 179 struct CreateTopologyContext
 180 {
 181
 182   /**
 183    * Function to call with number of connections
 184    */
 185   GNUNET_TESTING_NotifyConnections cont;
 186
 187   /**
 188    * Closure for connection notification
 189    */
 190   void *cls;
 191 };
 192
 193 #if OLD
 194 struct PeerConnection
 195 {
 196   /*
 197    * Linked list
 198    */
 199   struct PeerConnection *next;
 200
 201   /*
 202    * Pointer to daemon handle
 203    */
 204   struct GNUNET_TESTING_Daemon *daemon;
 205
 206 };
 207 #endif
 208
 209 struct InternalStartContext
 210 {
 211   /**
 212    * Pointer to peerdata
 213    */
 214   struct PeerData *peer;
 215
 216   /**
 217    * Timeout for peer startup
 218    */
 219   struct GNUNET_TIME_Relative timeout;
 220
 221   /**
 222    * Client callback for hostkey notification
 223    */
 224   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 225
 226   /**
 227    * Closure for hostkey_callback
 228    */
 229   void *hostkey_cls;
 230
 231   /**
 232    * Client callback for peer start notification
 233    */
 234   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 235
 236   /**
 237    * Closure for cb
 238    */
 239   void *start_cb_cls;
 240
 241   /**
 242    * Hostname, where to start the peer
 243    */
 244   const char *hostname;
 245 };
 246
 247 /**
 248  * Data we keep per peer.
 249  */
 250 struct PeerData
 251 {
 252   /**
 253    * (Initial) configuration of the host.
 254    * (initial because clients could change
 255    *  it and we would not know about those
 256    *  updates).
 257    */
 258   struct GNUNET_CONFIGURATION_Handle *cfg;
 259
 260   /**
 261    * Handle for controlling the daemon.
 262    */
 263   struct GNUNET_TESTING_Daemon *daemon;
 264
 265   /**
 266    * The peergroup this peer belongs to.
 267    */
 268   struct GNUNET_TESTING_PeerGroup *pg;
 269
 270   /**
 271    * Hash map of allowed peer connections (F2F created topology)
 272    */
 273   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 274
 275   /**
 276    * Hash map of blacklisted peers
 277    */
 278   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 279
 280   /**
 281    * Hash map of peer connections
 282    */
 283   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 284
 285   /**
 286    * Temporary hash map of peer connections
 287    */
 288   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 289
 290   /**
 291    * Temporary variable for topology creation, should be reset before
 292    * creating any topology so the count is valid once finished.
 293    */
 294   int num_connections;
 295
 296   /**
 297    * Context to keep track of peers being started, to
 298    * stagger hostkey generation and peer startup.
 299    */
 300   struct InternalStartContext internal_context;
 301 };
 302
 303
 304 /**
 305  * Data we keep per host.
 306  */
 307 struct HostData
 308 {
 309   /**
 310    * Name of the host.
 311    */
 312   char *hostname;
 313
 314   /**
 315    * Lowest port that we have not yet used
 316    * for GNUnet.
 317    */
 318   uint16_t minport;
 319 };
 320
 321 struct TopologyIterateContext
 322 {
 323   /**
 324    * Callback for notifying of two connected peers.
 325    */
 326   GNUNET_TESTING_NotifyTopology topology_cb;
 327
 328   /**
 329    * Closure for topology_cb
 330    */
 331   void *cls;
 332
 333   /**
 334    * Number of peers currently connected to.
 335    */
 336   unsigned int connected;
 337
 338   /**
 339    * Number of peers we have finished iterating.
 340    */
 341   unsigned int completed;
 342
 343   /**
 344    * Number of peers total.
 345    */
 346   unsigned int total;
 347 };
 348
 349 struct StatsIterateContext
 350 {
 351   /**
 352    * Handle to the statistics service.
 353    */
 354   struct GNUNET_STATISTICS_Handle *stats_handle;
 355
 356   /**
 357    * Handle for getting statistics.
 358    */
 359   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 360
 361   /**
 362    * Continuation to call once all stats information has been retrieved.
 363    */
 364   GNUNET_STATISTICS_Callback cont;
 365
 366   /**
 367    * Proc function to call on each value received.
 368    */
 369   GNUNET_TESTING_STATISTICS_Iterator proc;
 370
 371   /**
 372    * Closure for topology_cb
 373    */
 374   void *cls;
 375
 376   /**
 377    * Number of peers currently connected to.
 378    */
 379   unsigned int connected;
 380
 381   /**
 382    * Number of peers we have finished iterating.
 383    */
 384   unsigned int completed;
 385
 386   /**
 387    * Number of peers total.
 388    */
 389   unsigned int total;
 390 };
 391
 392 struct CoreContext
 393 {
 394   void *iter_context;
 395   struct GNUNET_TESTING_Daemon *daemon;
 396 };
 397
 398 /**
 399  * Handle to a group of GNUnet peers.
 400  */
 401 struct GNUNET_TESTING_PeerGroup
 402 {
 403   /**
 404    * Our scheduler.
 405    */
 406   struct GNUNET_SCHEDULER_Handle *sched;
 407
 408   /**
 409    * Configuration template.
 410    */
 411   const struct GNUNET_CONFIGURATION_Handle *cfg;
 412
 413   /**
 414    * Function to call on each started daemon.
 415    */
 416   //GNUNET_TESTING_NotifyDaemonRunning cb;
 417
 418   /**
 419    * Closure for cb.
 420    */
 421   //void *cb_cls;
 422
 423   /*
 424    * Function to call on each topology connection created
 425    */
 426   GNUNET_TESTING_NotifyConnection notify_connection;
 427
 428   /*
 429    * Callback for notify_connection
 430    */
 431   void *notify_connection_cls;
 432
 433   /**
 434    * NULL-terminated array of information about
 435    * hosts.
 436    */
 437   struct HostData *hosts;
 438
 439   /**
 440    * Array of "total" peers.
 441    */
 442   struct PeerData *peers;
 443
 444   /**
 445    * Number of peers in this group.
 446    */
 447   unsigned int total;
 448
 449   /**
 450    * At what time should we fail the peer startup process?
 451    */
 452   struct GNUNET_TIME_Absolute max_timeout;
 453
 454   /**
 455    * How many peers are being started right now?
 456    */
 457   unsigned int starting;
 458
 459   /**
 460    * How many peers have already been started?
 461    */
 462   unsigned int started;
 463 };
 464
 465 struct UpdateContext
 466 {
 467   struct GNUNET_CONFIGURATION_Handle *ret;
 468   const char *hostname;
 469   unsigned int nport;
 470   unsigned int upnum;
 471 };
 472
 473
 474 struct ConnectContext
 475 {
 476   struct GNUNET_TESTING_Daemon *first;
 477
 478   struct GNUNET_TESTING_Daemon *second;
 479
 480   struct GNUNET_TESTING_PeerGroup *pg;
 481 };
 482
 483 /**
 484  * Convert unique ID to hash code.
 485  *
 486  * @param uid unique ID to convert
 487  * @param hash set to uid (extended with zeros)
 488  */
 489 static void
 490 hash_from_uid (uint32_t uid,
 491                GNUNET_HashCode *hash)
 492 {
 493   memset (hash, 0, sizeof(GNUNET_HashCode));
 494   *((uint32_t*)hash) = uid;
 495 }
 496
 497 /**
 498  * Convert hash code to unique ID.
 499  *
 500  * @param uid unique ID to convert
 501  * @param hash set to uid (extended with zeros)
 502  */
 503 static void
 504 uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
 505 {
 506   memcpy (uid, hash, sizeof(uint32_t));
 507 }
 508
 509 /**
 510  * Number of connects we are waiting on, allows us to rate limit
 511  * connect attempts.
 512  */
 513 static int outstanding_connects;
 514
 515 /**
 516  * Get a topology from a string input.
 517  *
 518  * @param topology where to write the retrieved topology
 519  * @param topology_string The string to attempt to
 520  *        get a configuration value from
 521  * @return GNUNET_YES if topology string matched a
 522  *         known topology, GNUNET_NO if not
 523  */
 524 int
 525 GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
 526 {
 527   /**
 528    * Strings representing topologies in enum
 529    */
 530   static const char * topology_strings[] =
 531     {
 532       /**
 533        * A clique (everyone connected to everyone else).
 534        */
 535       "CLIQUE",
 536
 537       /**
 538        * Small-world network (2d torus plus random links).
 539        */
 540       "SMALL_WORLD",
 541
 542       /**
 543        * Small-world network (ring plus random links).
 544        */
 545       "SMALL_WORLD_RING",
 546
 547       /**
 548        * Ring topology.
 549        */
 550       "RING",
 551
 552       /**
 553        * 2-d torus.
 554        */
 555       "2D_TORUS",
 556
 557       /**
 558        * Random graph.
 559        */
 560       "ERDOS_RENYI",
 561
 562       /**
 563        * Certain percentage of peers are unable to communicate directly
 564        * replicating NAT conditions
 565        */
 566       "INTERNAT",
 567
 568       /**
 569        * Scale free topology.
 570        */
 571       "SCALE_FREE",
 572
 573       /**
 574        * Straight line topology.
 575        */
 576       "LINE",
 577
 578       /**
 579        * All peers are disconnected.
 580        */
 581       "NONE",
 582
 583       NULL
 584     };
 585
 586   int curr = 0;
 587   if (topology_string == NULL)
 588     return GNUNET_NO;
 589   while (topology_strings[curr] != NULL)
 590     {
 591       if (strcasecmp(topology_strings[curr], topology_string) == 0)
 592         {
 593           *topology = curr;
 594           return GNUNET_YES;
 595         }
 596       curr++;
 597     }
 598   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 599   return GNUNET_NO;
 600 }
 601
 602
 603 /**
 604  * Get connect topology option from string input.
 605  *
 606  * @param topology_option where to write the retrieved topology
 607  * @param topology_string The string to attempt to
 608  *        get a configuration value from
 609  * @return GNUNET_YES if string matched a known
 610  *         topology option, GNUNET_NO if not
 611  */
 612 int
 613 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
 614                                     char * topology_string)
 615 {
 616   /**
 617    * Options for connecting a topology as strings.
 618    */
 619   static const char * topology_option_strings[] =
 620     {
 621       /**
 622        * Try to connect all peers specified in the topology.
 623        */
 624       "CONNECT_ALL",
 625
 626       /**
 627        * Choose a random subset of connections to create.
 628        */
 629       "CONNECT_RANDOM_SUBSET",
 630
 631       /**
 632        * Create at least X connections for each peer.
 633        */
 634       "CONNECT_MINIMUM",
 635
 636       /**
 637        * Using a depth first search, create one connection
 638        * per peer.  If any are missed (graph disconnected)
 639        * start over at those peers until all have at least one
 640        * connection.
 641        */
 642       "CONNECT_DFS",
 643
 644       /**
 645        * No options specified.
 646        */
 647       "CONNECT_NONE",
 648
 649       NULL
 650     };
 651   int curr = 0;
 652
 653   if (topology_string == NULL)
 654     return GNUNET_NO;
 655   while (NULL != topology_option_strings[curr])
 656     {
 657       if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
 658         {
 659           *topology_option = curr;
 660           return GNUNET_YES;
 661         }
 662       curr++;
 663     }
 664   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 665   return GNUNET_NO;
 666 }
 667
 668 /**
 669  * Function to iterate over options.  Copies
 670  * the options to the target configuration,
 671  * updating PORT values as needed.
 672  *
 673  * @param cls closure
 674  * @param section name of the section
 675  * @param option name of the option
 676  * @param value value of the option
 677  */
 678 static void
 679 update_config (void *cls,
 680                const char *section, const char *option, const char *value)
 681 {
 682   struct UpdateContext *ctx = cls;
 683   unsigned int ival;
 684   char cval[12];
 685   char uval[128];
 686
 687   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 688     {
 689       if (ival != 0)
 690         {
 691           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 692           value = cval;
 693         }
 694     }
 695
 696   if (0 == strcmp (option, "UNIXPATH"))
 697     {
 698       GNUNET_snprintf (uval,
 699                        sizeof (uval),
 700                        "/tmp/test-service-%s-%u",
 701                        section,
 702                        ctx->upnum++);
 703       value = uval;
 704     }
 705
 706   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 707     {
 708       value = ctx->hostname;
 709     }
 710
 711   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 712 }
 713
 714
 715 /**
 716  * Create a new configuration using the given configuration
 717  * as a template; however, each PORT in the existing cfg
 718  * must be renumbered by incrementing "*port".  If we run
 719  * out of "*port" numbers, return NULL.
 720  *
 721  * @param cfg template configuration
 722  * @param port port numbers to use, update to reflect
 723  *             port numbers that were used
 724  * @param upnum number to make unix domain socket names unique
 725  * @param hostname hostname of the controlling host, to allow control connections from
 726  *
 727  * @return new configuration, NULL on error
 728  */
 729 static struct GNUNET_CONFIGURATION_Handle *
 730 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 731              uint16_t * port,
 732              uint32_t * upnum,
 733              const char *hostname)
 734 {
 735   struct UpdateContext uc;
 736   uint16_t orig;
 737   char *control_host;
 738   char *allowed_hosts;
 739
 740   orig = *port;
 741   uc.nport = *port;
 742   uc.upnum = *upnum;
 743   uc.ret = GNUNET_CONFIGURATION_create ();
 744   uc.hostname = hostname;
 745
 746   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 747   if (uc.nport >= HIGH_PORT)
 748     {
 749       *port = orig;
 750       GNUNET_CONFIGURATION_destroy (uc.ret);
 751       return NULL;
 752     }
 753
 754   if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 755     {
 756       if (hostname != NULL)
 757         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1; %s;", control_host, hostname);
 758       else
 759         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
 760
 761       GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
 762       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport", "ACCEPT_FROM", allowed_hosts);
 763       GNUNET_CONFIGURATION_set_value_string(uc.ret, "dht", "ACCEPT_FROM", allowed_hosts);
 764       GNUNET_CONFIGURATION_set_value_string(uc.ret, "statistics", "ACCEPT_FROM", allowed_hosts);
 765       GNUNET_free_non_null(control_host);
 766       GNUNET_free(allowed_hosts);
 767     }
 768
 769
 770   /* arm needs to know to allow connections from the host on which it is running,
 771    * otherwise gnunet-arm is unable to connect to it in some instances */
 772   if (hostname != NULL)
 773     {
 774       GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
 775       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", hostname);
 776       GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
 777       GNUNET_free(allowed_hosts);
 778     }
 779   else
 780     GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", "127.0.0.1");
 781
 782   *port = (uint16_t) uc.nport;
 783   *upnum = uc.upnum;
 784   return uc.ret;
 785 }
 786
 787
 788 /*
 789  * Add entries to the peers connect list
 790  *
 791  * @param pg the peer group we are working with
 792  * @param first index of the first peer
 793  * @param second index of the second peer
 794  *
 795  * @return the number of connections added
 796  *         technically should only be 0 or 2
 797  *
 798  */
 799 static int
 800 add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 801 {
 802   int added;
 803   int add_first;
 804   int add_second;
 805
 806   GNUNET_HashCode hash_first;
 807   GNUNET_HashCode hash_second;
 808
 809   hash_from_uid(first, &hash_first);
 810   hash_from_uid(second, &hash_second);
 811
 812   add_first = GNUNET_NO;
 813   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
 814     {
 815       add_first = GNUNET_YES;
 816     }
 817
 818   add_second = GNUNET_NO;
 819   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
 820     {
 821       add_second = GNUNET_YES;
 822     }
 823
 824   added = 0;
 825   if (add_first)
 826     {
 827       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 828       pg->peers[first].num_connections++;
 829       added++;
 830     }
 831
 832   if (add_second)
 833     {
 834       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 835       pg->peers[second].num_connections++;
 836       added++;
 837     }
 838
 839   return added;
 840 }
 841
 842
 843 /*
 844  * Add entries to the peers allowed connections list
 845  *
 846  * @param pg the peer group we are working with
 847  * @param first index of the first peer
 848  * @param second index of the second peer
 849  *
 850  * @return the number of connections added (can be 0, 1 or 2)
 851  *         technically should only be 0 or 2, but the small price
 852  *         of iterating over the lists (hashmaps in the future)
 853  *         for being sure doesn't bother me!
 854  *
 855  */
 856 static int
 857 add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 858 {
 859   int added;
 860 #if OLD
 861   struct PeerConnection *first_iter;
 862   struct PeerConnection *second_iter;
 863   struct PeerConnection *new_first;
 864   struct PeerConnection *new_second;
 865 #endif
 866   int add_first;
 867   int add_second;
 868
 869   GNUNET_HashCode hash_first;
 870   GNUNET_HashCode hash_second;
 871
 872   hash_from_uid(first, &hash_first);
 873   hash_from_uid(second, &hash_second);
 874
 875   add_first = GNUNET_NO;
 876   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
 877     {
 878       add_first = GNUNET_YES;
 879     }
 880
 881   add_second = GNUNET_NO;
 882   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
 883     {
 884       add_second = GNUNET_YES;
 885     }
 886 #if OLD
 887   first_iter = pg->peers[first].connected_peers;
 888   while (first_iter != NULL)
 889     {
 890       if (first_iter->daemon == pg->peers[second].daemon)
 891         add_first = GNUNET_NO;
 892       first_iter = first_iter->next;
 893     }
 894
 895   second_iter = pg->peers[second].connected_peers;
 896   add_second = GNUNET_YES;
 897   while (second_iter != NULL)
 898     {
 899       if (second_iter->daemon == pg->peers[first].daemon)
 900         add_second = GNUNET_NO;
 901       second_iter = second_iter->next;
 902     }
 903 #endif
 904
 905   added = 0;
 906   if (add_first)
 907     {
 908       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 909 #if OLD
 910       new_first = GNUNET_malloc(sizeof(struct PeerConnection));
 911       new_first->daemon = pg->peers[second].daemon;
 912       new_first->next = pg->peers[first].connected_peers;
 913       pg->peers[first].connected_peers = new_first;
 914 #endif
 915       pg->peers[first].num_connections++;
 916       added++;
 917     }
 918
 919   if (add_second)
 920     {
 921       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 922 #if OLD
 923       new_second = GNUNET_malloc(sizeof(struct PeerConnection));
 924       new_second->daemon = pg->peers[first].daemon;
 925       new_second->next = pg->peers[second].connected_peers;
 926       pg->peers[second].connected_peers = new_second;
 927       pg->peers[first].num_connections++;
 928 #endif
 929       pg->peers[second].num_connections++;
 930       added++;
 931     }
 932
 933   return added;
 934 }
 935
 936 /*
 937  * Add entries to the peers blacklisted list
 938  *
 939  * @param pg the peer group we are working with
 940  * @param first index of the first peer
 941  * @param second index of the second peer
 942  *
 943  * @return the number of connections added (can be 0, 1 or 2)
 944  *
 945  */
 946 static int
 947 blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 948 {
 949   int added;
 950   int add_first;
 951   int add_second;
 952   GNUNET_HashCode hash_first;
 953   GNUNET_HashCode hash_second;
 954
 955   hash_from_uid(first, &hash_first);
 956   hash_from_uid(second, &hash_second);
 957
 958   add_first = GNUNET_NO;
 959   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
 960     {
 961       add_first = GNUNET_YES;
 962     }
 963
 964   add_second = GNUNET_NO;
 965   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
 966     {
 967       add_second = GNUNET_YES;
 968     }
 969
 970   added = 0;
 971   if (add_first)
 972     {
 973       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 974       pg->peers[first].num_connections++;
 975       added++;
 976     }
 977
 978   if (add_second)
 979     {
 980       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 981       pg->peers[second].num_connections++;
 982       added++;
 983     }
 984
 985   return added;
 986 }
 987
 988 /*
 989  * Remove entries from the peers blacklisted list
 990  *
 991  * @param pg the peer group we are working with
 992  * @param first index of the first peer
 993  * @param second index of the second peer
 994  *
 995  * @return the number of connections removed (can be 0, 1 or 2)
 996  *
 997  */
 998 static int
 999 unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1000 {
1001   int removed;
1002   int remove_first;
1003   int remove_second;
1004   GNUNET_HashCode hash_first;
1005   GNUNET_HashCode hash_second;
1006
1007   hash_from_uid(first, &hash_first);
1008   hash_from_uid(second, &hash_second);
1009
1010   remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
1011   remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
1012
1013   removed = 0;
1014   if (remove_first)
1015     {
1016       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
1017       removed++;
1018     }
1019
1020   if (remove_second)
1021     {
1022       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
1023       removed++;
1024     }
1025
1026   return removed;
1027 }
1028
1029 /**
1030  * Scale free network construction as described in:
1031  *
1032  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1033  *
1034  * Start with a network of "one" peer, then progressively add
1035  * peers up to the total number.  At each step, iterate over
1036  * all possible peers and connect new peer based on number of
1037  * existing connections of the target peer.
1038  *
1039  * @param pg the peer group we are dealing with
1040  * @param proc the connection processor to use
1041  *
1042  * @return the number of connections created
1043  */
1044 static int
1045 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1046 {
1047
1048   unsigned int total_connections;
1049   unsigned int outer_count;
1050   unsigned int i;
1051   unsigned int previous_total_connections;
1052   double random;
1053   double probability;
1054
1055   GNUNET_assert(pg->total > 1);
1056
1057   /* Add a connection between the first two nodes */
1058   total_connections = proc(pg, 0, 1);
1059
1060   for (outer_count = 1; outer_count < pg->total; outer_count++)
1061     {
1062       previous_total_connections = total_connections;
1063       for (i = 0; i < outer_count; i++)
1064         {
1065           probability = pg->peers[i].num_connections / (double)previous_total_connections;
1066           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1067                                                       UINT64_MAX)) / ( (double) UINT64_MAX);
1068 #if VERBOSE_TESTING
1069           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1070                       "Considering connecting peer %d to peer %d\n",
1071                       outer_count, i);
1072 #endif
1073           if (random < probability)
1074             {
1075 #if VERBOSE_TESTING
1076               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1077                           "Connecting peer %d to peer %d\n",
1078                           outer_count, i);
1079 #endif
1080               total_connections += proc(pg, outer_count, i);
1081             }
1082         }
1083     }
1084
1085   return total_connections;
1086 }
1087
1088 /**
1089  * Create a topology given a peer group (set of running peers)
1090  * and a connection processor.
1091  *
1092  * @param pg the peergroup to create the topology on
1093  * @param proc the connection processor to call to actually set
1094  *        up connections between two peers
1095  *
1096  * @return the number of connections that were set up
1097  *
1098  */
1099 int
1100 create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1101 {
1102   unsigned int i, j;
1103   int nodeToConnect;
1104   unsigned int natLog;
1105   unsigned int randomPeer;
1106   double random, logNModifier, percentage;
1107   unsigned int smallWorldConnections;
1108   int connsPerPeer;
1109   char *p_string;
1110   int max;
1111   int min;
1112   unsigned int useAnd;
1113   int connect_attempts;
1114
1115   logNModifier = 0.5; /* FIXME: default value? */
1116   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1117                                                          "TESTING",
1118                                                          "LOGNMODIFIER",
1119                                                          &p_string))
1120     {
1121       if (sscanf(p_string, "%lf", &logNModifier) != 1)
1122         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1123                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1124                     p_string,
1125                     "LOGNMODIFIER",
1126                     "TESTING");
1127       GNUNET_free (p_string);
1128     }
1129   percentage = 0.5; /* FIXME: default percentage? */
1130   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1131                                                          "TESTING",
1132                                                          "PERCENTAGE",
1133                                                          &p_string))
1134     {
1135       if (sscanf(p_string, "%lf", &percentage) != 1)
1136         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1137                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1138                     p_string,
1139                     "PERCENTAGE",
1140                     "TESTING");
1141       GNUNET_free (p_string);
1142     }
1143   natLog = log (pg->total);
1144   connsPerPeer = ceil (natLog * logNModifier);
1145
1146   if (connsPerPeer % 2 == 1)
1147     connsPerPeer += 1;
1148
1149   smallWorldConnections = 0;
1150   connect_attempts = 0;
1151   for (i = 0; i < pg->total; i++)
1152     {
1153       useAnd = 0;
1154       max = i + connsPerPeer / 2;
1155       min = i - connsPerPeer / 2;
1156
1157       if (max > pg->total - 1)
1158         {
1159           max = max - pg->total;
1160           useAnd = 1;
1161         }
1162
1163       if (min < 0)
1164         {
1165           min = pg->total - 1 + min;
1166           useAnd = 1;
1167         }
1168
1169       for (j = 0; j < connsPerPeer / 2; j++)
1170         {
1171           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1172                                                       UINT64_MAX) / ( (double) UINT64_MAX));
1173           if (random < percentage)
1174             {
1175               /* Connect to uniformly selected random peer */
1176               randomPeer =
1177                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1178                                    pg->total);
1179               while ((((randomPeer < max) && (randomPeer > min))
1180                       && (useAnd == 0)) || (((randomPeer > min)
1181                                              || (randomPeer < max))
1182                                             && (useAnd == 1)))
1183                 {
1184                   randomPeer =
1185                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1186                                                          pg->total);
1187                 }
1188               smallWorldConnections +=
1189                 proc (pg, i, randomPeer);
1190             }
1191           else
1192             {
1193               nodeToConnect = i + j + 1;
1194               if (nodeToConnect > pg->total - 1)
1195                 {
1196                   nodeToConnect = nodeToConnect - pg->total;
1197                 }
1198               connect_attempts +=
1199                 proc (pg, i, nodeToConnect);
1200             }
1201         }
1202
1203     }
1204
1205   connect_attempts += smallWorldConnections;
1206
1207   return connect_attempts;
1208 }
1209
1210 /**
1211  * Create a topology given a peer group (set of running peers)
1212  * and a connection processor.
1213  *
1214  * @param pg the peergroup to create the topology on
1215  * @param proc the connection processor to call to actually set
1216  *        up connections between two peers
1217  *
1218  * @return the number of connections that were set up
1219  *
1220  */
1221 static int
1222 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1223 {
1224   unsigned int outer_count, inner_count;
1225   unsigned int cutoff;
1226   int connect_attempts;
1227   double nat_percentage;
1228   char *p_string;
1229
1230   nat_percentage = 0.6; /* FIXME: default percentage? */
1231   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1232                                                          "TESTING",
1233                                                          "NATPERCENTAGE",
1234                                                          &p_string))
1235     {
1236       if (sscanf(p_string, "%lf", &nat_percentage) != 1)
1237         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1238                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1239                     p_string,
1240                     "NATPERCENTAGE",
1241                     "TESTING");
1242       GNUNET_free (p_string);
1243     }
1244
1245
1246
1247   cutoff = (unsigned int) (nat_percentage * pg->total);
1248
1249   connect_attempts = 0;
1250
1251   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1252     {
1253       for (inner_count = outer_count + 1; inner_count < pg->total;
1254            inner_count++)
1255         {
1256           if ((outer_count > cutoff) || (inner_count > cutoff))
1257             {
1258 #if VERBOSE_TESTING
1259               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1260                           "Connecting peer %d to peer %d\n",
1261                           outer_count, inner_count);
1262 #endif
1263               connect_attempts += proc(pg, outer_count, inner_count);
1264             }
1265         }
1266     }
1267
1268   return connect_attempts;
1269
1270 }
1271
1272 /**
1273  * Create a topology given a peer group (set of running peers)
1274  * and a connection processor.
1275  *
1276  * @param pg the peergroup to create the topology on
1277  * @param proc the connection processor to call to actually set
1278  *        up connections between two peers
1279  *
1280  * @return the number of connections that were set up
1281  *
1282  */
1283 static int
1284 create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1285 {
1286   unsigned int i, j, k;
1287   unsigned int square;
1288   unsigned int rows;
1289   unsigned int cols;
1290   unsigned int toggle = 1;
1291   unsigned int nodeToConnect;
1292   unsigned int natLog;
1293   unsigned int node1Row;
1294   unsigned int node1Col;
1295   unsigned int node2Row;
1296   unsigned int node2Col;
1297   unsigned int distance;
1298   double probability, random, percentage;
1299   unsigned int smallWorldConnections;
1300   char *p_string;
1301   int connect_attempts;
1302   square = floor (sqrt (pg->total));
1303   rows = square;
1304   cols = square;
1305
1306   percentage = 0.5; /* FIXME: default percentage? */
1307   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1308                                                          "TESTING",
1309                                                          "PERCENTAGE",
1310                                                          &p_string))
1311     {
1312       if (sscanf(p_string, "%lf", &percentage) != 1)
1313         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1314                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1315                     p_string,
1316                     "PERCENTAGE",
1317                     "TESTING");
1318       GNUNET_free (p_string);
1319     }
1320   probability = 0.5; /* FIXME: default percentage? */
1321   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1322                                                          "TESTING",
1323                                                          "PROBABILITY",
1324                                                          &p_string))
1325     {
1326       if (sscanf(p_string, "%lf", &probability) != 1)
1327         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1328                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1329                     p_string,
1330                     "PROBABILITY",
1331                     "TESTING");
1332       GNUNET_free (p_string);
1333     }
1334   if (square * square != pg->total)
1335     {
1336       while (rows * cols < pg->total)
1337         {
1338           if (toggle % 2 == 0)
1339             rows++;
1340           else
1341             cols++;
1342
1343           toggle++;
1344         }
1345     }
1346 #if VERBOSE_TESTING
1347       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1348                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1349                   rows, cols);
1350 #endif
1351
1352   connect_attempts = 0;
1353   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1354    * to the node to its right and above.  Once this is over, we'll have our torus!
1355    * Special case for the last node (if the rows and columns are not equal), connect
1356    * to the first in the row to maintain topology.
1357    */
1358   for (i = 0; i < pg->total; i++)
1359     {
1360       /* First connect to the node to the right */
1361       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1362         nodeToConnect = i + 1;
1363       else if (i + 1 == pg->total)
1364         nodeToConnect = rows * cols - cols;
1365       else
1366         nodeToConnect = i - cols + 1;
1367
1368       connect_attempts += proc (pg, i, nodeToConnect);
1369
1370       if (i < cols)
1371         nodeToConnect = (rows * cols) - cols + i;
1372       else
1373         nodeToConnect = i - cols;
1374
1375       if (nodeToConnect < pg->total)
1376         connect_attempts += proc (pg, i, nodeToConnect);
1377     }
1378   natLog = log (pg->total);
1379 #if VERBOSE_TESTING > 2
1380   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1381               _("natural log of %d is %d, will run %d iterations\n"),
1382              pg->total, natLog, (int) (natLog * percentage));
1383   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
1384 #endif
1385   smallWorldConnections = 0;
1386   for (i = 0; i < (int) (natLog * percentage); i++)
1387     {
1388       for (j = 0; j < pg->total; j++)
1389         {
1390           /* Determine the row and column of node at position j on the 2d torus */
1391           node1Row = j / cols;
1392           node1Col = j - (node1Row * cols);
1393           for (k = 0; k < pg->total; k++)
1394             {
1395               /* Determine the row and column of node at position k on the 2d torus */
1396               node2Row = k / cols;
1397               node2Col = k - (node2Row * cols);
1398               /* Simple Cartesian distance */
1399               distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
1400               if (distance > 1)
1401                 {
1402                   /* Calculate probability as 1 over the square of the distance */
1403                   probability = 1.0 / (distance * distance);
1404                   /* Choose a random value between 0 and 1 */
1405                   random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1406                                                               UINT64_MAX)) / ( (double) UINT64_MAX);
1407                   /* If random < probability, then connect the two nodes */
1408                   if (random < probability)
1409                     smallWorldConnections += proc (pg, j, k);
1410
1411                 }
1412             }
1413         }
1414     }
1415   connect_attempts += smallWorldConnections;
1416 #if VERBOSE_TESTING > 2
1417           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1418                       _("Total connections added for small world: %d!\n"),
1419                       smallWorldConnections);
1420 #endif
1421   return connect_attempts;
1422 }
1423
1424 /**
1425  * Create a topology given a peer group (set of running peers)
1426  * and a connection processor.
1427  *
1428  * @param pg the peergroup to create the topology on
1429  * @param proc the connection processor to call to actually set
1430  *        up connections between two peers
1431  *
1432  * @return the number of connections that were set up
1433  *
1434  */
1435 static int
1436 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1437 {
1438   double temp_rand;
1439   unsigned int outer_count;
1440   unsigned int inner_count;
1441   int connect_attempts;
1442   double probability;
1443   char *p_string;
1444
1445   probability = 0.5; /* FIXME: default percentage? */
1446   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1447                                                          "TESTING",
1448                                                          "PROBABILITY",
1449                                                          &p_string))
1450     {
1451       if (sscanf(p_string, "%lf", &probability) != 1)
1452         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1453                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1454                     p_string,
1455                     "PROBABILITY",
1456                     "TESTING");
1457       GNUNET_free (p_string);
1458     }
1459   connect_attempts = 0;
1460   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1461     {
1462       for (inner_count = outer_count + 1; inner_count < pg->total;
1463            inner_count++)
1464         {
1465           temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1466                                                          UINT64_MAX)) / ( (double) UINT64_MAX);
1467 #if VERBOSE_TESTING
1468           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1469                       _("rand is %f probability is %f\n"), temp_rand,
1470                       probability);
1471 #endif
1472           if (temp_rand < probability)
1473             {
1474               connect_attempts += proc (pg, outer_count, inner_count);
1475             }
1476         }
1477     }
1478
1479   return connect_attempts;
1480 }
1481
1482 /**
1483  * Create a topology given a peer group (set of running peers)
1484  * and a connection processor.
1485  *
1486  * @param pg the peergroup to create the topology on
1487  * @param proc the connection processor to call to actually set
1488  *        up connections between two peers
1489  *
1490  * @return the number of connections that were set up
1491  *
1492  */
1493 static int
1494 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1495 {
1496   unsigned int i;
1497   unsigned int square;
1498   unsigned int rows;
1499   unsigned int cols;
1500   unsigned int toggle = 1;
1501   unsigned int nodeToConnect;
1502   int connect_attempts;
1503
1504   connect_attempts = 0;
1505
1506   square = floor (sqrt (pg->total));
1507   rows = square;
1508   cols = square;
1509
1510   if (square * square != pg->total)
1511     {
1512       while (rows * cols < pg->total)
1513         {
1514           if (toggle % 2 == 0)
1515             rows++;
1516           else
1517             cols++;
1518
1519           toggle++;
1520         }
1521     }
1522 #if VERBOSE_TESTING
1523       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1524                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1525                   rows, cols);
1526 #endif
1527   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1528    * to the node to its right and above.  Once this is over, we'll have our torus!
1529    * Special case for the last node (if the rows and columns are not equal), connect
1530    * to the first in the row to maintain topology.
1531    */
1532   for (i = 0; i < pg->total; i++)
1533     {
1534       /* First connect to the node to the right */
1535       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1536         nodeToConnect = i + 1;
1537       else if (i + 1 == pg->total)
1538         nodeToConnect = rows * cols - cols;
1539       else
1540         nodeToConnect = i - cols + 1;
1541 #if VERBOSE_TESTING
1542           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1543                       "Connecting peer %d to peer %d\n",
1544                       i, nodeToConnect);
1545 #endif
1546       connect_attempts += proc(pg, i, nodeToConnect);
1547
1548       /* Second connect to the node immediately above */
1549       if (i < cols)
1550         nodeToConnect = (rows * cols) - cols + i;
1551       else
1552         nodeToConnect = i - cols;
1553
1554       if (nodeToConnect < pg->total)
1555         {
1556 #if VERBOSE_TESTING
1557           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1558                       "Connecting peer %d to peer %d\n",
1559                       i, nodeToConnect);
1560 #endif
1561           connect_attempts += proc(pg, i, nodeToConnect);
1562         }
1563
1564     }
1565
1566   return connect_attempts;
1567 }
1568
1569
1570 /**
1571  * Create a topology given a peer group (set of running peers)
1572  * and a connection processor.
1573  *
1574  * @param pg the peergroup to create the topology on
1575  * @param proc the connection processor to call to actually set
1576  *        up connections between two peers
1577  *
1578  * @return the number of connections that were set up
1579  *
1580  */
1581 static int
1582 create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1583 {
1584   unsigned int outer_count;
1585   unsigned int inner_count;
1586   int connect_attempts;
1587
1588   connect_attempts = 0;
1589
1590   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1591     {
1592       for (inner_count = outer_count + 1; inner_count < pg->total;
1593            inner_count++)
1594         {
1595 #if VERBOSE_TESTING
1596           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1597                       "Connecting peer %d to peer %d\n",
1598                       outer_count, inner_count);
1599 #endif
1600           connect_attempts += proc(pg, outer_count, inner_count);
1601         }
1602     }
1603
1604   return connect_attempts;
1605 }
1606
1607 /**
1608  * Create a topology given a peer group (set of running peers)
1609  * and a connection processor.
1610  *
1611  * @param pg the peergroup to create the topology on
1612  * @param proc the connection processor to call to actually set
1613  *        up connections between two peers
1614  *
1615  * @return the number of connections that were set up
1616  *
1617  */
1618 static int
1619 create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1620 {
1621   unsigned int count;
1622   int connect_attempts;
1623
1624   connect_attempts = 0;
1625
1626   /* Connect each peer to the next highest numbered peer */
1627   for (count = 0; count < pg->total - 1; count++)
1628     {
1629 #if VERBOSE_TESTING
1630           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1631                       "Connecting peer %d to peer %d\n",
1632                       count, count + 1);
1633 #endif
1634       connect_attempts += proc(pg, count, count + 1);
1635     }
1636
1637   return connect_attempts;
1638 }
1639
1640 /**
1641  * Create a topology given a peer group (set of running peers)
1642  * and a connection processor.
1643  *
1644  * @param pg the peergroup to create the topology on
1645  * @param proc the connection processor to call to actually set
1646  *        up connections between two peers
1647  *
1648  * @return the number of connections that were set up
1649  *
1650  */
1651 static int
1652 create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1653 {
1654   unsigned int count;
1655   int connect_attempts;
1656
1657   connect_attempts = 0;
1658
1659   /* Connect each peer to the next highest numbered peer */
1660   for (count = 0; count < pg->total - 1; count++)
1661     {
1662 #if VERBOSE_TESTING
1663           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1664                       "Connecting peer %d to peer %d\n",
1665                       count, count + 1);
1666 #endif
1667       connect_attempts += proc(pg, count, count + 1);
1668     }
1669
1670   /* Connect the last peer to the first peer */
1671   connect_attempts += proc(pg, pg->total - 1, 0);
1672
1673   return connect_attempts;
1674 }
1675
1676
1677 /**
1678  * Iterator for writing friends of a peer to a file.
1679  *
1680  * @param cls closure, an open writable file handle
1681  * @param key the key the daemon was stored under
1682  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1683  *
1684  * @return GNUNET_YES to continue iteration
1685  *
1686  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1687  *       with a single file_iterator that takes a closure which contains
1688  *       the prefix to write before the peer.  Then this could be used
1689  *       for blacklisting multiple transports and writing the friend
1690  *       file.  I'm sure *someone* will complain loudly about other
1691  *       things that negate these functions even existing so no point in
1692  *       "fixing" now.
1693  */
1694 static int
1695 friend_file_iterator (void *cls,
1696                   const GNUNET_HashCode * key,
1697                   void *value)
1698 {
1699   FILE *temp_friend_handle = cls;
1700   struct GNUNET_TESTING_Daemon *peer = value;
1701   struct GNUNET_PeerIdentity *temppeer;
1702   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1703
1704   temppeer = &peer->id;
1705   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1706   fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
1707
1708   return GNUNET_YES;
1709 }
1710
1711 struct BlacklistContext
1712 {
1713   /*
1714    * The (open) file handle to write to
1715    */
1716   FILE *temp_file_handle;
1717
1718   /*
1719    * The transport that this peer will be blacklisted on.
1720    */
1721   char *transport;
1722 };
1723
1724 /**
1725  * Iterator for writing blacklist data to appropriate files.
1726  *
1727  * @param cls closure, an open writable file handle
1728  * @param key the key the daemon was stored under
1729  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1730  *
1731  * @return GNUNET_YES to continue iteration
1732  */
1733 static int
1734 blacklist_file_iterator (void *cls,
1735                          const GNUNET_HashCode * key,
1736                          void *value)
1737 {
1738   struct BlacklistContext *blacklist_ctx = cls;
1739   //FILE *temp_blacklist_handle = cls;
1740   struct GNUNET_TESTING_Daemon *peer = value;
1741   struct GNUNET_PeerIdentity *temppeer;
1742   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1743
1744   temppeer = &peer->id;
1745   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1746   fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
1747
1748   return GNUNET_YES;
1749 }
1750
1751 /*
1752  * Create the friend files based on the PeerConnection's
1753  * of each peer in the peer group, and copy the files
1754  * to the appropriate place
1755  *
1756  * @param pg the peer group we are dealing with
1757  */
1758 static int
1759 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
1760 {
1761   FILE *temp_friend_handle;
1762   unsigned int pg_iter;
1763   char *temp_service_path;
1764   pid_t *pidarr;
1765   char *arg;
1766   char * mytemp;
1767   enum GNUNET_OS_ProcessStatusType type;
1768   unsigned long return_code;
1769   int count;
1770   int ret;
1771   int max_wait = 10;
1772
1773   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1774   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1775     {
1776       mytemp = GNUNET_DISK_mktemp("friends");
1777       GNUNET_assert(mytemp != NULL);
1778       temp_friend_handle = fopen (mytemp, "wt");
1779       GNUNET_assert(temp_friend_handle != NULL);
1780       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
1781       fclose(temp_friend_handle);
1782
1783       if (GNUNET_OK !=
1784           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1785         {
1786           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1787                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1788                       "SERVICEHOME",
1789                       "PATHS");
1790           if (UNLINK (mytemp) != 0)
1791             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1792           GNUNET_free (mytemp);
1793           break;
1794         }
1795
1796       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1797         {
1798           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
1799           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1800                                          "mv", mytemp, arg, NULL);
1801 #if VERBOSE_TESTING
1802           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1803                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1804 #endif
1805
1806           GNUNET_free(arg);
1807         }
1808       else /* Remote, scp the file to the correct place */
1809         {
1810           if (NULL != pg->peers[pg_iter].daemon->username)
1811             GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1812           else
1813             GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1814           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1815                                          "scp", mytemp, arg, NULL);
1816
1817 #if VERBOSE_TESTING
1818           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1819                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1820 #endif
1821           GNUNET_free(arg);
1822         }
1823       GNUNET_free (temp_service_path);
1824       GNUNET_free (mytemp);
1825     }
1826
1827   count = 0;
1828   ret = GNUNET_SYSERR;
1829   while ((count < max_wait) && (ret != GNUNET_OK))
1830     {
1831       ret = GNUNET_OK;
1832       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1833         {
1834 #if VERBOSE_TESTING
1835           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1836                       _("Checking copy status of file %d\n"), pg_iter);
1837 #endif
1838           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1839             {
1840               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1841                 {
1842                   ret = GNUNET_SYSERR;
1843                 }
1844               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1845                 {
1846                   ret = GNUNET_SYSERR;
1847                 }
1848               else
1849                 {
1850                   pidarr[pg_iter] = 0;
1851 #if VERBOSE_TESTING
1852             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1853                       _("File %d copied\n"), pg_iter);
1854 #endif
1855                 }
1856             }
1857         }
1858       count++;
1859       if (ret == GNUNET_SYSERR)
1860         {
1861           /* FIXME: why sleep here? -CG */
1862           sleep(1);
1863         }
1864     }
1865
1866 #if VERBOSE_TESTING
1867     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1868                 _("Finished copying all friend files!\n"));
1869 #endif
1870   GNUNET_free(pidarr);
1871   return ret;
1872 }
1873
1874
1875 /*
1876  * Create the blacklist files based on the PeerConnection's
1877  * of each peer in the peer group, and copy the files
1878  * to the appropriate place.
1879  *
1880  * @param pg the peer group we are dealing with
1881  * @param transports space delimited list of transports to blacklist
1882  */
1883 static int
1884 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
1885 {
1886   FILE *temp_file_handle;
1887   static struct BlacklistContext blacklist_ctx;
1888   unsigned int pg_iter;
1889   char *temp_service_path;
1890   pid_t *pidarr;
1891   char *arg;
1892   char *mytemp;
1893   enum GNUNET_OS_ProcessStatusType type;
1894   unsigned long return_code;
1895   int count;
1896   int ret;
1897   int max_wait = 10;
1898   int transport_len;
1899   unsigned int i;
1900   char *pos;
1901   char *temp_transports;
1902
1903   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1904   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1905     {
1906       mytemp = GNUNET_DISK_mktemp("blacklist");
1907       GNUNET_assert(mytemp != NULL);
1908       temp_file_handle = fopen (mytemp, "wt");
1909       GNUNET_assert(temp_file_handle != NULL);
1910       temp_transports = GNUNET_strdup(transports);
1911       blacklist_ctx.temp_file_handle = temp_file_handle;
1912       transport_len = strlen(temp_transports) + 1;
1913       pos = NULL;
1914
1915       for (i = 0; i < transport_len; i++)
1916       {
1917         if ((temp_transports[i] == ' ') && (pos == NULL))
1918           continue; /* At start of string (whitespace) */
1919         else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
1920         {
1921           temp_transports[i] = '\0';
1922           blacklist_ctx.transport = pos;
1923           GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
1924           pos = NULL;
1925         } /* At beginning of actual string */
1926         else if (pos == NULL)
1927         {
1928           pos = &temp_transports[i];
1929         }
1930       }
1931
1932       GNUNET_free (temp_transports);
1933       fclose(temp_file_handle);
1934
1935       if (GNUNET_OK !=
1936           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1937         {
1938           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1939                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1940                       "SERVICEHOME",
1941                       "PATHS");
1942           if (UNLINK (mytemp) != 0)
1943             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1944           GNUNET_free (mytemp);
1945           break;
1946         }
1947
1948       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1949         {
1950           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
1951           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1952                                          "mv", mytemp, arg, NULL);
1953 #if VERBOSE_TESTING
1954           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1955                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1956 #endif
1957
1958           GNUNET_free(arg);
1959         }
1960       else /* Remote, scp the file to the correct place */
1961         {
1962           if (NULL != pg->peers[pg_iter].daemon->username)
1963             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1964           else
1965             GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1966           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1967                                          "scp", mytemp, arg, NULL);
1968
1969 #if VERBOSE_TESTING
1970           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1971                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1972 #endif
1973           GNUNET_free(arg);
1974         }
1975       GNUNET_free (temp_service_path);
1976       GNUNET_free (mytemp);
1977     }
1978
1979   count = 0;
1980   ret = GNUNET_SYSERR;
1981   while ((count < max_wait) && (ret != GNUNET_OK))
1982     {
1983       ret = GNUNET_OK;
1984       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1985         {
1986 #if VERBOSE_TESTING
1987           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1988                       _("Checking copy status of file %d\n"), pg_iter);
1989 #endif
1990           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1991             {
1992               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1993                 {
1994                   ret = GNUNET_SYSERR;
1995                 }
1996               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1997                 {
1998                   ret = GNUNET_SYSERR;
1999                 }
2000               else
2001                 {
2002                   pidarr[pg_iter] = 0;
2003 #if VERBOSE_TESTING
2004             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2005                       _("File %d copied\n"), pg_iter);
2006 #endif
2007                 }
2008             }
2009         }
2010       count++;
2011       if (ret == GNUNET_SYSERR)
2012         {
2013           /* FIXME: why sleep here? -CG */
2014           sleep(1);
2015         }
2016     }
2017
2018 #if VERBOSE_TESTING
2019     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2020                 _("Finished copying all blacklist files!\n"));
2021 #endif
2022   GNUNET_free(pidarr);
2023   return ret;
2024 }
2025
2026
2027 /**
2028  * Internal notification of a connection, kept so that we can ensure some connections
2029  * happen instead of flooding all testing daemons with requests to connect.
2030  */
2031 static void internal_connect_notify (void *cls,
2032                                      const struct GNUNET_PeerIdentity *first,
2033                                      const struct GNUNET_PeerIdentity *second,
2034                                      uint32_t distance,
2035                                      const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2036                                      const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2037                                      struct GNUNET_TESTING_Daemon *first_daemon,
2038                                      struct GNUNET_TESTING_Daemon *second_daemon,
2039                                      const char *emsg)
2040 {
2041   struct GNUNET_TESTING_PeerGroup *pg = cls;
2042   outstanding_connects--;
2043
2044   pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
2045 }
2046
2047
2048 /**
2049  * Either delay a connection (because there are too many outstanding)
2050  * or schedule it for right now.
2051  *
2052  * @param cls a connection context
2053  * @param tc the task runtime context
2054  */
2055 static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2056 {
2057   struct ConnectContext *connect_context = cls;
2058
2059   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2060     return;
2061
2062   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2063     {
2064 #if VERBOSE_TESTING > 2
2065           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2066                       _("Delaying connect, we have too many outstanding connections!\n"));
2067 #endif
2068       GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
2069     }
2070   else
2071     {
2072 #if VERBOSE_TESTING > 2
2073           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2074                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2075 #endif
2076       outstanding_connects++;
2077       GNUNET_TESTING_daemons_connect (connect_context->first,
2078                                       connect_context->second,
2079                                       CONNECT_TIMEOUT,
2080                                       CONNECT_ATTEMPTS,
2081                                       &internal_connect_notify,
2082                                       connect_context->pg);
2083       GNUNET_free(connect_context);
2084     }
2085 }
2086
2087
2088 /**
2089  * Iterator for actually scheduling connections to be created
2090  * between two peers.
2091  *
2092  * @param cls closure, a GNUNET_TESTING_Daemon
2093  * @param key the key the second Daemon was stored under
2094  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2095  *
2096  * @return GNUNET_YES to continue iteration
2097  */
2098 static int
2099 connect_iterator (void *cls,
2100                   const GNUNET_HashCode * key,
2101                   void *value)
2102 {
2103   struct PeerData *first = cls;
2104   struct GNUNET_TESTING_Daemon *second = value;
2105   struct ConnectContext *connect_context;
2106
2107   connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2108   connect_context->pg = first->pg;
2109   connect_context->first = first->daemon;
2110   connect_context->second = second;
2111   GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
2112
2113   return GNUNET_YES;
2114 }
2115
2116
2117 /**
2118  * Iterator for copying all entries in the allowed hashmap to the
2119  * connect hashmap.
2120  *
2121  * @param cls closure, a GNUNET_TESTING_Daemon
2122  * @param key the key the second Daemon was stored under
2123  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2124  *
2125  * @return GNUNET_YES to continue iteration
2126  */
2127 static int
2128 copy_topology_iterator (void *cls,
2129                   const GNUNET_HashCode * key,
2130                   void *value)
2131 {
2132   struct PeerData *first = cls;
2133
2134   GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2135
2136   return GNUNET_YES;
2137 }
2138
2139 /**
2140  * Make the peers to connect the same as those that are allowed to be
2141  * connected.
2142  *
2143  * @param pg the peer group
2144  */
2145 static int
2146 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2147 {
2148   unsigned int pg_iter;
2149   int ret;
2150   int total;
2151
2152   total = 0;
2153   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2154     {
2155       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
2156       if (GNUNET_SYSERR == ret)
2157         return GNUNET_SYSERR;
2158
2159       total = total + ret;
2160     }
2161
2162   return total;
2163 }
2164
2165
2166 /**
2167  * Connect the topology as specified by the PeerConnection's
2168  * of each peer in the peer group
2169  *
2170  * @param pg the peer group we are dealing with
2171  * @return the number of connections that will be attempted
2172  */
2173 static int
2174 connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
2175 {
2176   unsigned int pg_iter;
2177   int ret;
2178   int total;
2179 #if OLD
2180   struct PeerConnection *connection_iter;
2181   struct ConnectContext *connect_context;
2182 #endif
2183
2184   total = 0;
2185   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2186     {
2187       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
2188       if (GNUNET_SYSERR == ret)
2189         return GNUNET_SYSERR;
2190
2191       total = total + ret;
2192
2193 #if OLD
2194       connection_iter = ;
2195       while (connection_iter != NULL)
2196         {
2197           connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2198           connect_context->pg = pg;
2199           connect_context->first = ;
2200           connect_context->second = connection_iter->daemon;
2201           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
2202           connection_iter = connection_iter->next;
2203         }
2204 #endif
2205     }
2206   return total;
2207 }
2208
2209
2210 /**
2211  * Takes a peer group and creates a topology based on the
2212  * one specified.  Creates a topology means generates friend
2213  * files for the peers so they can only connect to those allowed
2214  * by the topology.  This will only have an effect once peers
2215  * are started if the FRIENDS_ONLY option is set in the base
2216  * config.  Also takes an optional restrict topology which
2217  * disallows connections based on a particular transport
2218  * UNLESS they are specified in the restricted topology.
2219  *
2220  * @param pg the peer group struct representing the running peers
2221  * @param topology which topology to connect the peers in
2222  * @param restrict_topology allow only direct TCP connections in this topology
2223  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2224  * @param restrict_transports space delimited list of transports to blacklist
2225  *                            to create restricted topology
2226  *
2227  * @return the maximum number of connections were all allowed peers
2228  *         connected to each other
2229  */
2230 int
2231 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2232                                 enum GNUNET_TESTING_Topology topology,
2233                                 enum GNUNET_TESTING_Topology restrict_topology,
2234                                 char *restrict_transports)
2235 {
2236   int ret;
2237   int num_connections;
2238   int unblacklisted_connections;
2239
2240   GNUNET_assert (pg->notify_connection != NULL);
2241   ret = GNUNET_OK;
2242
2243   switch (topology)
2244     {
2245     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2246 #if VERBOSE_TESTING
2247       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2248                   _("Creating clique topology\n"));
2249 #endif
2250       num_connections = create_clique (pg, &add_allowed_connections);
2251       break;
2252     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2253 #if VERBOSE_TESTING
2254       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2255                   _("Creating small world (ring) topology\n"));
2256 #endif
2257       num_connections = create_small_world_ring (pg, &add_allowed_connections);
2258       break;
2259     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2260 #if VERBOSE_TESTING
2261       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2262                   _("Creating small world (2d-torus) topology\n"));
2263 #endif
2264       num_connections = create_small_world (pg, &add_allowed_connections);
2265       break;
2266     case GNUNET_TESTING_TOPOLOGY_RING:
2267 #if VERBOSE_TESTING
2268       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2269                   _("Creating ring topology\n"));
2270 #endif
2271       num_connections = create_ring (pg, &add_allowed_connections);
2272       break;
2273     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2274 #if VERBOSE_TESTING
2275       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2276                   _("Creating 2d torus topology\n"));
2277 #endif
2278       num_connections = create_2d_torus (pg, &add_allowed_connections);
2279       break;
2280     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2281 #if VERBOSE_TESTING
2282       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2283                   _("Creating Erdos-Renyi topology\n"));
2284 #endif
2285       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2286       break;
2287     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2288 #if VERBOSE_TESTING
2289       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2290                   _("Creating InterNAT topology\n"));
2291 #endif
2292       num_connections = create_nated_internet (pg, &add_allowed_connections);
2293       break;
2294     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2295 #if VERBOSE_TESTING
2296       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2297                   _("Creating Scale Free topology\n"));
2298 #endif
2299       num_connections = create_scale_free (pg, &add_allowed_connections);
2300       break;
2301     case GNUNET_TESTING_TOPOLOGY_LINE:
2302 #if VERBOSE_TESTING
2303       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2304                   _("Creating straight line topology\n"));
2305 #endif
2306       num_connections = create_line (pg, &add_allowed_connections);
2307       break;
2308     case GNUNET_TESTING_TOPOLOGY_NONE:
2309 #if VERBOSE_TESTING
2310       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2311                   _("Creating no allowed topology (all peers can connect at core level)\n"));
2312 #endif
2313       num_connections = 0;
2314       break;
2315     default:
2316       num_connections = 0;
2317       break;
2318     }
2319
2320   if (num_connections < 0)
2321     return GNUNET_SYSERR;
2322
2323   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2324     {
2325       ret = create_and_copy_friend_files(pg);
2326       if (ret != GNUNET_OK)
2327         {
2328 #if VERBOSE_TESTING
2329           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2330                       _("Failed during friend file copying!\n"));
2331 #endif
2332           return GNUNET_SYSERR;
2333         }
2334       else
2335         {
2336 #if VERBOSE_TESTING
2337               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2338                           _("Friend files created/copied successfully!\n"));
2339 #endif
2340         }
2341     }
2342
2343   /* Use the create clique method to initially set all connections as blacklisted. */
2344   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2345     create_clique (pg, &blacklist_connections);
2346   unblacklisted_connections = 0;
2347   /* Un-blacklist connections as per the topology specified */
2348   switch (restrict_topology)
2349     {
2350     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2351 #if VERBOSE_TESTING
2352       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2353                   _("Blacklisting all but clique topology\n"));
2354 #endif
2355       unblacklisted_connections = create_clique (pg, &unblacklist_connections);
2356       break;
2357     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2358 #if VERBOSE_TESTING
2359       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2360                   _("Blacklisting all but small world (ring) topology\n"));
2361 #endif
2362       unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
2363       break;
2364     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2365 #if VERBOSE_TESTING
2366       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2367                   _("Blacklisting all but small world (2d-torus) topology\n"));
2368 #endif
2369       unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
2370       break;
2371     case GNUNET_TESTING_TOPOLOGY_RING:
2372 #if VERBOSE_TESTING
2373       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2374                   _("Blacklisting all but ring topology\n"));
2375 #endif
2376       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2377       break;
2378     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2379 #if VERBOSE_TESTING
2380       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2381                   _("Blacklisting all but 2d torus topology\n"));
2382 #endif
2383       unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
2384       break;
2385     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2386 #if VERBOSE_TESTING
2387       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2388                   _("Blacklisting all but Erdos-Renyi topology\n"));
2389 #endif
2390       unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
2391       break;
2392     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2393 #if VERBOSE_TESTING
2394       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2395                   _("Blacklisting all but InterNAT topology\n"));
2396 #endif
2397       unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
2398       break;
2399     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2400 #if VERBOSE_TESTING
2401       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2402                   _("Blacklisting all but Scale Free topology\n"));
2403 #endif
2404       unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
2405       break;
2406     case GNUNET_TESTING_TOPOLOGY_LINE:
2407 #if VERBOSE_TESTING
2408       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2409                   _("Blacklisting all but straight line topology\n"));
2410 #endif
2411       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2412       break;
2413     case GNUNET_TESTING_TOPOLOGY_NONE:
2414 #if VERBOSE_TESTING
2415       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2416                   _("Creating no blacklist topology (all peers can connect at transport level)\n"));
2417 #endif
2418     default:
2419       break;
2420     }
2421
2422   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2423   {
2424     ret = create_and_copy_blacklist_files(pg, restrict_transports);
2425     if (ret != GNUNET_OK)
2426       {
2427 #if VERBOSE_TESTING
2428         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2429                     _("Failed during blacklist file copying!\n"));
2430 #endif
2431         return GNUNET_SYSERR;
2432       }
2433     else
2434       {
2435 #if VERBOSE_TESTING
2436         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2437                     _("Blacklist files created/copied successfully!\n"));
2438 #endif
2439       }
2440   }
2441   return num_connections;
2442 }
2443
2444 struct RandomContext
2445 {
2446   /**
2447    * The peergroup
2448    */
2449   struct GNUNET_TESTING_PeerGroup *pg;
2450
2451   /**
2452    * uid of the first peer
2453    */
2454   uint32_t first_uid;
2455
2456   /**
2457    * Peer data for first peer.
2458    */
2459   struct PeerData *first;
2460
2461   /**
2462    * Random percentage to use
2463    */
2464   double percentage;
2465 };
2466
2467 struct MinimumContext
2468 {
2469   /**
2470    * The peergroup
2471    */
2472   struct GNUNET_TESTING_PeerGroup *pg;
2473
2474   /**
2475    * uid of the first peer
2476    */
2477   uint32_t first_uid;
2478
2479   /**
2480    * Peer data for first peer.
2481    */
2482   struct PeerData *first;
2483
2484   /**
2485    * Number of conns per peer
2486    */
2487   unsigned int num_to_add;
2488
2489   /**
2490    * Permuted array of all possible connections.  Only add the Nth
2491    * peer if it's in the Nth position.
2492    */
2493   unsigned int *pg_array;
2494
2495   /**
2496    * What number is the current element we are iterating over?
2497    */
2498   unsigned int current;
2499 };
2500
2501 struct DFSContext
2502 {
2503   /**
2504    * The peergroup
2505    */
2506   struct GNUNET_TESTING_PeerGroup *pg;
2507
2508   /**
2509    * uid of the first peer
2510    */
2511   uint32_t first_uid;
2512
2513   /**
2514    * uid of the second peer
2515    */
2516   uint32_t second_uid;
2517
2518   /**
2519    * Peer data for first peer.
2520    */
2521   struct PeerData *first;
2522
2523   /**
2524    * Which peer has been chosen as the one to add?
2525    */
2526   unsigned int chosen;
2527
2528   /**
2529    * What number is the current element we are iterating over?
2530    */
2531   unsigned int current;
2532 };
2533
2534 /**
2535  * Iterator for choosing random peers to connect.
2536  *
2537  * @param cls closure, a RandomContext
2538  * @param key the key the second Daemon was stored under
2539  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2540  *
2541  * @return GNUNET_YES to continue iteration
2542  */
2543 static int
2544 random_connect_iterator (void *cls,
2545                   const GNUNET_HashCode * key,
2546                   void *value)
2547 {
2548   struct RandomContext *random_ctx = cls;
2549   double random_number;
2550   uint32_t second_pos;
2551   GNUNET_HashCode first_hash;
2552   random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
2553                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
2554   if (random_number < random_ctx->percentage)
2555   {
2556     GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2557   }
2558   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2559   uid_from_hash(key, &second_pos);
2560   hash_from_uid(random_ctx->first_uid, &first_hash);
2561   GNUNET_assert(random_ctx->pg->total > second_pos);
2562   GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
2563
2564   return GNUNET_YES;
2565 }
2566
2567 /**
2568  * Iterator for adding at least X peers to a peers connection set.
2569  *
2570  * @param cls closure, MinimumContext
2571  * @param key the key the second Daemon was stored under
2572  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2573  *
2574  * @return GNUNET_YES to continue iteration
2575  */
2576 static int
2577 minimum_connect_iterator (void *cls,
2578                   const GNUNET_HashCode * key,
2579                   void *value)
2580 {
2581   struct MinimumContext *min_ctx = cls;
2582   uint32_t second_pos;
2583   GNUNET_HashCode first_hash;
2584   unsigned int i;
2585
2586   if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2587   {
2588     for (i = 0; i < min_ctx->num_to_add; i++)
2589     {
2590       if (min_ctx->pg_array[i] == min_ctx->current)
2591       {
2592         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2593         uid_from_hash(key, &second_pos);
2594         hash_from_uid(min_ctx->first_uid, &first_hash);
2595         GNUNET_assert(min_ctx->pg->total > second_pos);
2596         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2597         /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2598         GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
2599       }
2600     }
2601     min_ctx->current++;
2602     return GNUNET_YES;
2603   }
2604   else
2605     return GNUNET_NO; /* We can stop iterating, we have enough peers! */
2606
2607 }
2608
2609
2610 /**
2611  * Iterator for adding peers to a connection set based on a depth first search.
2612  *
2613  * @param cls closure, MinimumContext
2614  * @param key the key the second daemon was stored under
2615  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2616  *
2617  * @return GNUNET_YES to continue iteration
2618  */
2619 static int
2620 dfs_connect_iterator (void *cls,
2621                   const GNUNET_HashCode * key,
2622                   void *value)
2623 {
2624   struct DFSContext *dfs_ctx = cls;
2625   GNUNET_HashCode first_hash;
2626
2627   if (dfs_ctx->current == dfs_ctx->chosen)
2628     {
2629       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2630       uid_from_hash(key, &dfs_ctx->second_uid);
2631       hash_from_uid(dfs_ctx->first_uid, &first_hash);
2632       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2633       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
2634       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
2635       return GNUNET_NO; /* We have found our peer, don't iterate more */
2636     }
2637
2638   dfs_ctx->current++;
2639   return GNUNET_YES;
2640 }
2641
2642
2643 /**
2644  * From the set of connections possible, choose percentage percent of connections
2645  * to actually connect.
2646  *
2647  * @param pg the peergroup we are dealing with
2648  * @param percentage what percent of total connections to make
2649  */
2650 void
2651 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
2652 {
2653   struct RandomContext random_ctx;
2654   uint32_t pg_iter;
2655
2656   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2657     {
2658       random_ctx.first_uid = pg_iter;
2659       random_ctx.first = &pg->peers[pg_iter];
2660       random_ctx.percentage = percentage;
2661       random_ctx.pg = pg;
2662       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
2663       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
2664       /* Now remove the old connections */
2665       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2666       /* And replace with the random set */
2667       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2668     }
2669 }
2670
2671 /**
2672  * From the set of connections possible, choose at least num connections per
2673  * peer.
2674  *
2675  * @param pg the peergroup we are dealing with
2676  * @param num how many connections at least should each peer have (if possible)?
2677  */
2678 static void
2679 choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2680 {
2681   struct MinimumContext minimum_ctx;
2682   uint32_t pg_iter;
2683
2684   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2685    {
2686      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2687    }
2688
2689   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2690     {
2691       minimum_ctx.first_uid = pg_iter;
2692       minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK,
2693                                                           GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
2694       minimum_ctx.first = &pg->peers[pg_iter];
2695       minimum_ctx.pg = pg;
2696       minimum_ctx.num_to_add = num;
2697       minimum_ctx.current = 0;
2698       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
2699                                             &minimum_connect_iterator,
2700                                             &minimum_ctx);
2701     }
2702
2703   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2704     {
2705       /* Remove the "old" connections */
2706       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2707       /* And replace with the working set */
2708       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2709     }
2710
2711 }
2712
2713
2714 static unsigned int
2715 count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
2716 {
2717   unsigned int count;
2718   unsigned int pg_iter;
2719
2720   count = 0;
2721
2722   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2723     {
2724       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2725     }
2726
2727   return count;
2728 }
2729
2730
2731 static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
2732 {
2733   unsigned int count;
2734   unsigned int pg_iter;
2735
2736   count = 0;
2737
2738   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2739     {
2740       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
2741     }
2742
2743   return count;
2744 }
2745
2746 /**
2747  * From the set of connections possible, choose at least num connections per
2748  * peer based on depth first traversal of peer connections.  If DFS leaves
2749  * peers unconnected, ensure those peers get connections.
2750  *
2751  * @param pg the peergroup we are dealing with
2752  * @param num how many connections at least should each peer have (if possible)?
2753  */
2754 void
2755 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2756 {
2757   struct DFSContext dfs_ctx;
2758   uint32_t pg_iter;
2759   uint32_t dfs_count;
2760   uint32_t starting_peer;
2761   uint32_t least_connections;
2762   GNUNET_HashCode second_hash;
2763
2764   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2765     {
2766       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2767     }
2768
2769   starting_peer = 0;
2770   dfs_count = 0;
2771   while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
2772     {
2773       if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
2774         {
2775           least_connections = -1; /* Set to very high number */
2776           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2777             {
2778               if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
2779                 {
2780                   starting_peer = pg_iter;
2781                   least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2782                 }
2783             }
2784         }
2785
2786       if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
2787         {
2788           dfs_count = 0;
2789           continue;
2790         }
2791
2792       /* Choose a random peer from the chosen peers set of connections to add */
2793       dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
2794       dfs_ctx.first_uid = starting_peer;
2795       dfs_ctx.first = &pg->peers[starting_peer];
2796       dfs_ctx.pg = pg;
2797       dfs_ctx.current = 0;
2798
2799       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
2800       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
2801       hash_from_uid(dfs_ctx.second_uid, &second_hash);
2802       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
2803       starting_peer = dfs_ctx.second_uid;
2804     }
2805
2806   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2807     {
2808       /* Remove the "old" connections */
2809       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2810       /* And replace with the working set */
2811       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2812     }
2813 }
2814
2815 /**
2816  * Internal callback for topology information for a particular peer.
2817  */
2818 static void
2819 internal_topology_callback(void *cls,
2820                            const struct GNUNET_PeerIdentity *peer,
2821                            struct GNUNET_TIME_Relative latency, uint32_t distance)
2822 {
2823   struct CoreContext *core_ctx = cls;
2824   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
2825
2826   if (peer == NULL) /* Either finished, or something went wrong */
2827     {
2828       iter_ctx->completed++;
2829       iter_ctx->connected--;
2830       /* One core context allocated per iteration, must free! */
2831       GNUNET_free(core_ctx);
2832     }
2833   else
2834     {
2835       iter_ctx->topology_cb(iter_ctx->cls, &core_ctx->daemon->id, peer, latency, distance, NULL);
2836     }
2837
2838   if (iter_ctx->completed == iter_ctx->total)
2839     {
2840       iter_ctx->topology_cb(iter_ctx->cls, NULL, NULL, GNUNET_TIME_relative_get_zero(), 0, NULL);
2841       /* Once all are done, free the iteration context */
2842       GNUNET_free(iter_ctx);
2843     }
2844 }
2845
2846
2847 /**
2848  * Check running topology iteration tasks, if below max start a new one, otherwise
2849  * schedule for some time in the future.
2850  */
2851 static void
2852 schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2853 {
2854   struct CoreContext *core_context = cls;
2855   struct TopologyIterateContext *topology_context = (struct TopologyIterateContext *)core_context->iter_context;
2856   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2857     return;
2858
2859   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2860     {
2861 #if VERBOSE_TESTING > 2
2862           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2863                       _("Delaying connect, we have too many outstanding connections!\n"));
2864 #endif
2865       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_topology, core_context);
2866     }
2867   else
2868     {
2869 #if VERBOSE_TESTING > 2
2870           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2871                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2872 #endif
2873       topology_context->connected++;
2874       if (GNUNET_OK != GNUNET_CORE_iterate_peers (core_context->daemon->sched, core_context->daemon->cfg, &internal_topology_callback, core_context))
2875         internal_topology_callback(core_context, NULL, GNUNET_TIME_relative_get_zero(), 0);
2876
2877     }
2878 }
2879
2880 /**
2881  * Iterate over all (running) peers in the peer group, retrieve
2882  * all connections that each currently has.
2883  */
2884 void
2885 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyTopology cb, void *cls)
2886 {
2887   struct TopologyIterateContext *topology_context;
2888   struct CoreContext *core_ctx;
2889   unsigned int i;
2890   unsigned int total_count;
2891
2892   /* Allocate a single topology iteration context */
2893   topology_context = GNUNET_malloc(sizeof(struct TopologyIterateContext));
2894   topology_context->topology_cb = cb;
2895   topology_context->cls = cls;
2896   total_count = 0;
2897   for (i = 0; i < pg->total; i++)
2898     {
2899       if (pg->peers[i].daemon->running == GNUNET_YES)
2900         {
2901           /* Allocate one core context per core we need to connect to */
2902           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
2903           core_ctx->daemon = pg->peers[i].daemon;
2904           /* Set back pointer to topology iteration context */
2905           core_ctx->iter_context = topology_context;
2906           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_topology, core_ctx);
2907           total_count++;
2908         }
2909     }
2910   topology_context->total = total_count;
2911   return;
2912 }
2913
2914 /**
2915  * Callback function to process statistic values.
2916  * This handler is here only really to insert a peer
2917  * identity (or daemon) so the statistics can be uniquely
2918  * tied to a single running peer.
2919  *
2920  * @param cls closure
2921  * @param subsystem name of subsystem that created the statistic
2922  * @param name the name of the datum
2923  * @param value the current value
2924  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
2925  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
2926  */
2927 static int internal_stats_callback (void *cls,
2928                                     const char *subsystem,
2929                                     const char *name,
2930                                     uint64_t value,
2931                                     int is_persistent)
2932 {
2933   struct CoreContext *core_context = cls;
2934   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2935
2936   return stats_context->proc(stats_context->cls, &core_context->daemon->id, subsystem, name, value, is_persistent);
2937 }
2938
2939 /**
2940  * Internal continuation call for statistics iteration.
2941  *
2942  * @param cls closure, the CoreContext for this iteration
2943  * @param success whether or not the statistics iterations
2944  *        was canceled or not (we don't care)
2945  */
2946 static void internal_stats_cont (void *cls, int success)
2947 {
2948   struct CoreContext *core_context = cls;
2949   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2950
2951   stats_context->connected--;
2952   stats_context->completed++;
2953
2954   if (stats_context->completed == stats_context->total)
2955     {
2956       stats_context->cont(stats_context->cls, GNUNET_YES);
2957       if (stats_context->stats_handle != NULL)
2958         GNUNET_STATISTICS_destroy(stats_context->stats_handle, GNUNET_NO);
2959       GNUNET_free(stats_context);
2960     }
2961   GNUNET_free(core_context);
2962 }
2963
2964 /**
2965  * Check running topology iteration tasks, if below max start a new one, otherwise
2966  * schedule for some time in the future.
2967  */
2968 static void
2969 schedule_get_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2970 {
2971   struct CoreContext *core_context = cls;
2972   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2973
2974   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2975     return;
2976
2977   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2978     {
2979 #if VERBOSE_TESTING > 2
2980           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2981                       _("Delaying connect, we have too many outstanding connections!\n"));
2982 #endif
2983       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_statistics, core_context);
2984     }
2985   else
2986     {
2987 #if VERBOSE_TESTING > 2
2988           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2989                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2990 #endif
2991
2992       stats_context->connected++;
2993       stats_context->stats_handle = GNUNET_STATISTICS_create(core_context->daemon->sched, "testing", core_context->daemon->cfg);
2994       if (stats_context->stats_handle == NULL)
2995         {
2996           internal_stats_cont (core_context, GNUNET_NO);
2997           return;
2998         }
2999
3000       stats_context->stats_get_handle = GNUNET_STATISTICS_get(stats_context->stats_handle, NULL, NULL, GNUNET_TIME_relative_get_forever(), &internal_stats_cont, &internal_stats_callback, core_context);
3001       if (stats_context->stats_get_handle == NULL)
3002          internal_stats_cont (core_context, GNUNET_NO);
3003
3004     }
3005 }
3006
3007
3008 /**
3009  * Iterate over all (running) peers in the peer group, retrieve
3010  * all statistics from each.
3011  */
3012 void
3013 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3014                                GNUNET_STATISTICS_Callback cont,
3015                                GNUNET_TESTING_STATISTICS_Iterator proc, void *cls)
3016 {
3017   struct StatsIterateContext *stats_context;
3018   struct CoreContext *core_ctx;
3019   unsigned int i;
3020   unsigned int total_count;
3021
3022   /* Allocate a single stats iteration context */
3023   stats_context = GNUNET_malloc(sizeof(struct StatsIterateContext));
3024   stats_context->cont = cont;
3025   stats_context->proc = proc;
3026   stats_context->cls = cls;
3027   total_count = 0;
3028   for (i = 0; i < pg->total; i++)
3029     {
3030       if (pg->peers[i].daemon->running == GNUNET_YES)
3031         {
3032           /* Allocate one core context per core we need to connect to */
3033           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
3034           core_ctx->daemon = pg->peers[i].daemon;
3035           /* Set back pointer to topology iteration context */
3036           core_ctx->iter_context = stats_context;
3037           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_statistics, core_ctx);
3038           total_count++;
3039         }
3040     }
3041   stats_context->total = total_count;
3042   return;
3043 }
3044
3045 /**
3046  * There are many ways to connect peers that are supported by this function.
3047  * To connect peers in the same topology that was created via the
3048  * GNUNET_TESTING_create_topology, the topology variable must be set to
3049  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3050  * a new instance of that topology will be generated and attempted to be
3051  * connected.  This could result in some connections being impossible,
3052  * because some topologies are non-deterministic.
3053  *
3054  * @param pg the peer group struct representing the running peers
3055  * @param topology which topology to connect the peers in
3056  * @param options options for connecting the topology
3057  * @param option_modifier modifier for options that take a parameter
3058  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3059  */
3060 int
3061 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3062                                  enum GNUNET_TESTING_Topology topology,
3063                                  enum GNUNET_TESTING_TopologyOption options,
3064                                  double option_modifier)
3065 {
3066   switch (topology)
3067       {
3068       case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3069 #if VERBOSE_TOPOLOGY
3070       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3071                   _("Creating clique CONNECT topology\n"));
3072 #endif
3073         create_clique (pg, &add_actual_connections);
3074         break;
3075       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3076 #if VERBOSE_TOPOLOGY
3077       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3078                   _("Creating small world (ring) CONNECT topology\n"));
3079 #endif
3080         create_small_world_ring (pg, &add_actual_connections);
3081         break;
3082       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3083 #if VERBOSE_TOPOLOGY
3084       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3085                   _("Creating small world (2d-torus) CONNECT topology\n"));
3086 #endif
3087         create_small_world (pg, &add_actual_connections);
3088         break;
3089       case GNUNET_TESTING_TOPOLOGY_RING:
3090 #if VERBOSE_TOPOLOGY
3091       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3092                   _("Creating ring CONNECT topology\n"));
3093 #endif
3094         create_ring (pg, &add_actual_connections);
3095         break;
3096       case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3097 #if VERBOSE_TOPOLOGY
3098       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3099                   _("Creating 2d torus CONNECT topology\n"));
3100 #endif
3101         create_2d_torus (pg, &add_actual_connections);
3102         break;
3103       case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3104 #if VERBOSE_TOPOLOGY
3105       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3106                   _("Creating Erdos-Renyi CONNECT topology\n"));
3107 #endif
3108         create_erdos_renyi (pg, &add_actual_connections);
3109         break;
3110       case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3111 #if VERBOSE_TOPOLOGY
3112       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3113                   _("Creating InterNAT CONNECT topology\n"));
3114 #endif
3115         create_nated_internet (pg, &add_actual_connections);
3116         break;
3117       case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3118 #if VERBOSE_TOPOLOGY
3119       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3120                   _("Creating Scale Free CONNECT topology\n"));
3121 #endif
3122         create_scale_free (pg, &add_actual_connections);
3123         break;
3124       case GNUNET_TESTING_TOPOLOGY_LINE:
3125 #if VERBOSE_TOPOLOGY
3126       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3127                   _("Creating straight line CONNECT topology\n"));
3128 #endif
3129         create_line (pg, &add_actual_connections);
3130         break;
3131       case GNUNET_TESTING_TOPOLOGY_NONE:
3132 #if VERBOSE_TOPOLOGY
3133         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3134                   _("Creating no CONNECT topology\n"));
3135 #endif
3136         copy_allowed_topology(pg);
3137         break;
3138       default:
3139         GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3140                    _("Unknown topology specification, can't connect peers!\n"));
3141         return GNUNET_SYSERR;
3142       }
3143
3144   switch (options)
3145     {
3146     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3147 #if VERBOSE_TOPOLOGY
3148       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3149                   _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
3150 #endif
3151       choose_random_connections(pg, option_modifier);
3152       break;
3153     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3154 #if VERBOSE_TOPOLOGY
3155       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3156                   _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3157 #endif
3158       choose_minimum(pg, (unsigned int)option_modifier);
3159       break;
3160     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3161 #if VERBOSE_TOPOLOGY
3162       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3163                   _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3164 #endif
3165       perform_dfs(pg, (int)option_modifier);
3166       break;
3167     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3168       break;
3169     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3170       break;
3171     default:
3172       break;
3173     }
3174
3175   return connect_topology(pg);
3176 }
3177
3178 /**
3179  * Callback that is called whenever a hostkey is generated
3180  * for a peer.  Call the real callback and decrement the
3181  * starting counter for the peergroup.
3182  *
3183  * @param cls closure
3184  * @param id identifier for the daemon, NULL on error
3185  * @param d handle for the daemon
3186  * @param emsg error message (NULL on success)
3187  */
3188 static void internal_hostkey_callback (void *cls,
3189                                        const struct GNUNET_PeerIdentity *id,
3190                                        struct GNUNET_TESTING_Daemon *d,
3191                                        const char *emsg)
3192 {
3193   struct InternalStartContext *internal_context = cls;
3194   internal_context->peer->pg->starting--;
3195   internal_context->peer->pg->started++;
3196   if (internal_context->hostkey_callback != NULL)
3197     internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
3198   else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
3199     {
3200       internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
3201       GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
3202     }
3203 }
3204
3205 /**
3206  * Callback that is called whenever a peer has finished starting.
3207  * Call the real callback and decrement the starting counter
3208  * for the peergroup.
3209  *
3210  * @param cls closure
3211  * @param id identifier for the daemon, NULL on error
3212  * @param d handle for the daemon
3213  * @param emsg error message (NULL on success)
3214  */
3215 static void internal_startup_callback (void *cls,
3216                                        const struct GNUNET_PeerIdentity *id,
3217                                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3218                                        struct GNUNET_TESTING_Daemon *d,
3219                                        const char *emsg)
3220 {
3221   struct InternalStartContext *internal_context = cls;
3222   internal_context->peer->pg->starting--;
3223   if (internal_context->start_cb != NULL)
3224     internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
3225 }
3226
3227 static void
3228 internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3229 {
3230   struct InternalStartContext *internal_context = cls;
3231
3232   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3233     {
3234       return;
3235     }
3236
3237   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3238     {
3239       internal_context->peer->pg->starting++;
3240       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3241     }
3242   else
3243     {
3244       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
3245     }
3246 }
3247
3248 static void
3249 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3250 {
3251   struct InternalStartContext *internal_context = cls;
3252
3253   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3254     {
3255       return;
3256     }
3257
3258   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
3259     {
3260       internal_context->peer->pg->starting++;
3261       internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
3262                                                                     internal_context->peer->cfg,
3263                                                                     internal_context->timeout,
3264                                                                     internal_context->hostname,
3265                                                                     &internal_hostkey_callback,
3266                                                                     internal_context,
3267                                                                     &internal_startup_callback,
3268                                                                     internal_context);
3269     }
3270   else
3271     {
3272       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
3273     }
3274 }
3275
3276 /**
3277  * Function which continues a peer group starting up
3278  * after successfully generating hostkeys for each peer.
3279  *
3280  * @param pg the peer group to continue starting
3281  *
3282  */
3283 void
3284 GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
3285 {
3286   unsigned int i;
3287
3288   pg->starting = 0;
3289   for (i = 0; i < pg->total; i++)
3290     {
3291       GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
3292       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
3293     }
3294 }
3295
3296 /**
3297  * Start count gnunet instances with the same set of transports and
3298  * applications.  The port numbers (any option called "PORT") will be
3299  * adjusted to ensure that no two peers running on the same system
3300  * have the same port(s) in their respective configurations.
3301  *
3302  * @param sched scheduler to use
3303  * @param cfg configuration template to use
3304  * @param total number of daemons to start
3305  * @param timeout total time allowed for peers to start
3306  * @param hostkey_callback function to call on each peers hostkey generation
3307  *        if NULL, peers will be started by this call, if non-null,
3308  *        GNUNET_TESTING_daemons_continue_startup must be called after
3309  *        successful hostkey generation
3310  * @param hostkey_cls closure for hostkey callback
3311  * @param cb function to call on each daemon that was started
3312  * @param cb_cls closure for cb
3313  * @param connect_callback function to call each time two hosts are connected
3314  * @param connect_callback_cls closure for connect_callback
3315  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
3316  *
3317  * @return NULL on error, otherwise handle to control peer group
3318  */
3319 struct GNUNET_TESTING_PeerGroup *
3320 GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
3321                               const struct GNUNET_CONFIGURATION_Handle *cfg,
3322                               unsigned int total,
3323                               struct GNUNET_TIME_Relative timeout,
3324                               GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
3325                               void *hostkey_cls,
3326                               GNUNET_TESTING_NotifyDaemonRunning cb,
3327                               void *cb_cls,
3328                               GNUNET_TESTING_NotifyConnection
3329                               connect_callback, void *connect_callback_cls,
3330                               const struct GNUNET_TESTING_Host *hostnames)
3331 {
3332   struct GNUNET_TESTING_PeerGroup *pg;
3333   const struct GNUNET_TESTING_Host *hostpos;
3334 #if 0
3335   char *pos;
3336   const char *rpos;
3337   char *start;
3338 #endif
3339   const char *hostname;
3340   char *baseservicehome;
3341   char *newservicehome;
3342   char *tmpdir;
3343   struct GNUNET_CONFIGURATION_Handle *pcfg;
3344   unsigned int off;
3345   unsigned int hostcnt;
3346   uint16_t minport;
3347   uint32_t upnum;
3348
3349   if (0 == total)
3350     {
3351       GNUNET_break (0);
3352       return NULL;
3353     }
3354   upnum = 0;
3355   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
3356   pg->sched = sched;
3357   pg->cfg = cfg;
3358   pg->notify_connection = connect_callback;
3359   pg->notify_connection_cls = connect_callback_cls;
3360   pg->total = total;
3361   pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
3362   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
3363   if (NULL != hostnames)
3364     {
3365       off = 2;
3366       hostpos = hostnames;
3367       while (hostpos != NULL)
3368         {
3369           hostpos = hostpos->next;
3370           off++;
3371         }
3372       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3373       off = 0;
3374
3375       hostpos = hostnames;
3376       while (hostpos != NULL)
3377         {
3378           pg->hosts[off].minport = LOW_PORT;
3379           pg->hosts[off++].hostname = GNUNET_strdup(hostpos->hostname);
3380           hostpos = hostpos->next;
3381         }
3382
3383       if (off == 0)
3384         {
3385           GNUNET_free (pg->hosts);
3386           pg->hosts = NULL;
3387         }
3388       hostcnt = off;
3389       minport = 0;
3390
3391 #if NO_LL
3392       off = 2;
3393       /* skip leading spaces */
3394       while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
3395         hostnames++;
3396       rpos = hostnames;
3397       while ('\0' != *rpos)
3398         {
3399           if (isspace ( (unsigned char) *rpos))
3400             off++;
3401           rpos++;
3402         }
3403       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3404       off = 0;
3405       start = GNUNET_strdup (hostnames);
3406       pos = start;
3407       while ('\0' != *pos)
3408         {
3409           if (isspace ( (unsigned char) *pos))
3410             {
3411               *pos = '\0';
3412               if (strlen (start) > 0)
3413                 {
3414                   pg->hosts[off].minport = LOW_PORT;
3415                   pg->hosts[off++].hostname = start;
3416                 }
3417               start = pos + 1;
3418             }
3419           pos++;
3420         }
3421       if (strlen (start) > 0)
3422         {
3423           pg->hosts[off].minport = LOW_PORT;
3424           pg->hosts[off++].hostname = start;
3425         }
3426       if (off == 0)
3427         {
3428           GNUNET_free (start);
3429           GNUNET_free (pg->hosts);
3430           pg->hosts = NULL;
3431         }
3432       hostcnt = off;
3433       minport = 0;              /* make gcc happy */
3434 #endif
3435     }
3436   else
3437     {
3438       hostcnt = 0;
3439       minport = LOW_PORT;
3440     }
3441   for (off = 0; off < total; off++)
3442     {
3443       if (hostcnt > 0)
3444         {
3445           hostname = pg->hosts[off % hostcnt].hostname;
3446           pcfg = make_config (cfg,
3447                               &pg->hosts[off % hostcnt].minport,
3448                               &upnum,
3449                               hostname);
3450         }
3451       else
3452         {
3453           hostname = NULL;
3454           pcfg = make_config (cfg,
3455                               &minport,
3456                               &upnum,
3457                               hostname);
3458         }
3459
3460       if (NULL == pcfg)
3461         {
3462           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3463                       _
3464                       ("Could not create configuration for peer number %u on `%s'!\n"),
3465                       off, hostname == NULL ? "localhost" : hostname);
3466           continue;
3467         }
3468
3469       if (GNUNET_YES ==
3470           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
3471                                                  &baseservicehome))
3472         {
3473           GNUNET_asprintf (&newservicehome,
3474                            "%s/%d/", baseservicehome, off);
3475           GNUNET_free (baseservicehome);
3476         }
3477       else
3478         {
3479           tmpdir = getenv ("TMPDIR");
3480           tmpdir = tmpdir ? tmpdir : "/tmp";
3481           GNUNET_asprintf (&newservicehome,
3482                            "%s/%s/%d/",
3483                            tmpdir,
3484                            "gnunet-testing-test-test", off);
3485         }
3486       GNUNET_CONFIGURATION_set_value_string (pcfg,
3487                                              "PATHS",
3488                                              "SERVICEHOME", newservicehome);
3489       GNUNET_free (newservicehome);
3490       pg->peers[off].cfg = pcfg;
3491       pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
3492       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
3493       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
3494       pg->peers[off].pg = pg;
3495
3496       pg->peers[off].internal_context.peer = &pg->peers[off];
3497       pg->peers[off].internal_context.timeout = timeout;
3498       pg->peers[off].internal_context.hostname = hostname;
3499       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
3500       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
3501       pg->peers[off].internal_context.start_cb = cb;
3502       pg->peers[off].internal_context.start_cb_cls = cb_cls;
3503
3504       GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
3505
3506     }
3507   return pg;
3508 }
3509
3510 /*
3511  * Get a daemon by number, so callers don't have to do nasty
3512  * offsetting operation.
3513  */
3514 struct GNUNET_TESTING_Daemon *
3515 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
3516 {
3517   if (position < pg->total)
3518     return pg->peers[position].daemon;
3519   else
3520     return NULL;
3521 }
3522
3523 /**
3524  * Prototype of a function that will be called when a
3525  * particular operation was completed the testing library.
3526  *
3527  * @param cls closure (a struct RestartContext)
3528  * @param id id of the peer that was restarted
3529  * @param cfg handle to the configuration of the peer
3530  * @param d handle to the daemon that was restarted
3531  * @param emsg NULL on success
3532  */
3533 void restart_callback (void *cls,
3534                        const struct GNUNET_PeerIdentity *id,
3535                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3536                        struct GNUNET_TESTING_Daemon *d,
3537                        const char *emsg)
3538 {
3539   struct RestartContext *restart_context = cls;
3540
3541   if (emsg == NULL)
3542     {
3543       restart_context->peers_restarted++;
3544     }
3545   else
3546     {
3547       restart_context->peers_restart_failed++;
3548     }
3549
3550   if (restart_context->peers_restarted == restart_context->peer_group->total)
3551     {
3552       restart_context->callback(restart_context->callback_cls, NULL);
3553       GNUNET_free(restart_context);
3554     }
3555   else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
3556     {
3557       restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
3558       GNUNET_free(restart_context);
3559     }
3560
3561 }
3562
3563 /**
3564  * Callback for informing us about a successful
3565  * or unsuccessful churn stop call.
3566  *
3567  * @param cls a ChurnContext
3568  * @param emsg NULL on success, non-NULL on failure
3569  *
3570  */
3571 void
3572 churn_stop_callback (void *cls, const char *emsg)
3573 {
3574   struct ChurnContext *churn_ctx = cls;
3575   unsigned int total_left;
3576   char *error_message;
3577
3578   error_message = NULL;
3579   if (emsg != NULL)
3580     {
3581       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3582                  "Churn stop callback failed with error `%s'\n", emsg);
3583       churn_ctx->num_failed_stop++;
3584     }
3585   else
3586     {
3587       churn_ctx->num_to_stop--;
3588     }
3589
3590 #if DEBUG_CHURN
3591   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3592              "Stopped peer, %d left.\n",
3593              churn_ctx->num_to_stop);
3594 #endif
3595   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3596
3597   if (total_left == 0)
3598   {
3599     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3600       {
3601         GNUNET_asprintf(&error_message,
3602                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3603                         churn_ctx->num_failed_start,
3604                         churn_ctx->num_failed_stop);
3605       }
3606     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3607     GNUNET_free_non_null(error_message);
3608     GNUNET_free(churn_ctx);
3609   }
3610 }
3611
3612 /**
3613  * Callback for informing us about a successful
3614  * or unsuccessful churn start call.
3615  *
3616  * @param cls a ChurnContext
3617  * @param id the peer identity of the started peer
3618  * @param cfg the handle to the configuration of the peer
3619  * @param d handle to the daemon for the peer
3620  * @param emsg NULL on success, non-NULL on failure
3621  *
3622  */
3623 void
3624 churn_start_callback (void *cls,
3625                       const struct GNUNET_PeerIdentity *id,
3626                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3627                       struct GNUNET_TESTING_Daemon *d,
3628                       const char *emsg)
3629 {
3630   struct ChurnContext *churn_ctx = cls;
3631   unsigned int total_left;
3632   char *error_message;
3633
3634   error_message = NULL;
3635   if (emsg != NULL)
3636     {
3637       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3638                   "Churn stop callback failed with error `%s'\n",
3639                   emsg);
3640       churn_ctx->num_failed_start++;
3641     }
3642   else
3643     {
3644       churn_ctx->num_to_start--;
3645     }
3646
3647 #if DEBUG_CHURN
3648   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3649              "Started peer, %d left.\n",
3650              churn_ctx->num_to_start);
3651 #endif
3652
3653   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3654
3655   if (total_left == 0)
3656   {
3657     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3658       GNUNET_asprintf(&error_message,
3659                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3660                       churn_ctx->num_failed_start,
3661                       churn_ctx->num_failed_stop);
3662     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3663     GNUNET_free_non_null(error_message);
3664     GNUNET_free(churn_ctx);
3665   }
3666 }
3667
3668
3669 /**
3670  * Simulate churn by stopping some peers (and possibly
3671  * re-starting others if churn is called multiple times).  This
3672  * function can only be used to create leave-join churn (peers "never"
3673  * leave for good).  First "voff" random peers that are currently
3674  * online will be taken offline; then "von" random peers that are then
3675  * offline will be put back online.  No notifications will be
3676  * generated for any of these operations except for the callback upon
3677  * completion.
3678  *
3679  * @param pg handle for the peer group
3680  * @param voff number of peers that should go offline
3681  * @param von number of peers that should come back online;
3682  *            must be zero on first call (since "testbed_start"
3683  *            always starts all of the peers)
3684  * @param timeout how long to wait for operations to finish before
3685  *        giving up
3686  * @param cb function to call at the end
3687  * @param cb_cls closure for cb
3688  */
3689 void
3690 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
3691                               unsigned int voff,
3692                               unsigned int von,
3693                               struct GNUNET_TIME_Relative timeout,
3694                               GNUNET_TESTING_NotifyCompletion cb,
3695                               void *cb_cls)
3696 {
3697   struct ChurnContext *churn_ctx;
3698   unsigned int running;
3699   unsigned int stopped;
3700   unsigned int total_running;
3701   unsigned int total_stopped;
3702   unsigned int i;
3703   unsigned int *running_arr;
3704   unsigned int *stopped_arr;
3705   unsigned int *running_permute;
3706   unsigned int *stopped_permute;
3707
3708   running = 0;
3709   stopped = 0;
3710
3711   if ((von == 0) && (voff == 0)) /* No peers at all? */
3712     {
3713       cb(cb_cls, NULL);
3714       return;
3715     }
3716
3717   for (i = 0; i < pg->total; i++)
3718   {
3719     if (pg->peers[i].daemon->running == GNUNET_YES)
3720     {
3721       GNUNET_assert(running != -1);
3722       running++;
3723     }
3724     else
3725     {
3726       GNUNET_assert(stopped != -1);
3727       stopped++;
3728     }
3729   }
3730
3731   if (voff > running)
3732   {
3733     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
3734     cb(cb_cls, "Trying to stop more peers than are currently running!");
3735     return;
3736   }
3737
3738   if (von > stopped)
3739   {
3740     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
3741     cb(cb_cls, "Trying to start more peers than are currently stopped!");
3742     return;
3743   }
3744
3745   churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3746
3747   running_arr = NULL;
3748   if (running > 0)
3749     running_arr = GNUNET_malloc(running * sizeof(unsigned int));
3750
3751   stopped_arr = NULL;
3752   if (stopped > 0)
3753     stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
3754
3755   running_permute = NULL;
3756   stopped_permute = NULL;
3757
3758   if (running > 0)
3759     running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
3760   if (stopped > 0)
3761     stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
3762
3763   total_running = running;
3764   total_stopped = stopped;
3765   running = 0;
3766   stopped = 0;
3767
3768   churn_ctx->num_to_start = von;
3769   churn_ctx->num_to_stop = voff;
3770   churn_ctx->cb = cb;
3771   churn_ctx->cb_cls = cb_cls;
3772
3773   for (i = 0; i < pg->total; i++)
3774   {
3775     if (pg->peers[i].daemon->running == GNUNET_YES)
3776     {
3777       GNUNET_assert((running_arr != NULL) && (total_running > running));
3778       running_arr[running] = i;
3779       running++;
3780     }
3781     else
3782     {
3783       GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
3784       stopped_arr[stopped] = i;
3785       stopped++;
3786     }
3787   }
3788
3789   for (i = 0; i < voff; i++)
3790   {
3791 #if DEBUG_CHURN
3792     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
3793 #endif
3794     GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
3795                                 timeout,
3796                                 &churn_stop_callback, churn_ctx,
3797                                 GNUNET_NO, GNUNET_YES);
3798   }
3799
3800   for (i = 0; i < von; i++)
3801     {
3802 #if DEBUG_CHURN
3803       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
3804 #endif
3805       GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
3806                                           timeout, &churn_start_callback, churn_ctx);
3807   }
3808
3809   GNUNET_free_non_null(running_arr);
3810   GNUNET_free_non_null(stopped_arr);
3811   GNUNET_free_non_null(running_permute);
3812   GNUNET_free_non_null(stopped_permute);
3813 }
3814
3815
3816 /**
3817  * Restart all peers in the given group.
3818  *
3819  * @param pg the handle to the peer group
3820  * @param callback function to call on completion (or failure)
3821  * @param callback_cls closure for the callback function
3822  */
3823 void
3824 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
3825                                 GNUNET_TESTING_NotifyCompletion callback,
3826                                 void *callback_cls)
3827 {
3828   struct RestartContext *restart_context;
3829   unsigned int off;
3830
3831   if (pg->total > 0)
3832     {
3833       restart_context = GNUNET_malloc(sizeof(struct RestartContext));
3834       restart_context->peer_group = pg;
3835       restart_context->peers_restarted = 0;
3836       restart_context->callback = callback;
3837       restart_context->callback_cls = callback_cls;
3838
3839       for (off = 0; off < pg->total; off++)
3840         {
3841           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
3842         }
3843     }
3844 }
3845
3846 /**
3847  * Start or stop an individual peer from the given group.
3848  *
3849  * @param pg handle to the peer group
3850  * @param offset which peer to start or stop
3851  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
3852  * @param timeout how long to wait for shutdown
3853  * @param cb function to call at the end
3854  * @param cb_cls closure for cb
3855  */
3856 void
3857 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
3858                              unsigned int offset,
3859                              int desired_status,
3860                              struct GNUNET_TIME_Relative timeout,
3861                              GNUNET_TESTING_NotifyCompletion cb,
3862                              void *cb_cls)
3863 {
3864   struct ChurnContext *churn_ctx;
3865
3866   if (GNUNET_NO == desired_status)
3867     {
3868       if (NULL != pg->peers[offset].daemon)
3869         {
3870           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3871           churn_ctx->num_to_start = 0;
3872           churn_ctx->num_to_stop = 1;
3873           churn_ctx->cb = cb;
3874           churn_ctx->cb_cls = cb_cls;
3875           GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon,
3876                                      timeout, &churn_stop_callback, churn_ctx,
3877                                      GNUNET_NO, GNUNET_YES);
3878         }
3879     }
3880   else if (GNUNET_YES == desired_status)
3881     {
3882       if (NULL == pg->peers[offset].daemon)
3883         {
3884           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3885           churn_ctx->num_to_start = 1;
3886           churn_ctx->num_to_stop = 0;
3887           churn_ctx->cb = cb;
3888           churn_ctx->cb_cls = cb_cls;
3889           GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon,
3890                                               timeout, &churn_start_callback, churn_ctx);
3891         }
3892     }
3893   else
3894     GNUNET_break (0);
3895 }
3896
3897
3898 /**
3899  * Callback for shutting down peers in a peer group.
3900  *
3901  * @param cls closure (struct ShutdownContext)
3902  * @param emsg NULL on success
3903  */
3904 void internal_shutdown_callback (void *cls,
3905                                  const char *emsg)
3906 {
3907   struct ShutdownContext *shutdown_ctx = cls;
3908
3909   shutdown_ctx->outstanding--;
3910   if (emsg == NULL)
3911     {
3912       shutdown_ctx->peers_down++;
3913     }
3914   else
3915     {
3916       shutdown_ctx->peers_failed++;
3917     }
3918
3919   if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
3920     {
3921       if (shutdown_ctx->peers_failed > 0)
3922         shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
3923       else
3924         shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
3925       GNUNET_free(shutdown_ctx);
3926     }
3927 }
3928
3929 /**
3930  * Individual shutdown context for a particular peer.
3931  */
3932 struct PeerShutdownContext
3933 {
3934   /**
3935    * Pointer to the high level shutdown context.
3936    */
3937   struct ShutdownContext *shutdown_ctx;
3938
3939   /**
3940    * The daemon handle for the peer to shut down.
3941    */
3942   struct GNUNET_TESTING_Daemon *daemon;
3943 };
3944
3945 /**
3946  * Task to rate limit the number of outstanding peer shutdown
3947  * requests.  This is necessary for making sure we don't do
3948  * too many ssh connections at once, but is generally nicer
3949  * to any system as well (graduated task starts, as opposed
3950  * to calling gnunet-arm N times all at once).
3951  */
3952 static void
3953 schedule_shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3954 {
3955   struct PeerShutdownContext *peer_shutdown_ctx = cls;
3956   struct ShutdownContext *shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
3957
3958   GNUNET_assert(peer_shutdown_ctx != NULL);
3959   GNUNET_assert(shutdown_ctx != NULL);
3960
3961   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
3962     GNUNET_SCHEDULER_add_delayed(peer_shutdown_ctx->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_shutdown_task, peer_shutdown_ctx);
3963   else
3964     {
3965       shutdown_ctx->outstanding++;
3966       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, shutdown_ctx->timeout, &internal_shutdown_callback, shutdown_ctx, GNUNET_YES, GNUNET_NO);
3967       GNUNET_free(peer_shutdown_ctx);
3968     }
3969 }
3970 /**
3971  * Shutdown all peers started in the given group.
3972  *
3973  * @param pg handle to the peer group
3974  * @param timeout how long to wait for shutdown
3975  * @param cb callback to notify upon success or failure
3976  * @param cb_cls closure for cb
3977  */
3978 void
3979 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
3980                              struct GNUNET_TIME_Relative timeout,
3981                              GNUNET_TESTING_NotifyCompletion cb,
3982                              void *cb_cls)
3983 {
3984   unsigned int off;
3985   struct ShutdownContext *shutdown_ctx;
3986   struct PeerShutdownContext *peer_shutdown_ctx;
3987
3988   GNUNET_assert(pg->total > 0);
3989
3990   shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
3991   shutdown_ctx->cb = cb;
3992   shutdown_ctx->cb_cls = cb_cls;
3993   shutdown_ctx->total_peers = pg->total;
3994   shutdown_ctx->timeout = timeout;
3995   /* shtudown_ctx->outstanding = 0; */
3996
3997   for (off = 0; off < pg->total; off++)
3998     {
3999       GNUNET_assert(NULL != pg->peers[off].daemon);
4000       peer_shutdown_ctx = GNUNET_malloc(sizeof(struct PeerShutdownContext));
4001       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4002       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4003       GNUNET_SCHEDULER_add_now(pg->peers[off].daemon->sched, &schedule_shutdown_task, peer_shutdown_ctx);
4004       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4005       if (NULL != pg->peers[off].cfg)
4006         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4007       if (pg->peers[off].allowed_peers != NULL)
4008         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
4009       if (pg->peers[off].connect_peers != NULL)
4010         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
4011       if (pg->peers[off].blacklisted_peers != NULL)
4012         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
4013     }
4014   GNUNET_free (pg->peers);
4015   if (NULL != pg->hosts)
4016     {
4017       GNUNET_free (pg->hosts[0].hostname);
4018       GNUNET_free (pg->hosts);
4019     }
4020   GNUNET_free (pg);
4021 }
4022
4023
4024 /* end of testing_group.c */