src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define DEBUG_CHURN GNUNET_NO
  36
  37 /**
  38  * Lowest port used for GNUnet testing.  Should be high enough to not
  39  * conflict with other applications running on the hosts but be low
  40  * enough to not conflict with client-ports (typically starting around
  41  * 32k).
  42  */
  43 #define LOW_PORT 10000
  44
  45 /**
  46  * Highest port used for GNUnet testing.  Should be low enough to not
  47  * conflict with the port range for "local" ports (client apps; see
  48  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  49  */
  50 #define HIGH_PORT 56000
  51
  52 #define MAX_OUTSTANDING_CONNECTIONS 10
  53
  54 #define MAX_CONCURRENT_HOSTKEYS 10
  55
  56 #define MAX_CONCURRENT_STARTING 10
  57
  58 #define MAX_CONCURRENT_SHUTDOWN 10
  59
  60 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
  61
  62 #define CONNECT_ATTEMPTS 8
  63
  64 /**
  65  * Prototype of a function called whenever two peers would be connected
  66  * in a certain topology.
  67  */
  68 typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg,
  69                                                   unsigned int first,
  70                                                   unsigned int second);
  71
  72
  73 /**
  74  * Context for handling churning a peer group
  75  */
  76 struct ChurnContext
  77 {
  78   /**
  79    * Callback used to notify of churning finished
  80    */
  81   GNUNET_TESTING_NotifyCompletion cb;
  82
  83   /**
  84    * Closure for callback
  85    */
  86   void *cb_cls;
  87
  88   /**
  89    * Number of peers that still need to be started
  90    */
  91   unsigned int num_to_start;
  92
  93   /**
  94    * Number of peers that still need to be stopped
  95    */
  96   unsigned int num_to_stop;
  97
  98   /**
  99    * Number of peers that failed to start
 100    */
 101   unsigned int num_failed_start;
 102
 103   /**
 104    * Number of peers that failed to stop
 105    */
 106   unsigned int num_failed_stop;
 107 };
 108
 109 struct RestartContext
 110 {
 111   /**
 112    * The group of peers being restarted
 113    */
 114   struct GNUNET_TESTING_PeerGroup *peer_group;
 115
 116   /**
 117    * How many peers have been restarted thus far
 118    */
 119   unsigned int peers_restarted;
 120
 121   /**
 122    * How many peers got an error when restarting
 123    */
 124   unsigned int peers_restart_failed;
 125
 126   /**
 127    * The function to call once all peers have been restarted
 128    */
 129   GNUNET_TESTING_NotifyCompletion callback;
 130
 131   /**
 132    * Closure for callback function
 133    */
 134   void *callback_cls;
 135
 136 };
 137
 138
 139 struct ShutdownContext
 140 {
 141   /**
 142    * Total peers to wait for
 143    */
 144   unsigned int total_peers;
 145
 146   /**
 147    * Number of peers successfully shut down
 148    */
 149   unsigned int peers_down;
 150
 151   /**
 152    * Number of peers failed to shut down
 153    */
 154   unsigned int peers_failed;
 155
 156   /**
 157    * Number of peers we have started shutting
 158    * down.  If too many, wait on them.
 159    */
 160   unsigned int outstanding;
 161
 162   /**
 163    * Timeout for shutdown.
 164    */
 165   struct GNUNET_TIME_Relative timeout;
 166
 167   /**
 168    * Callback to call when all peers either
 169    * shutdown or failed to shutdown
 170    */
 171   GNUNET_TESTING_NotifyCompletion cb;
 172
 173   /**
 174    * Closure for cb
 175    */
 176   void *cb_cls;
 177 };
 178
 179 struct CreateTopologyContext
 180 {
 181
 182   /**
 183    * Function to call with number of connections
 184    */
 185   GNUNET_TESTING_NotifyConnections cont;
 186
 187   /**
 188    * Closure for connection notification
 189    */
 190   void *cls;
 191 };
 192
 193 #if OLD
 194 struct PeerConnection
 195 {
 196   /*
 197    * Linked list
 198    */
 199   struct PeerConnection *next;
 200
 201   /*
 202    * Pointer to daemon handle
 203    */
 204   struct GNUNET_TESTING_Daemon *daemon;
 205
 206 };
 207 #endif
 208
 209 struct InternalStartContext
 210 {
 211   /**
 212    * Pointer to peerdata
 213    */
 214   struct PeerData *peer;
 215
 216   /**
 217    * Timeout for peer startup
 218    */
 219   struct GNUNET_TIME_Relative timeout;
 220
 221   /**
 222    * Client callback for hostkey notification
 223    */
 224   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 225
 226   /**
 227    * Closure for hostkey_callback
 228    */
 229   void *hostkey_cls;
 230
 231   /**
 232    * Client callback for peer start notification
 233    */
 234   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 235
 236   /**
 237    * Closure for cb
 238    */
 239   void *start_cb_cls;
 240
 241   /**
 242    * Hostname, where to start the peer
 243    */
 244   const char *hostname;
 245 };
 246
 247 /**
 248  * Data we keep per peer.
 249  */
 250 struct PeerData
 251 {
 252   /**
 253    * (Initial) configuration of the host.
 254    * (initial because clients could change
 255    *  it and we would not know about those
 256    *  updates).
 257    */
 258   struct GNUNET_CONFIGURATION_Handle *cfg;
 259
 260   /**
 261    * Handle for controlling the daemon.
 262    */
 263   struct GNUNET_TESTING_Daemon *daemon;
 264
 265   /**
 266    * The peergroup this peer belongs to.
 267    */
 268   struct GNUNET_TESTING_PeerGroup *pg;
 269
 270   /**
 271    * Hash map of allowed peer connections (F2F created topology)
 272    */
 273   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 274
 275   /**
 276    * Hash map of blacklisted peers
 277    */
 278   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 279
 280   /**
 281    * Hash map of peer connections
 282    */
 283   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 284
 285   /**
 286    * Temporary hash map of peer connections
 287    */
 288   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 289
 290   /**
 291    * Temporary variable for topology creation, should be reset before
 292    * creating any topology so the count is valid once finished.
 293    */
 294   int num_connections;
 295
 296   /**
 297    * Context to keep track of peers being started, to
 298    * stagger hostkey generation and peer startup.
 299    */
 300   struct InternalStartContext internal_context;
 301 };
 302
 303
 304 /**
 305  * Data we keep per host.
 306  */
 307 struct HostData
 308 {
 309   /**
 310    * Name of the host.
 311    */
 312   char *hostname;
 313
 314   /**
 315    * Lowest port that we have not yet used
 316    * for GNUnet.
 317    */
 318   uint16_t minport;
 319 };
 320
 321 struct TopologyIterateContext
 322 {
 323   /**
 324    * Callback for notifying of two connected peers.
 325    */
 326   GNUNET_TESTING_NotifyTopology topology_cb;
 327
 328   /**
 329    * Closure for topology_cb
 330    */
 331   void *cls;
 332
 333   /**
 334    * Number of peers currently connected to.
 335    */
 336   unsigned int connected;
 337
 338   /**
 339    * Number of peers we have finished iterating.
 340    */
 341   unsigned int completed;
 342
 343   /**
 344    * Number of peers total.
 345    */
 346   unsigned int total;
 347 };
 348
 349 struct StatsIterateContext
 350 {
 351   /**
 352    * Continuation to call once all stats information has been retrieved.
 353    */
 354   GNUNET_STATISTICS_Callback cont;
 355
 356   /**
 357    * Proc function to call on each value received.
 358    */
 359   GNUNET_TESTING_STATISTICS_Iterator proc;
 360
 361   /**
 362    * Closure for topology_cb
 363    */
 364   void *cls;
 365
 366   /**
 367    * Number of peers currently connected to.
 368    */
 369   unsigned int connected;
 370
 371   /**
 372    * Number of peers we have finished iterating.
 373    */
 374   unsigned int completed;
 375
 376   /**
 377    * Number of peers total.
 378    */
 379   unsigned int total;
 380 };
 381
 382 struct CoreContext
 383 {
 384   void *iter_context;
 385   struct GNUNET_TESTING_Daemon *daemon;
 386 };
 387
 388 struct StatsCoreContext
 389 {
 390   void *iter_context;
 391   struct GNUNET_TESTING_Daemon *daemon;
 392   /**
 393    * Handle to the statistics service.
 394    */
 395   struct GNUNET_STATISTICS_Handle *stats_handle;
 396
 397   /**
 398    * Handle for getting statistics.
 399    */
 400   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 401 };
 402
 403 /**
 404  * Handle to a group of GNUnet peers.
 405  */
 406 struct GNUNET_TESTING_PeerGroup
 407 {
 408   /**
 409    * Our scheduler.
 410    */
 411   struct GNUNET_SCHEDULER_Handle *sched;
 412
 413   /**
 414    * Configuration template.
 415    */
 416   const struct GNUNET_CONFIGURATION_Handle *cfg;
 417
 418   /**
 419    * Function to call on each started daemon.
 420    */
 421   //GNUNET_TESTING_NotifyDaemonRunning cb;
 422
 423   /**
 424    * Closure for cb.
 425    */
 426   //void *cb_cls;
 427
 428   /*
 429    * Function to call on each topology connection created
 430    */
 431   GNUNET_TESTING_NotifyConnection notify_connection;
 432
 433   /*
 434    * Callback for notify_connection
 435    */
 436   void *notify_connection_cls;
 437
 438   /**
 439    * NULL-terminated array of information about
 440    * hosts.
 441    */
 442   struct HostData *hosts;
 443
 444   /**
 445    * Array of "total" peers.
 446    */
 447   struct PeerData *peers;
 448
 449   /**
 450    * Number of peers in this group.
 451    */
 452   unsigned int total;
 453
 454   /**
 455    * At what time should we fail the peer startup process?
 456    */
 457   struct GNUNET_TIME_Absolute max_timeout;
 458
 459   /**
 460    * How many peers are being started right now?
 461    */
 462   unsigned int starting;
 463
 464   /**
 465    * How many peers have already been started?
 466    */
 467   unsigned int started;
 468 };
 469
 470 struct UpdateContext
 471 {
 472   struct GNUNET_CONFIGURATION_Handle *ret;
 473   const char *hostname;
 474   unsigned int nport;
 475   unsigned int upnum;
 476 };
 477
 478
 479 struct ConnectContext
 480 {
 481   struct GNUNET_TESTING_Daemon *first;
 482
 483   struct GNUNET_TESTING_Daemon *second;
 484
 485   struct GNUNET_TESTING_PeerGroup *pg;
 486 };
 487
 488 /**
 489  * Convert unique ID to hash code.
 490  *
 491  * @param uid unique ID to convert
 492  * @param hash set to uid (extended with zeros)
 493  */
 494 static void
 495 hash_from_uid (uint32_t uid,
 496                GNUNET_HashCode *hash)
 497 {
 498   memset (hash, 0, sizeof(GNUNET_HashCode));
 499   *((uint32_t*)hash) = uid;
 500 }
 501
 502 /**
 503  * Convert hash code to unique ID.
 504  *
 505  * @param uid unique ID to convert
 506  * @param hash set to uid (extended with zeros)
 507  */
 508 static void
 509 uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
 510 {
 511   memcpy (uid, hash, sizeof(uint32_t));
 512 }
 513
 514 /**
 515  * Number of connects we are waiting on, allows us to rate limit
 516  * connect attempts.
 517  */
 518 static int outstanding_connects;
 519
 520 /**
 521  * Get a topology from a string input.
 522  *
 523  * @param topology where to write the retrieved topology
 524  * @param topology_string The string to attempt to
 525  *        get a configuration value from
 526  * @return GNUNET_YES if topology string matched a
 527  *         known topology, GNUNET_NO if not
 528  */
 529 int
 530 GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
 531 {
 532   /**
 533    * Strings representing topologies in enum
 534    */
 535   static const char * topology_strings[] =
 536     {
 537       /**
 538        * A clique (everyone connected to everyone else).
 539        */
 540       "CLIQUE",
 541
 542       /**
 543        * Small-world network (2d torus plus random links).
 544        */
 545       "SMALL_WORLD",
 546
 547       /**
 548        * Small-world network (ring plus random links).
 549        */
 550       "SMALL_WORLD_RING",
 551
 552       /**
 553        * Ring topology.
 554        */
 555       "RING",
 556
 557       /**
 558        * 2-d torus.
 559        */
 560       "2D_TORUS",
 561
 562       /**
 563        * Random graph.
 564        */
 565       "ERDOS_RENYI",
 566
 567       /**
 568        * Certain percentage of peers are unable to communicate directly
 569        * replicating NAT conditions
 570        */
 571       "INTERNAT",
 572
 573       /**
 574        * Scale free topology.
 575        */
 576       "SCALE_FREE",
 577
 578       /**
 579        * Straight line topology.
 580        */
 581       "LINE",
 582
 583       /**
 584        * All peers are disconnected.
 585        */
 586       "NONE",
 587
 588       NULL
 589     };
 590
 591   int curr = 0;
 592   if (topology_string == NULL)
 593     return GNUNET_NO;
 594   while (topology_strings[curr] != NULL)
 595     {
 596       if (strcasecmp(topology_strings[curr], topology_string) == 0)
 597         {
 598           *topology = curr;
 599           return GNUNET_YES;
 600         }
 601       curr++;
 602     }
 603   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 604   return GNUNET_NO;
 605 }
 606
 607
 608 /**
 609  * Get connect topology option from string input.
 610  *
 611  * @param topology_option where to write the retrieved topology
 612  * @param topology_string The string to attempt to
 613  *        get a configuration value from
 614  * @return GNUNET_YES if string matched a known
 615  *         topology option, GNUNET_NO if not
 616  */
 617 int
 618 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
 619                                     char * topology_string)
 620 {
 621   /**
 622    * Options for connecting a topology as strings.
 623    */
 624   static const char * topology_option_strings[] =
 625     {
 626       /**
 627        * Try to connect all peers specified in the topology.
 628        */
 629       "CONNECT_ALL",
 630
 631       /**
 632        * Choose a random subset of connections to create.
 633        */
 634       "CONNECT_RANDOM_SUBSET",
 635
 636       /**
 637        * Create at least X connections for each peer.
 638        */
 639       "CONNECT_MINIMUM",
 640
 641       /**
 642        * Using a depth first search, create one connection
 643        * per peer.  If any are missed (graph disconnected)
 644        * start over at those peers until all have at least one
 645        * connection.
 646        */
 647       "CONNECT_DFS",
 648
 649       /**
 650        * No options specified.
 651        */
 652       "CONNECT_NONE",
 653
 654       NULL
 655     };
 656   int curr = 0;
 657
 658   if (topology_string == NULL)
 659     return GNUNET_NO;
 660   while (NULL != topology_option_strings[curr])
 661     {
 662       if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
 663         {
 664           *topology_option = curr;
 665           return GNUNET_YES;
 666         }
 667       curr++;
 668     }
 669   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 670   return GNUNET_NO;
 671 }
 672
 673 /**
 674  * Function to iterate over options.  Copies
 675  * the options to the target configuration,
 676  * updating PORT values as needed.
 677  *
 678  * @param cls closure
 679  * @param section name of the section
 680  * @param option name of the option
 681  * @param value value of the option
 682  */
 683 static void
 684 update_config (void *cls,
 685                const char *section, const char *option, const char *value)
 686 {
 687   struct UpdateContext *ctx = cls;
 688   unsigned int ival;
 689   char cval[12];
 690   char uval[128];
 691
 692   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 693     {
 694       if (ival != 0)
 695         {
 696           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 697           value = cval;
 698         }
 699     }
 700
 701   if (0 == strcmp (option, "UNIXPATH"))
 702     {
 703       GNUNET_snprintf (uval,
 704                        sizeof (uval),
 705                        "/tmp/test-service-%s-%u",
 706                        section,
 707                        ctx->upnum++);
 708       value = uval;
 709     }
 710
 711   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 712     {
 713       value = ctx->hostname;
 714     }
 715
 716   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 717 }
 718
 719
 720 /**
 721  * Create a new configuration using the given configuration
 722  * as a template; however, each PORT in the existing cfg
 723  * must be renumbered by incrementing "*port".  If we run
 724  * out of "*port" numbers, return NULL.
 725  *
 726  * @param cfg template configuration
 727  * @param port port numbers to use, update to reflect
 728  *             port numbers that were used
 729  * @param upnum number to make unix domain socket names unique
 730  * @param hostname hostname of the controlling host, to allow control connections from
 731  *
 732  * @return new configuration, NULL on error
 733  */
 734 static struct GNUNET_CONFIGURATION_Handle *
 735 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 736              uint16_t * port,
 737              uint32_t * upnum,
 738              const char *hostname)
 739 {
 740   struct UpdateContext uc;
 741   uint16_t orig;
 742   char *control_host;
 743   char *allowed_hosts;
 744
 745   orig = *port;
 746   uc.nport = *port;
 747   uc.upnum = *upnum;
 748   uc.ret = GNUNET_CONFIGURATION_create ();
 749   uc.hostname = hostname;
 750
 751   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 752   if (uc.nport >= HIGH_PORT)
 753     {
 754       *port = orig;
 755       GNUNET_CONFIGURATION_destroy (uc.ret);
 756       return NULL;
 757     }
 758
 759   if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 760     {
 761       if (hostname != NULL)
 762         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1; %s;", control_host, hostname);
 763       else
 764         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
 765
 766       GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
 767       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport", "ACCEPT_FROM", allowed_hosts);
 768       GNUNET_CONFIGURATION_set_value_string(uc.ret, "dht", "ACCEPT_FROM", allowed_hosts);
 769       GNUNET_CONFIGURATION_set_value_string(uc.ret, "statistics", "ACCEPT_FROM", allowed_hosts);
 770       GNUNET_free_non_null(control_host);
 771       GNUNET_free(allowed_hosts);
 772     }
 773
 774
 775   /* arm needs to know to allow connections from the host on which it is running,
 776    * otherwise gnunet-arm is unable to connect to it in some instances */
 777   if (hostname != NULL)
 778     {
 779       GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
 780       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", hostname);
 781       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", hostname);
 782       GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
 783       GNUNET_free(allowed_hosts);
 784     }
 785   else
 786     {
 787       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", "127.0.0.1");
 788       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", "127.0.0.1");
 789     }
 790
 791   *port = (uint16_t) uc.nport;
 792   *upnum = uc.upnum;
 793   return uc.ret;
 794 }
 795
 796
 797 /*
 798  * Add entries to the peers connect list
 799  *
 800  * @param pg the peer group we are working with
 801  * @param first index of the first peer
 802  * @param second index of the second peer
 803  *
 804  * @return the number of connections added
 805  *         technically should only be 0 or 2
 806  *
 807  */
 808 static int
 809 add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 810 {
 811   int added;
 812   int add_first;
 813   int add_second;
 814
 815   GNUNET_HashCode hash_first;
 816   GNUNET_HashCode hash_second;
 817
 818   hash_from_uid(first, &hash_first);
 819   hash_from_uid(second, &hash_second);
 820
 821   add_first = GNUNET_NO;
 822   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
 823     {
 824       add_first = GNUNET_YES;
 825     }
 826
 827   add_second = GNUNET_NO;
 828   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
 829     {
 830       add_second = GNUNET_YES;
 831     }
 832
 833   added = 0;
 834   if (add_first)
 835     {
 836       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 837       pg->peers[first].num_connections++;
 838       added++;
 839     }
 840
 841   if (add_second)
 842     {
 843       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 844       pg->peers[second].num_connections++;
 845       added++;
 846     }
 847
 848   return added;
 849 }
 850
 851
 852 /*
 853  * Add entries to the peers allowed connections list
 854  *
 855  * @param pg the peer group we are working with
 856  * @param first index of the first peer
 857  * @param second index of the second peer
 858  *
 859  * @return the number of connections added (can be 0, 1 or 2)
 860  *         technically should only be 0 or 2, but the small price
 861  *         of iterating over the lists (hashmaps in the future)
 862  *         for being sure doesn't bother me!
 863  *
 864  */
 865 static int
 866 add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 867 {
 868   int added;
 869 #if OLD
 870   struct PeerConnection *first_iter;
 871   struct PeerConnection *second_iter;
 872   struct PeerConnection *new_first;
 873   struct PeerConnection *new_second;
 874 #endif
 875   int add_first;
 876   int add_second;
 877
 878   GNUNET_HashCode hash_first;
 879   GNUNET_HashCode hash_second;
 880
 881   hash_from_uid(first, &hash_first);
 882   hash_from_uid(second, &hash_second);
 883
 884   add_first = GNUNET_NO;
 885   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
 886     {
 887       add_first = GNUNET_YES;
 888     }
 889
 890   add_second = GNUNET_NO;
 891   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
 892     {
 893       add_second = GNUNET_YES;
 894     }
 895 #if OLD
 896   first_iter = pg->peers[first].connected_peers;
 897   while (first_iter != NULL)
 898     {
 899       if (first_iter->daemon == pg->peers[second].daemon)
 900         add_first = GNUNET_NO;
 901       first_iter = first_iter->next;
 902     }
 903
 904   second_iter = pg->peers[second].connected_peers;
 905   add_second = GNUNET_YES;
 906   while (second_iter != NULL)
 907     {
 908       if (second_iter->daemon == pg->peers[first].daemon)
 909         add_second = GNUNET_NO;
 910       second_iter = second_iter->next;
 911     }
 912 #endif
 913
 914   added = 0;
 915   if (add_first)
 916     {
 917       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 918 #if OLD
 919       new_first = GNUNET_malloc(sizeof(struct PeerConnection));
 920       new_first->daemon = pg->peers[second].daemon;
 921       new_first->next = pg->peers[first].connected_peers;
 922       pg->peers[first].connected_peers = new_first;
 923 #endif
 924       pg->peers[first].num_connections++;
 925       added++;
 926     }
 927
 928   if (add_second)
 929     {
 930       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 931 #if OLD
 932       new_second = GNUNET_malloc(sizeof(struct PeerConnection));
 933       new_second->daemon = pg->peers[first].daemon;
 934       new_second->next = pg->peers[second].connected_peers;
 935       pg->peers[second].connected_peers = new_second;
 936       pg->peers[first].num_connections++;
 937 #endif
 938       pg->peers[second].num_connections++;
 939       added++;
 940     }
 941
 942   return added;
 943 }
 944
 945 /*
 946  * Add entries to the peers blacklisted list
 947  *
 948  * @param pg the peer group we are working with
 949  * @param first index of the first peer
 950  * @param second index of the second peer
 951  *
 952  * @return the number of connections added (can be 0, 1 or 2)
 953  *
 954  */
 955 static int
 956 blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 957 {
 958   int added;
 959   int add_first;
 960   int add_second;
 961   GNUNET_HashCode hash_first;
 962   GNUNET_HashCode hash_second;
 963
 964   hash_from_uid(first, &hash_first);
 965   hash_from_uid(second, &hash_second);
 966
 967   add_first = GNUNET_NO;
 968   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
 969     {
 970       add_first = GNUNET_YES;
 971     }
 972
 973   add_second = GNUNET_NO;
 974   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
 975     {
 976       add_second = GNUNET_YES;
 977     }
 978
 979   added = 0;
 980   if (add_first)
 981     {
 982       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 983       pg->peers[first].num_connections++;
 984       added++;
 985     }
 986
 987   if (add_second)
 988     {
 989       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 990       pg->peers[second].num_connections++;
 991       added++;
 992     }
 993
 994   return added;
 995 }
 996
 997 /*
 998  * Remove entries from the peers blacklisted list
 999  *
1000  * @param pg the peer group we are working with
1001  * @param first index of the first peer
1002  * @param second index of the second peer
1003  *
1004  * @return the number of connections removed (can be 0, 1 or 2)
1005  *
1006  */
1007 static int
1008 unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1009 {
1010   int removed;
1011   int remove_first;
1012   int remove_second;
1013   GNUNET_HashCode hash_first;
1014   GNUNET_HashCode hash_second;
1015
1016   hash_from_uid(first, &hash_first);
1017   hash_from_uid(second, &hash_second);
1018
1019   remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
1020   remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
1021
1022   removed = 0;
1023   if (remove_first)
1024     {
1025       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
1026       removed++;
1027     }
1028
1029   if (remove_second)
1030     {
1031       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
1032       removed++;
1033     }
1034
1035   return removed;
1036 }
1037
1038 /**
1039  * Scale free network construction as described in:
1040  *
1041  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1042  *
1043  * Start with a network of "one" peer, then progressively add
1044  * peers up to the total number.  At each step, iterate over
1045  * all possible peers and connect new peer based on number of
1046  * existing connections of the target peer.
1047  *
1048  * @param pg the peer group we are dealing with
1049  * @param proc the connection processor to use
1050  *
1051  * @return the number of connections created
1052  */
1053 static int
1054 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1055 {
1056
1057   unsigned int total_connections;
1058   unsigned int outer_count;
1059   unsigned int i;
1060   unsigned int previous_total_connections;
1061   double random;
1062   double probability;
1063
1064   GNUNET_assert(pg->total > 1);
1065
1066   /* Add a connection between the first two nodes */
1067   total_connections = proc(pg, 0, 1);
1068
1069   for (outer_count = 1; outer_count < pg->total; outer_count++)
1070     {
1071       previous_total_connections = total_connections;
1072       for (i = 0; i < outer_count; i++)
1073         {
1074           probability = pg->peers[i].num_connections / (double)previous_total_connections;
1075           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1076                                                       UINT64_MAX)) / ( (double) UINT64_MAX);
1077 #if VERBOSE_TESTING
1078           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1079                       "Considering connecting peer %d to peer %d\n",
1080                       outer_count, i);
1081 #endif
1082           if (random < probability)
1083             {
1084 #if VERBOSE_TESTING
1085               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1086                           "Connecting peer %d to peer %d\n",
1087                           outer_count, i);
1088 #endif
1089               total_connections += proc(pg, outer_count, i);
1090             }
1091         }
1092     }
1093
1094   return total_connections;
1095 }
1096
1097 /**
1098  * Create a topology given a peer group (set of running peers)
1099  * and a connection processor.
1100  *
1101  * @param pg the peergroup to create the topology on
1102  * @param proc the connection processor to call to actually set
1103  *        up connections between two peers
1104  *
1105  * @return the number of connections that were set up
1106  *
1107  */
1108 int
1109 create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1110 {
1111   unsigned int i, j;
1112   int nodeToConnect;
1113   unsigned int natLog;
1114   unsigned int randomPeer;
1115   double random, logNModifier, percentage;
1116   unsigned int smallWorldConnections;
1117   int connsPerPeer;
1118   char *p_string;
1119   int max;
1120   int min;
1121   unsigned int useAnd;
1122   int connect_attempts;
1123
1124   logNModifier = 0.5; /* FIXME: default value? */
1125   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1126                                                          "TESTING",
1127                                                          "LOGNMODIFIER",
1128                                                          &p_string))
1129     {
1130       if (sscanf(p_string, "%lf", &logNModifier) != 1)
1131         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1132                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1133                     p_string,
1134                     "LOGNMODIFIER",
1135                     "TESTING");
1136       GNUNET_free (p_string);
1137     }
1138   percentage = 0.5; /* FIXME: default percentage? */
1139   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1140                                                          "TESTING",
1141                                                          "PERCENTAGE",
1142                                                          &p_string))
1143     {
1144       if (sscanf(p_string, "%lf", &percentage) != 1)
1145         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1146                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1147                     p_string,
1148                     "PERCENTAGE",
1149                     "TESTING");
1150       GNUNET_free (p_string);
1151     }
1152   natLog = log (pg->total);
1153   connsPerPeer = ceil (natLog * logNModifier);
1154
1155   if (connsPerPeer % 2 == 1)
1156     connsPerPeer += 1;
1157
1158   smallWorldConnections = 0;
1159   connect_attempts = 0;
1160   for (i = 0; i < pg->total; i++)
1161     {
1162       useAnd = 0;
1163       max = i + connsPerPeer / 2;
1164       min = i - connsPerPeer / 2;
1165
1166       if (max > pg->total - 1)
1167         {
1168           max = max - pg->total;
1169           useAnd = 1;
1170         }
1171
1172       if (min < 0)
1173         {
1174           min = pg->total - 1 + min;
1175           useAnd = 1;
1176         }
1177
1178       for (j = 0; j < connsPerPeer / 2; j++)
1179         {
1180           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1181                                                       UINT64_MAX) / ( (double) UINT64_MAX));
1182           if (random < percentage)
1183             {
1184               /* Connect to uniformly selected random peer */
1185               randomPeer =
1186                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1187                                    pg->total);
1188               while ((((randomPeer < max) && (randomPeer > min))
1189                       && (useAnd == 0)) || (((randomPeer > min)
1190                                              || (randomPeer < max))
1191                                             && (useAnd == 1)))
1192                 {
1193                   randomPeer =
1194                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1195                                                          pg->total);
1196                 }
1197               smallWorldConnections +=
1198                 proc (pg, i, randomPeer);
1199             }
1200           else
1201             {
1202               nodeToConnect = i + j + 1;
1203               if (nodeToConnect > pg->total - 1)
1204                 {
1205                   nodeToConnect = nodeToConnect - pg->total;
1206                 }
1207               connect_attempts +=
1208                 proc (pg, i, nodeToConnect);
1209             }
1210         }
1211
1212     }
1213
1214   connect_attempts += smallWorldConnections;
1215
1216   return connect_attempts;
1217 }
1218
1219 /**
1220  * Create a topology given a peer group (set of running peers)
1221  * and a connection processor.
1222  *
1223  * @param pg the peergroup to create the topology on
1224  * @param proc the connection processor to call to actually set
1225  *        up connections between two peers
1226  *
1227  * @return the number of connections that were set up
1228  *
1229  */
1230 static int
1231 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1232 {
1233   unsigned int outer_count, inner_count;
1234   unsigned int cutoff;
1235   int connect_attempts;
1236   double nat_percentage;
1237   char *p_string;
1238
1239   nat_percentage = 0.6; /* FIXME: default percentage? */
1240   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1241                                                          "TESTING",
1242                                                          "NATPERCENTAGE",
1243                                                          &p_string))
1244     {
1245       if (sscanf(p_string, "%lf", &nat_percentage) != 1)
1246         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1247                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1248                     p_string,
1249                     "NATPERCENTAGE",
1250                     "TESTING");
1251       GNUNET_free (p_string);
1252     }
1253
1254
1255
1256   cutoff = (unsigned int) (nat_percentage * pg->total);
1257
1258   connect_attempts = 0;
1259
1260   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1261     {
1262       for (inner_count = outer_count + 1; inner_count < pg->total;
1263            inner_count++)
1264         {
1265           if ((outer_count > cutoff) || (inner_count > cutoff))
1266             {
1267 #if VERBOSE_TESTING
1268               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1269                           "Connecting peer %d to peer %d\n",
1270                           outer_count, inner_count);
1271 #endif
1272               connect_attempts += proc(pg, outer_count, inner_count);
1273             }
1274         }
1275     }
1276
1277   return connect_attempts;
1278
1279 }
1280
1281 /**
1282  * Create a topology given a peer group (set of running peers)
1283  * and a connection processor.
1284  *
1285  * @param pg the peergroup to create the topology on
1286  * @param proc the connection processor to call to actually set
1287  *        up connections between two peers
1288  *
1289  * @return the number of connections that were set up
1290  *
1291  */
1292 static int
1293 create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1294 {
1295   unsigned int i, j, k;
1296   unsigned int square;
1297   unsigned int rows;
1298   unsigned int cols;
1299   unsigned int toggle = 1;
1300   unsigned int nodeToConnect;
1301   unsigned int natLog;
1302   unsigned int node1Row;
1303   unsigned int node1Col;
1304   unsigned int node2Row;
1305   unsigned int node2Col;
1306   unsigned int distance;
1307   double probability, random, percentage;
1308   unsigned int smallWorldConnections;
1309   char *p_string;
1310   int connect_attempts;
1311   square = floor (sqrt (pg->total));
1312   rows = square;
1313   cols = square;
1314
1315   percentage = 0.5; /* FIXME: default percentage? */
1316   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1317                                                          "TESTING",
1318                                                          "PERCENTAGE",
1319                                                          &p_string))
1320     {
1321       if (sscanf(p_string, "%lf", &percentage) != 1)
1322         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1323                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1324                     p_string,
1325                     "PERCENTAGE",
1326                     "TESTING");
1327       GNUNET_free (p_string);
1328     }
1329   probability = 0.5; /* FIXME: default percentage? */
1330   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1331                                                          "TESTING",
1332                                                          "PROBABILITY",
1333                                                          &p_string))
1334     {
1335       if (sscanf(p_string, "%lf", &probability) != 1)
1336         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1337                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1338                     p_string,
1339                     "PROBABILITY",
1340                     "TESTING");
1341       GNUNET_free (p_string);
1342     }
1343   if (square * square != pg->total)
1344     {
1345       while (rows * cols < pg->total)
1346         {
1347           if (toggle % 2 == 0)
1348             rows++;
1349           else
1350             cols++;
1351
1352           toggle++;
1353         }
1354     }
1355 #if VERBOSE_TESTING
1356       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1357                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1358                   rows, cols);
1359 #endif
1360
1361   connect_attempts = 0;
1362   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1363    * to the node to its right and above.  Once this is over, we'll have our torus!
1364    * Special case for the last node (if the rows and columns are not equal), connect
1365    * to the first in the row to maintain topology.
1366    */
1367   for (i = 0; i < pg->total; i++)
1368     {
1369       /* First connect to the node to the right */
1370       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1371         nodeToConnect = i + 1;
1372       else if (i + 1 == pg->total)
1373         nodeToConnect = rows * cols - cols;
1374       else
1375         nodeToConnect = i - cols + 1;
1376
1377       connect_attempts += proc (pg, i, nodeToConnect);
1378
1379       if (i < cols)
1380         nodeToConnect = (rows * cols) - cols + i;
1381       else
1382         nodeToConnect = i - cols;
1383
1384       if (nodeToConnect < pg->total)
1385         connect_attempts += proc (pg, i, nodeToConnect);
1386     }
1387   natLog = log (pg->total);
1388 #if VERBOSE_TESTING > 2
1389   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1390               _("natural log of %d is %d, will run %d iterations\n"),
1391              pg->total, natLog, (int) (natLog * percentage));
1392   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
1393 #endif
1394   smallWorldConnections = 0;
1395   for (i = 0; i < (int) (natLog * percentage); i++)
1396     {
1397       for (j = 0; j < pg->total; j++)
1398         {
1399           /* Determine the row and column of node at position j on the 2d torus */
1400           node1Row = j / cols;
1401           node1Col = j - (node1Row * cols);
1402           for (k = 0; k < pg->total; k++)
1403             {
1404               /* Determine the row and column of node at position k on the 2d torus */
1405               node2Row = k / cols;
1406               node2Col = k - (node2Row * cols);
1407               /* Simple Cartesian distance */
1408               distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
1409               if (distance > 1)
1410                 {
1411                   /* Calculate probability as 1 over the square of the distance */
1412                   probability = 1.0 / (distance * distance);
1413                   /* Choose a random value between 0 and 1 */
1414                   random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1415                                                               UINT64_MAX)) / ( (double) UINT64_MAX);
1416                   /* If random < probability, then connect the two nodes */
1417                   if (random < probability)
1418                     smallWorldConnections += proc (pg, j, k);
1419
1420                 }
1421             }
1422         }
1423     }
1424   connect_attempts += smallWorldConnections;
1425 #if VERBOSE_TESTING > 2
1426           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1427                       _("Total connections added for small world: %d!\n"),
1428                       smallWorldConnections);
1429 #endif
1430   return connect_attempts;
1431 }
1432
1433 /**
1434  * Create a topology given a peer group (set of running peers)
1435  * and a connection processor.
1436  *
1437  * @param pg the peergroup to create the topology on
1438  * @param proc the connection processor to call to actually set
1439  *        up connections between two peers
1440  *
1441  * @return the number of connections that were set up
1442  *
1443  */
1444 static int
1445 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1446 {
1447   double temp_rand;
1448   unsigned int outer_count;
1449   unsigned int inner_count;
1450   int connect_attempts;
1451   double probability;
1452   char *p_string;
1453
1454   probability = 0.5; /* FIXME: default percentage? */
1455   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1456                                                          "TESTING",
1457                                                          "PROBABILITY",
1458                                                          &p_string))
1459     {
1460       if (sscanf(p_string, "%lf", &probability) != 1)
1461         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1462                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1463                     p_string,
1464                     "PROBABILITY",
1465                     "TESTING");
1466       GNUNET_free (p_string);
1467     }
1468   connect_attempts = 0;
1469   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1470     {
1471       for (inner_count = outer_count + 1; inner_count < pg->total;
1472            inner_count++)
1473         {
1474           temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1475                                                          UINT64_MAX)) / ( (double) UINT64_MAX);
1476 #if VERBOSE_TESTING
1477           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1478                       _("rand is %f probability is %f\n"), temp_rand,
1479                       probability);
1480 #endif
1481           if (temp_rand < probability)
1482             {
1483               connect_attempts += proc (pg, outer_count, inner_count);
1484             }
1485         }
1486     }
1487
1488   return connect_attempts;
1489 }
1490
1491 /**
1492  * Create a topology given a peer group (set of running peers)
1493  * and a connection processor.
1494  *
1495  * @param pg the peergroup to create the topology on
1496  * @param proc the connection processor to call to actually set
1497  *        up connections between two peers
1498  *
1499  * @return the number of connections that were set up
1500  *
1501  */
1502 static int
1503 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1504 {
1505   unsigned int i;
1506   unsigned int square;
1507   unsigned int rows;
1508   unsigned int cols;
1509   unsigned int toggle = 1;
1510   unsigned int nodeToConnect;
1511   int connect_attempts;
1512
1513   connect_attempts = 0;
1514
1515   square = floor (sqrt (pg->total));
1516   rows = square;
1517   cols = square;
1518
1519   if (square * square != pg->total)
1520     {
1521       while (rows * cols < pg->total)
1522         {
1523           if (toggle % 2 == 0)
1524             rows++;
1525           else
1526             cols++;
1527
1528           toggle++;
1529         }
1530     }
1531 #if VERBOSE_TESTING
1532       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1533                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1534                   rows, cols);
1535 #endif
1536   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1537    * to the node to its right and above.  Once this is over, we'll have our torus!
1538    * Special case for the last node (if the rows and columns are not equal), connect
1539    * to the first in the row to maintain topology.
1540    */
1541   for (i = 0; i < pg->total; i++)
1542     {
1543       /* First connect to the node to the right */
1544       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1545         nodeToConnect = i + 1;
1546       else if (i + 1 == pg->total)
1547         nodeToConnect = rows * cols - cols;
1548       else
1549         nodeToConnect = i - cols + 1;
1550 #if VERBOSE_TESTING
1551           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1552                       "Connecting peer %d to peer %d\n",
1553                       i, nodeToConnect);
1554 #endif
1555       connect_attempts += proc(pg, i, nodeToConnect);
1556
1557       /* Second connect to the node immediately above */
1558       if (i < cols)
1559         nodeToConnect = (rows * cols) - cols + i;
1560       else
1561         nodeToConnect = i - cols;
1562
1563       if (nodeToConnect < pg->total)
1564         {
1565 #if VERBOSE_TESTING
1566           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1567                       "Connecting peer %d to peer %d\n",
1568                       i, nodeToConnect);
1569 #endif
1570           connect_attempts += proc(pg, i, nodeToConnect);
1571         }
1572
1573     }
1574
1575   return connect_attempts;
1576 }
1577
1578
1579 /**
1580  * Create a topology given a peer group (set of running peers)
1581  * and a connection processor.
1582  *
1583  * @param pg the peergroup to create the topology on
1584  * @param proc the connection processor to call to actually set
1585  *        up connections between two peers
1586  *
1587  * @return the number of connections that were set up
1588  *
1589  */
1590 static int
1591 create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1592 {
1593   unsigned int outer_count;
1594   unsigned int inner_count;
1595   int connect_attempts;
1596
1597   connect_attempts = 0;
1598
1599   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1600     {
1601       for (inner_count = outer_count + 1; inner_count < pg->total;
1602            inner_count++)
1603         {
1604 #if VERBOSE_TESTING
1605           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1606                       "Connecting peer %d to peer %d\n",
1607                       outer_count, inner_count);
1608 #endif
1609           connect_attempts += proc(pg, outer_count, inner_count);
1610         }
1611     }
1612
1613   return connect_attempts;
1614 }
1615
1616 /**
1617  * Create a topology given a peer group (set of running peers)
1618  * and a connection processor.
1619  *
1620  * @param pg the peergroup to create the topology on
1621  * @param proc the connection processor to call to actually set
1622  *        up connections between two peers
1623  *
1624  * @return the number of connections that were set up
1625  *
1626  */
1627 static int
1628 create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1629 {
1630   unsigned int count;
1631   int connect_attempts;
1632
1633   connect_attempts = 0;
1634
1635   /* Connect each peer to the next highest numbered peer */
1636   for (count = 0; count < pg->total - 1; count++)
1637     {
1638 #if VERBOSE_TESTING
1639           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1640                       "Connecting peer %d to peer %d\n",
1641                       count, count + 1);
1642 #endif
1643       connect_attempts += proc(pg, count, count + 1);
1644     }
1645
1646   return connect_attempts;
1647 }
1648
1649 /**
1650  * Create a topology given a peer group (set of running peers)
1651  * and a connection processor.
1652  *
1653  * @param pg the peergroup to create the topology on
1654  * @param proc the connection processor to call to actually set
1655  *        up connections between two peers
1656  *
1657  * @return the number of connections that were set up
1658  *
1659  */
1660 static int
1661 create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1662 {
1663   unsigned int count;
1664   int connect_attempts;
1665
1666   connect_attempts = 0;
1667
1668   /* Connect each peer to the next highest numbered peer */
1669   for (count = 0; count < pg->total - 1; count++)
1670     {
1671 #if VERBOSE_TESTING
1672           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1673                       "Connecting peer %d to peer %d\n",
1674                       count, count + 1);
1675 #endif
1676       connect_attempts += proc(pg, count, count + 1);
1677     }
1678
1679   /* Connect the last peer to the first peer */
1680   connect_attempts += proc(pg, pg->total - 1, 0);
1681
1682   return connect_attempts;
1683 }
1684
1685
1686 /**
1687  * Iterator for writing friends of a peer to a file.
1688  *
1689  * @param cls closure, an open writable file handle
1690  * @param key the key the daemon was stored under
1691  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1692  *
1693  * @return GNUNET_YES to continue iteration
1694  *
1695  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1696  *       with a single file_iterator that takes a closure which contains
1697  *       the prefix to write before the peer.  Then this could be used
1698  *       for blacklisting multiple transports and writing the friend
1699  *       file.  I'm sure *someone* will complain loudly about other
1700  *       things that negate these functions even existing so no point in
1701  *       "fixing" now.
1702  */
1703 static int
1704 friend_file_iterator (void *cls,
1705                   const GNUNET_HashCode * key,
1706                   void *value)
1707 {
1708   FILE *temp_friend_handle = cls;
1709   struct GNUNET_TESTING_Daemon *peer = value;
1710   struct GNUNET_PeerIdentity *temppeer;
1711   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1712
1713   temppeer = &peer->id;
1714   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1715   fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
1716
1717   return GNUNET_YES;
1718 }
1719
1720 struct BlacklistContext
1721 {
1722   /*
1723    * The (open) file handle to write to
1724    */
1725   FILE *temp_file_handle;
1726
1727   /*
1728    * The transport that this peer will be blacklisted on.
1729    */
1730   char *transport;
1731 };
1732
1733 /**
1734  * Iterator for writing blacklist data to appropriate files.
1735  *
1736  * @param cls closure, an open writable file handle
1737  * @param key the key the daemon was stored under
1738  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1739  *
1740  * @return GNUNET_YES to continue iteration
1741  */
1742 static int
1743 blacklist_file_iterator (void *cls,
1744                          const GNUNET_HashCode * key,
1745                          void *value)
1746 {
1747   struct BlacklistContext *blacklist_ctx = cls;
1748   //FILE *temp_blacklist_handle = cls;
1749   struct GNUNET_TESTING_Daemon *peer = value;
1750   struct GNUNET_PeerIdentity *temppeer;
1751   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1752
1753   temppeer = &peer->id;
1754   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1755   fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
1756
1757   return GNUNET_YES;
1758 }
1759
1760 /*
1761  * Create the friend files based on the PeerConnection's
1762  * of each peer in the peer group, and copy the files
1763  * to the appropriate place
1764  *
1765  * @param pg the peer group we are dealing with
1766  */
1767 static int
1768 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
1769 {
1770   FILE *temp_friend_handle;
1771   unsigned int pg_iter;
1772   char *temp_service_path;
1773   pid_t *pidarr;
1774   char *arg;
1775   char * mytemp;
1776   enum GNUNET_OS_ProcessStatusType type;
1777   unsigned long return_code;
1778   int count;
1779   int ret;
1780   int max_wait = 10;
1781
1782   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1783   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1784     {
1785       mytemp = GNUNET_DISK_mktemp("friends");
1786       GNUNET_assert(mytemp != NULL);
1787       temp_friend_handle = fopen (mytemp, "wt");
1788       GNUNET_assert(temp_friend_handle != NULL);
1789       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
1790       fclose(temp_friend_handle);
1791
1792       if (GNUNET_OK !=
1793           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1794         {
1795           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1796                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1797                       "SERVICEHOME",
1798                       "PATHS");
1799           if (UNLINK (mytemp) != 0)
1800             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1801           GNUNET_free (mytemp);
1802           break;
1803         }
1804
1805       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1806         {
1807           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
1808           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1809                                          "mv", mytemp, arg, NULL);
1810 #if VERBOSE_TESTING
1811           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1812                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1813 #endif
1814
1815           GNUNET_free(arg);
1816         }
1817       else /* Remote, scp the file to the correct place */
1818         {
1819           if (NULL != pg->peers[pg_iter].daemon->username)
1820             GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1821           else
1822             GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1823           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1824                                          "scp", mytemp, arg, NULL);
1825
1826 #if VERBOSE_TESTING
1827           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1828                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1829 #endif
1830           GNUNET_free(arg);
1831         }
1832       GNUNET_free (temp_service_path);
1833       GNUNET_free (mytemp);
1834     }
1835
1836   count = 0;
1837   ret = GNUNET_SYSERR;
1838   while ((count < max_wait) && (ret != GNUNET_OK))
1839     {
1840       ret = GNUNET_OK;
1841       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1842         {
1843 #if VERBOSE_TESTING
1844           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1845                       _("Checking copy status of file %d\n"), pg_iter);
1846 #endif
1847           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1848             {
1849               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1850                 {
1851                   ret = GNUNET_SYSERR;
1852                 }
1853               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1854                 {
1855                   ret = GNUNET_SYSERR;
1856                 }
1857               else
1858                 {
1859                   pidarr[pg_iter] = 0;
1860 #if VERBOSE_TESTING
1861             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1862                       _("File %d copied\n"), pg_iter);
1863 #endif
1864                 }
1865             }
1866         }
1867       count++;
1868       if (ret == GNUNET_SYSERR)
1869         {
1870           /* FIXME: why sleep here? -CG */
1871           sleep(1);
1872         }
1873     }
1874
1875 #if VERBOSE_TESTING
1876     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1877                 _("Finished copying all friend files!\n"));
1878 #endif
1879   GNUNET_free(pidarr);
1880   return ret;
1881 }
1882
1883
1884 /*
1885  * Create the blacklist files based on the PeerConnection's
1886  * of each peer in the peer group, and copy the files
1887  * to the appropriate place.
1888  *
1889  * @param pg the peer group we are dealing with
1890  * @param transports space delimited list of transports to blacklist
1891  */
1892 static int
1893 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
1894 {
1895   FILE *temp_file_handle;
1896   static struct BlacklistContext blacklist_ctx;
1897   unsigned int pg_iter;
1898   char *temp_service_path;
1899   pid_t *pidarr;
1900   char *arg;
1901   char *mytemp;
1902   enum GNUNET_OS_ProcessStatusType type;
1903   unsigned long return_code;
1904   int count;
1905   int ret;
1906   int max_wait = 10;
1907   int transport_len;
1908   unsigned int i;
1909   char *pos;
1910   char *temp_transports;
1911
1912   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1913   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1914     {
1915       mytemp = GNUNET_DISK_mktemp("blacklist");
1916       GNUNET_assert(mytemp != NULL);
1917       temp_file_handle = fopen (mytemp, "wt");
1918       GNUNET_assert(temp_file_handle != NULL);
1919       temp_transports = GNUNET_strdup(transports);
1920       blacklist_ctx.temp_file_handle = temp_file_handle;
1921       transport_len = strlen(temp_transports) + 1;
1922       pos = NULL;
1923
1924       for (i = 0; i < transport_len; i++)
1925       {
1926         if ((temp_transports[i] == ' ') && (pos == NULL))
1927           continue; /* At start of string (whitespace) */
1928         else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
1929         {
1930           temp_transports[i] = '\0';
1931           blacklist_ctx.transport = pos;
1932           GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
1933           pos = NULL;
1934         } /* At beginning of actual string */
1935         else if (pos == NULL)
1936         {
1937           pos = &temp_transports[i];
1938         }
1939       }
1940
1941       GNUNET_free (temp_transports);
1942       fclose(temp_file_handle);
1943
1944       if (GNUNET_OK !=
1945           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1946         {
1947           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1948                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1949                       "SERVICEHOME",
1950                       "PATHS");
1951           if (UNLINK (mytemp) != 0)
1952             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1953           GNUNET_free (mytemp);
1954           break;
1955         }
1956
1957       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1958         {
1959           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
1960           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1961                                          "mv", mytemp, arg, NULL);
1962 #if VERBOSE_TESTING
1963           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1964                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1965 #endif
1966
1967           GNUNET_free(arg);
1968         }
1969       else /* Remote, scp the file to the correct place */
1970         {
1971           if (NULL != pg->peers[pg_iter].daemon->username)
1972             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1973           else
1974             GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1975           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1976                                          "scp", mytemp, arg, NULL);
1977
1978 #if VERBOSE_TESTING
1979           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1980                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1981 #endif
1982           GNUNET_free(arg);
1983         }
1984       GNUNET_free (temp_service_path);
1985       GNUNET_free (mytemp);
1986     }
1987
1988   count = 0;
1989   ret = GNUNET_SYSERR;
1990   while ((count < max_wait) && (ret != GNUNET_OK))
1991     {
1992       ret = GNUNET_OK;
1993       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1994         {
1995 #if VERBOSE_TESTING
1996           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1997                       _("Checking copy status of file %d\n"), pg_iter);
1998 #endif
1999           if (pidarr[pg_iter] != 0) /* Check for already completed! */
2000             {
2001               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
2002                 {
2003                   ret = GNUNET_SYSERR;
2004                 }
2005               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
2006                 {
2007                   ret = GNUNET_SYSERR;
2008                 }
2009               else
2010                 {
2011                   pidarr[pg_iter] = 0;
2012 #if VERBOSE_TESTING
2013             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2014                       _("File %d copied\n"), pg_iter);
2015 #endif
2016                 }
2017             }
2018         }
2019       count++;
2020       if (ret == GNUNET_SYSERR)
2021         {
2022           /* FIXME: why sleep here? -CG */
2023           sleep(1);
2024         }
2025     }
2026
2027 #if VERBOSE_TESTING
2028     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2029                 _("Finished copying all blacklist files!\n"));
2030 #endif
2031   GNUNET_free(pidarr);
2032   return ret;
2033 }
2034
2035
2036 /**
2037  * Internal notification of a connection, kept so that we can ensure some connections
2038  * happen instead of flooding all testing daemons with requests to connect.
2039  */
2040 static void internal_connect_notify (void *cls,
2041                                      const struct GNUNET_PeerIdentity *first,
2042                                      const struct GNUNET_PeerIdentity *second,
2043                                      uint32_t distance,
2044                                      const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2045                                      const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2046                                      struct GNUNET_TESTING_Daemon *first_daemon,
2047                                      struct GNUNET_TESTING_Daemon *second_daemon,
2048                                      const char *emsg)
2049 {
2050   struct GNUNET_TESTING_PeerGroup *pg = cls;
2051   outstanding_connects--;
2052
2053   pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
2054 }
2055
2056
2057 /**
2058  * Either delay a connection (because there are too many outstanding)
2059  * or schedule it for right now.
2060  *
2061  * @param cls a connection context
2062  * @param tc the task runtime context
2063  */
2064 static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2065 {
2066   struct ConnectContext *connect_context = cls;
2067
2068   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2069     return;
2070
2071   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2072     {
2073 #if VERBOSE_TESTING > 2
2074           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2075                       _("Delaying connect, we have too many outstanding connections!\n"));
2076 #endif
2077       GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
2078     }
2079   else
2080     {
2081 #if VERBOSE_TESTING > 2
2082           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2083                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2084 #endif
2085       outstanding_connects++;
2086       GNUNET_TESTING_daemons_connect (connect_context->first,
2087                                       connect_context->second,
2088                                       CONNECT_TIMEOUT,
2089                                       CONNECT_ATTEMPTS,
2090                                       &internal_connect_notify,
2091                                       connect_context->pg);
2092       GNUNET_free(connect_context);
2093     }
2094 }
2095
2096
2097 /**
2098  * Iterator for actually scheduling connections to be created
2099  * between two peers.
2100  *
2101  * @param cls closure, a GNUNET_TESTING_Daemon
2102  * @param key the key the second Daemon was stored under
2103  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2104  *
2105  * @return GNUNET_YES to continue iteration
2106  */
2107 static int
2108 connect_iterator (void *cls,
2109                   const GNUNET_HashCode * key,
2110                   void *value)
2111 {
2112   struct PeerData *first = cls;
2113   struct GNUNET_TESTING_Daemon *second = value;
2114   struct ConnectContext *connect_context;
2115
2116   connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2117   connect_context->pg = first->pg;
2118   connect_context->first = first->daemon;
2119   connect_context->second = second;
2120   GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
2121
2122   return GNUNET_YES;
2123 }
2124
2125
2126 /**
2127  * Iterator for copying all entries in the allowed hashmap to the
2128  * connect hashmap.
2129  *
2130  * @param cls closure, a GNUNET_TESTING_Daemon
2131  * @param key the key the second Daemon was stored under
2132  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2133  *
2134  * @return GNUNET_YES to continue iteration
2135  */
2136 static int
2137 copy_topology_iterator (void *cls,
2138                   const GNUNET_HashCode * key,
2139                   void *value)
2140 {
2141   struct PeerData *first = cls;
2142
2143   GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2144
2145   return GNUNET_YES;
2146 }
2147
2148 /**
2149  * Make the peers to connect the same as those that are allowed to be
2150  * connected.
2151  *
2152  * @param pg the peer group
2153  */
2154 static int
2155 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2156 {
2157   unsigned int pg_iter;
2158   int ret;
2159   int total;
2160
2161   total = 0;
2162   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2163     {
2164       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
2165       if (GNUNET_SYSERR == ret)
2166         return GNUNET_SYSERR;
2167
2168       total = total + ret;
2169     }
2170
2171   return total;
2172 }
2173
2174
2175 /**
2176  * Connect the topology as specified by the PeerConnection's
2177  * of each peer in the peer group
2178  *
2179  * @param pg the peer group we are dealing with
2180  * @return the number of connections that will be attempted
2181  */
2182 static int
2183 connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
2184 {
2185   unsigned int pg_iter;
2186   int ret;
2187   int total;
2188 #if OLD
2189   struct PeerConnection *connection_iter;
2190   struct ConnectContext *connect_context;
2191 #endif
2192
2193   total = 0;
2194   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2195     {
2196       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
2197       if (GNUNET_SYSERR == ret)
2198         return GNUNET_SYSERR;
2199
2200       total = total + ret;
2201
2202 #if OLD
2203       connection_iter = ;
2204       while (connection_iter != NULL)
2205         {
2206           connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2207           connect_context->pg = pg;
2208           connect_context->first = ;
2209           connect_context->second = connection_iter->daemon;
2210           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
2211           connection_iter = connection_iter->next;
2212         }
2213 #endif
2214     }
2215   return total;
2216 }
2217
2218
2219 /**
2220  * Takes a peer group and creates a topology based on the
2221  * one specified.  Creates a topology means generates friend
2222  * files for the peers so they can only connect to those allowed
2223  * by the topology.  This will only have an effect once peers
2224  * are started if the FRIENDS_ONLY option is set in the base
2225  * config.  Also takes an optional restrict topology which
2226  * disallows connections based on a particular transport
2227  * UNLESS they are specified in the restricted topology.
2228  *
2229  * @param pg the peer group struct representing the running peers
2230  * @param topology which topology to connect the peers in
2231  * @param restrict_topology allow only direct TCP connections in this topology
2232  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2233  * @param restrict_transports space delimited list of transports to blacklist
2234  *                            to create restricted topology
2235  *
2236  * @return the maximum number of connections were all allowed peers
2237  *         connected to each other
2238  */
2239 int
2240 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2241                                 enum GNUNET_TESTING_Topology topology,
2242                                 enum GNUNET_TESTING_Topology restrict_topology,
2243                                 char *restrict_transports)
2244 {
2245   int ret;
2246   int num_connections;
2247   int unblacklisted_connections;
2248
2249   GNUNET_assert (pg->notify_connection != NULL);
2250   ret = GNUNET_OK;
2251
2252   switch (topology)
2253     {
2254     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2255 #if VERBOSE_TESTING
2256       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2257                   _("Creating clique topology\n"));
2258 #endif
2259       num_connections = create_clique (pg, &add_allowed_connections);
2260       break;
2261     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2262 #if VERBOSE_TESTING
2263       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2264                   _("Creating small world (ring) topology\n"));
2265 #endif
2266       num_connections = create_small_world_ring (pg, &add_allowed_connections);
2267       break;
2268     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2269 #if VERBOSE_TESTING
2270       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2271                   _("Creating small world (2d-torus) topology\n"));
2272 #endif
2273       num_connections = create_small_world (pg, &add_allowed_connections);
2274       break;
2275     case GNUNET_TESTING_TOPOLOGY_RING:
2276 #if VERBOSE_TESTING
2277       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2278                   _("Creating ring topology\n"));
2279 #endif
2280       num_connections = create_ring (pg, &add_allowed_connections);
2281       break;
2282     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2283 #if VERBOSE_TESTING
2284       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2285                   _("Creating 2d torus topology\n"));
2286 #endif
2287       num_connections = create_2d_torus (pg, &add_allowed_connections);
2288       break;
2289     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2290 #if VERBOSE_TESTING
2291       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2292                   _("Creating Erdos-Renyi topology\n"));
2293 #endif
2294       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2295       break;
2296     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2297 #if VERBOSE_TESTING
2298       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2299                   _("Creating InterNAT topology\n"));
2300 #endif
2301       num_connections = create_nated_internet (pg, &add_allowed_connections);
2302       break;
2303     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2304 #if VERBOSE_TESTING
2305       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2306                   _("Creating Scale Free topology\n"));
2307 #endif
2308       num_connections = create_scale_free (pg, &add_allowed_connections);
2309       break;
2310     case GNUNET_TESTING_TOPOLOGY_LINE:
2311 #if VERBOSE_TESTING
2312       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2313                   _("Creating straight line topology\n"));
2314 #endif
2315       num_connections = create_line (pg, &add_allowed_connections);
2316       break;
2317     case GNUNET_TESTING_TOPOLOGY_NONE:
2318 #if VERBOSE_TESTING
2319       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2320                   _("Creating no allowed topology (all peers can connect at core level)\n"));
2321 #endif
2322       num_connections = 0;
2323       break;
2324     default:
2325       num_connections = 0;
2326       break;
2327     }
2328
2329   if (num_connections < 0)
2330     return GNUNET_SYSERR;
2331
2332   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2333     {
2334       ret = create_and_copy_friend_files(pg);
2335       if (ret != GNUNET_OK)
2336         {
2337 #if VERBOSE_TESTING
2338           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2339                       _("Failed during friend file copying!\n"));
2340 #endif
2341           return GNUNET_SYSERR;
2342         }
2343       else
2344         {
2345 #if VERBOSE_TESTING
2346               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2347                           _("Friend files created/copied successfully!\n"));
2348 #endif
2349         }
2350     }
2351
2352   /* Use the create clique method to initially set all connections as blacklisted. */
2353   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2354     create_clique (pg, &blacklist_connections);
2355   unblacklisted_connections = 0;
2356   /* Un-blacklist connections as per the topology specified */
2357   switch (restrict_topology)
2358     {
2359     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2360 #if VERBOSE_TESTING
2361       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2362                   _("Blacklisting all but clique topology\n"));
2363 #endif
2364       unblacklisted_connections = create_clique (pg, &unblacklist_connections);
2365       break;
2366     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2367 #if VERBOSE_TESTING
2368       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2369                   _("Blacklisting all but small world (ring) topology\n"));
2370 #endif
2371       unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
2372       break;
2373     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2374 #if VERBOSE_TESTING
2375       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2376                   _("Blacklisting all but small world (2d-torus) topology\n"));
2377 #endif
2378       unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
2379       break;
2380     case GNUNET_TESTING_TOPOLOGY_RING:
2381 #if VERBOSE_TESTING
2382       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2383                   _("Blacklisting all but ring topology\n"));
2384 #endif
2385       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2386       break;
2387     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2388 #if VERBOSE_TESTING
2389       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2390                   _("Blacklisting all but 2d torus topology\n"));
2391 #endif
2392       unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
2393       break;
2394     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2395 #if VERBOSE_TESTING
2396       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2397                   _("Blacklisting all but Erdos-Renyi topology\n"));
2398 #endif
2399       unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
2400       break;
2401     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2402 #if VERBOSE_TESTING
2403       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2404                   _("Blacklisting all but InterNAT topology\n"));
2405 #endif
2406       unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
2407       break;
2408     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2409 #if VERBOSE_TESTING
2410       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2411                   _("Blacklisting all but Scale Free topology\n"));
2412 #endif
2413       unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
2414       break;
2415     case GNUNET_TESTING_TOPOLOGY_LINE:
2416 #if VERBOSE_TESTING
2417       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2418                   _("Blacklisting all but straight line topology\n"));
2419 #endif
2420       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2421       break;
2422     case GNUNET_TESTING_TOPOLOGY_NONE:
2423 #if VERBOSE_TESTING
2424       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2425                   _("Creating no blacklist topology (all peers can connect at transport level)\n"));
2426 #endif
2427     default:
2428       break;
2429     }
2430
2431   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2432   {
2433     ret = create_and_copy_blacklist_files(pg, restrict_transports);
2434     if (ret != GNUNET_OK)
2435       {
2436 #if VERBOSE_TESTING
2437         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2438                     _("Failed during blacklist file copying!\n"));
2439 #endif
2440         return GNUNET_SYSERR;
2441       }
2442     else
2443       {
2444 #if VERBOSE_TESTING
2445         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2446                     _("Blacklist files created/copied successfully!\n"));
2447 #endif
2448       }
2449   }
2450   return num_connections;
2451 }
2452
2453 struct RandomContext
2454 {
2455   /**
2456    * The peergroup
2457    */
2458   struct GNUNET_TESTING_PeerGroup *pg;
2459
2460   /**
2461    * uid of the first peer
2462    */
2463   uint32_t first_uid;
2464
2465   /**
2466    * Peer data for first peer.
2467    */
2468   struct PeerData *first;
2469
2470   /**
2471    * Random percentage to use
2472    */
2473   double percentage;
2474 };
2475
2476 struct MinimumContext
2477 {
2478   /**
2479    * The peergroup
2480    */
2481   struct GNUNET_TESTING_PeerGroup *pg;
2482
2483   /**
2484    * uid of the first peer
2485    */
2486   uint32_t first_uid;
2487
2488   /**
2489    * Peer data for first peer.
2490    */
2491   struct PeerData *first;
2492
2493   /**
2494    * Number of conns per peer
2495    */
2496   unsigned int num_to_add;
2497
2498   /**
2499    * Permuted array of all possible connections.  Only add the Nth
2500    * peer if it's in the Nth position.
2501    */
2502   unsigned int *pg_array;
2503
2504   /**
2505    * What number is the current element we are iterating over?
2506    */
2507   unsigned int current;
2508 };
2509
2510 struct DFSContext
2511 {
2512   /**
2513    * The peergroup
2514    */
2515   struct GNUNET_TESTING_PeerGroup *pg;
2516
2517   /**
2518    * uid of the first peer
2519    */
2520   uint32_t first_uid;
2521
2522   /**
2523    * uid of the second peer
2524    */
2525   uint32_t second_uid;
2526
2527   /**
2528    * Peer data for first peer.
2529    */
2530   struct PeerData *first;
2531
2532   /**
2533    * Which peer has been chosen as the one to add?
2534    */
2535   unsigned int chosen;
2536
2537   /**
2538    * What number is the current element we are iterating over?
2539    */
2540   unsigned int current;
2541 };
2542
2543 /**
2544  * Iterator for choosing random peers to connect.
2545  *
2546  * @param cls closure, a RandomContext
2547  * @param key the key the second Daemon was stored under
2548  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2549  *
2550  * @return GNUNET_YES to continue iteration
2551  */
2552 static int
2553 random_connect_iterator (void *cls,
2554                   const GNUNET_HashCode * key,
2555                   void *value)
2556 {
2557   struct RandomContext *random_ctx = cls;
2558   double random_number;
2559   uint32_t second_pos;
2560   GNUNET_HashCode first_hash;
2561   random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
2562                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
2563   if (random_number < random_ctx->percentage)
2564   {
2565     GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2566   }
2567   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2568   uid_from_hash(key, &second_pos);
2569   hash_from_uid(random_ctx->first_uid, &first_hash);
2570   GNUNET_assert(random_ctx->pg->total > second_pos);
2571   GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
2572
2573   return GNUNET_YES;
2574 }
2575
2576 /**
2577  * Iterator for adding at least X peers to a peers connection set.
2578  *
2579  * @param cls closure, MinimumContext
2580  * @param key the key the second Daemon was stored under
2581  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2582  *
2583  * @return GNUNET_YES to continue iteration
2584  */
2585 static int
2586 minimum_connect_iterator (void *cls,
2587                   const GNUNET_HashCode * key,
2588                   void *value)
2589 {
2590   struct MinimumContext *min_ctx = cls;
2591   uint32_t second_pos;
2592   GNUNET_HashCode first_hash;
2593   unsigned int i;
2594
2595   if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2596   {
2597     for (i = 0; i < min_ctx->num_to_add; i++)
2598     {
2599       if (min_ctx->pg_array[i] == min_ctx->current)
2600       {
2601         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2602         uid_from_hash(key, &second_pos);
2603         hash_from_uid(min_ctx->first_uid, &first_hash);
2604         GNUNET_assert(min_ctx->pg->total > second_pos);
2605         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2606         /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2607         GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
2608       }
2609     }
2610     min_ctx->current++;
2611     return GNUNET_YES;
2612   }
2613   else
2614     return GNUNET_NO; /* We can stop iterating, we have enough peers! */
2615
2616 }
2617
2618
2619 /**
2620  * Iterator for adding peers to a connection set based on a depth first search.
2621  *
2622  * @param cls closure, MinimumContext
2623  * @param key the key the second daemon was stored under
2624  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2625  *
2626  * @return GNUNET_YES to continue iteration
2627  */
2628 static int
2629 dfs_connect_iterator (void *cls,
2630                   const GNUNET_HashCode * key,
2631                   void *value)
2632 {
2633   struct DFSContext *dfs_ctx = cls;
2634   GNUNET_HashCode first_hash;
2635
2636   if (dfs_ctx->current == dfs_ctx->chosen)
2637     {
2638       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2639       uid_from_hash(key, &dfs_ctx->second_uid);
2640       hash_from_uid(dfs_ctx->first_uid, &first_hash);
2641       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2642       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
2643       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
2644       return GNUNET_NO; /* We have found our peer, don't iterate more */
2645     }
2646
2647   dfs_ctx->current++;
2648   return GNUNET_YES;
2649 }
2650
2651
2652 /**
2653  * From the set of connections possible, choose percentage percent of connections
2654  * to actually connect.
2655  *
2656  * @param pg the peergroup we are dealing with
2657  * @param percentage what percent of total connections to make
2658  */
2659 void
2660 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
2661 {
2662   struct RandomContext random_ctx;
2663   uint32_t pg_iter;
2664
2665   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2666     {
2667       random_ctx.first_uid = pg_iter;
2668       random_ctx.first = &pg->peers[pg_iter];
2669       random_ctx.percentage = percentage;
2670       random_ctx.pg = pg;
2671       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
2672       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
2673       /* Now remove the old connections */
2674       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2675       /* And replace with the random set */
2676       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2677     }
2678 }
2679
2680 /**
2681  * From the set of connections possible, choose at least num connections per
2682  * peer.
2683  *
2684  * @param pg the peergroup we are dealing with
2685  * @param num how many connections at least should each peer have (if possible)?
2686  */
2687 static void
2688 choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2689 {
2690   struct MinimumContext minimum_ctx;
2691   uint32_t pg_iter;
2692
2693   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2694    {
2695      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2696    }
2697
2698   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2699     {
2700       minimum_ctx.first_uid = pg_iter;
2701       minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK,
2702                                                           GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
2703       minimum_ctx.first = &pg->peers[pg_iter];
2704       minimum_ctx.pg = pg;
2705       minimum_ctx.num_to_add = num;
2706       minimum_ctx.current = 0;
2707       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
2708                                             &minimum_connect_iterator,
2709                                             &minimum_ctx);
2710     }
2711
2712   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2713     {
2714       /* Remove the "old" connections */
2715       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2716       /* And replace with the working set */
2717       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2718     }
2719
2720 }
2721
2722
2723 static unsigned int
2724 count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
2725 {
2726   unsigned int count;
2727   unsigned int pg_iter;
2728
2729   count = 0;
2730
2731   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2732     {
2733       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2734     }
2735
2736   return count;
2737 }
2738
2739
2740 static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
2741 {
2742   unsigned int count;
2743   unsigned int pg_iter;
2744
2745   count = 0;
2746
2747   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2748     {
2749       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
2750     }
2751
2752   return count;
2753 }
2754
2755 /**
2756  * From the set of connections possible, choose at least num connections per
2757  * peer based on depth first traversal of peer connections.  If DFS leaves
2758  * peers unconnected, ensure those peers get connections.
2759  *
2760  * @param pg the peergroup we are dealing with
2761  * @param num how many connections at least should each peer have (if possible)?
2762  */
2763 void
2764 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2765 {
2766   struct DFSContext dfs_ctx;
2767   uint32_t pg_iter;
2768   uint32_t dfs_count;
2769   uint32_t starting_peer;
2770   uint32_t least_connections;
2771   GNUNET_HashCode second_hash;
2772
2773   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2774     {
2775       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2776     }
2777
2778   starting_peer = 0;
2779   dfs_count = 0;
2780   while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
2781     {
2782       if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
2783         {
2784           least_connections = -1; /* Set to very high number */
2785           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2786             {
2787               if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
2788                 {
2789                   starting_peer = pg_iter;
2790                   least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2791                 }
2792             }
2793         }
2794
2795       if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
2796         {
2797           dfs_count = 0;
2798           continue;
2799         }
2800
2801       /* Choose a random peer from the chosen peers set of connections to add */
2802       dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
2803       dfs_ctx.first_uid = starting_peer;
2804       dfs_ctx.first = &pg->peers[starting_peer];
2805       dfs_ctx.pg = pg;
2806       dfs_ctx.current = 0;
2807
2808       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
2809       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
2810       hash_from_uid(dfs_ctx.second_uid, &second_hash);
2811       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
2812       starting_peer = dfs_ctx.second_uid;
2813     }
2814
2815   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2816     {
2817       /* Remove the "old" connections */
2818       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2819       /* And replace with the working set */
2820       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2821     }
2822 }
2823
2824 /**
2825  * Internal callback for topology information for a particular peer.
2826  */
2827 static void
2828 internal_topology_callback(void *cls,
2829                            const struct GNUNET_PeerIdentity *peer,
2830                            struct GNUNET_TIME_Relative latency, uint32_t distance)
2831 {
2832   struct CoreContext *core_ctx = cls;
2833   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
2834
2835   if (peer == NULL) /* Either finished, or something went wrong */
2836     {
2837       iter_ctx->completed++;
2838       iter_ctx->connected--;
2839       /* One core context allocated per iteration, must free! */
2840       GNUNET_free(core_ctx);
2841     }
2842   else
2843     {
2844       iter_ctx->topology_cb(iter_ctx->cls, &core_ctx->daemon->id, peer, latency, distance, NULL);
2845     }
2846
2847   if (iter_ctx->completed == iter_ctx->total)
2848     {
2849       iter_ctx->topology_cb(iter_ctx->cls, NULL, NULL, GNUNET_TIME_relative_get_zero(), 0, NULL);
2850       /* Once all are done, free the iteration context */
2851       GNUNET_free(iter_ctx);
2852     }
2853 }
2854
2855
2856 /**
2857  * Check running topology iteration tasks, if below max start a new one, otherwise
2858  * schedule for some time in the future.
2859  */
2860 static void
2861 schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2862 {
2863   struct CoreContext *core_context = cls;
2864   struct TopologyIterateContext *topology_context = (struct TopologyIterateContext *)core_context->iter_context;
2865   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2866     return;
2867
2868   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2869     {
2870 #if VERBOSE_TESTING > 2
2871           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2872                       _("Delaying connect, we have too many outstanding connections!\n"));
2873 #endif
2874       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_topology, core_context);
2875     }
2876   else
2877     {
2878 #if VERBOSE_TESTING > 2
2879           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2880                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2881 #endif
2882       topology_context->connected++;
2883       if (GNUNET_OK != GNUNET_CORE_iterate_peers (core_context->daemon->sched, core_context->daemon->cfg, &internal_topology_callback, core_context))
2884         internal_topology_callback(core_context, NULL, GNUNET_TIME_relative_get_zero(), 0);
2885
2886     }
2887 }
2888
2889 /**
2890  * Iterate over all (running) peers in the peer group, retrieve
2891  * all connections that each currently has.
2892  */
2893 void
2894 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyTopology cb, void *cls)
2895 {
2896   struct TopologyIterateContext *topology_context;
2897   struct CoreContext *core_ctx;
2898   unsigned int i;
2899   unsigned int total_count;
2900
2901   /* Allocate a single topology iteration context */
2902   topology_context = GNUNET_malloc(sizeof(struct TopologyIterateContext));
2903   topology_context->topology_cb = cb;
2904   topology_context->cls = cls;
2905   total_count = 0;
2906   for (i = 0; i < pg->total; i++)
2907     {
2908       if (pg->peers[i].daemon->running == GNUNET_YES)
2909         {
2910           /* Allocate one core context per core we need to connect to */
2911           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
2912           core_ctx->daemon = pg->peers[i].daemon;
2913           /* Set back pointer to topology iteration context */
2914           core_ctx->iter_context = topology_context;
2915           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_topology, core_ctx);
2916           total_count++;
2917         }
2918     }
2919   topology_context->total = total_count;
2920   return;
2921 }
2922
2923 /**
2924  * Callback function to process statistic values.
2925  * This handler is here only really to insert a peer
2926  * identity (or daemon) so the statistics can be uniquely
2927  * tied to a single running peer.
2928  *
2929  * @param cls closure
2930  * @param subsystem name of subsystem that created the statistic
2931  * @param name the name of the datum
2932  * @param value the current value
2933  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
2934  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
2935  */
2936 static int internal_stats_callback (void *cls,
2937                                     const char *subsystem,
2938                                     const char *name,
2939                                     uint64_t value,
2940                                     int is_persistent)
2941 {
2942   struct StatsCoreContext *core_context = cls;
2943   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2944
2945   return stats_context->proc(stats_context->cls, &core_context->daemon->id, subsystem, name, value, is_persistent);
2946 }
2947
2948 /**
2949  * Internal continuation call for statistics iteration.
2950  *
2951  * @param cls closure, the CoreContext for this iteration
2952  * @param success whether or not the statistics iterations
2953  *        was canceled or not (we don't care)
2954  */
2955 static void internal_stats_cont (void *cls, int success)
2956 {
2957   struct StatsCoreContext *core_context = cls;
2958   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2959
2960   stats_context->connected--;
2961   stats_context->completed++;
2962
2963   if (stats_context->completed == stats_context->total)
2964     {
2965       stats_context->cont(stats_context->cls, GNUNET_YES);
2966       GNUNET_free(stats_context);
2967     }
2968
2969   if (core_context->stats_handle != NULL)
2970     GNUNET_STATISTICS_destroy(core_context->stats_handle, GNUNET_NO);
2971
2972   GNUNET_free(core_context);
2973 }
2974
2975 /**
2976  * Check running topology iteration tasks, if below max start a new one, otherwise
2977  * schedule for some time in the future.
2978  */
2979 static void
2980 schedule_get_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2981 {
2982   struct StatsCoreContext *core_context = cls;
2983   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
2984
2985   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2986     return;
2987
2988   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2989     {
2990 #if VERBOSE_TESTING > 2
2991           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2992                       _("Delaying connect, we have too many outstanding connections!\n"));
2993 #endif
2994       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_statistics, core_context);
2995     }
2996   else
2997     {
2998 #if VERBOSE_TESTING > 2
2999           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3000                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
3001 #endif
3002
3003       stats_context->connected++;
3004       core_context->stats_handle = GNUNET_STATISTICS_create(core_context->daemon->sched, "testing", core_context->daemon->cfg);
3005       if (core_context->stats_handle == NULL)
3006         {
3007           internal_stats_cont (core_context, GNUNET_NO);
3008           return;
3009         }
3010
3011       core_context->stats_get_handle = GNUNET_STATISTICS_get(core_context->stats_handle, NULL, NULL, GNUNET_TIME_relative_get_forever(), &internal_stats_cont, &internal_stats_callback, core_context);
3012       if (core_context->stats_get_handle == NULL)
3013         internal_stats_cont (core_context, GNUNET_NO);
3014
3015     }
3016 }
3017
3018
3019 /**
3020  * Iterate over all (running) peers in the peer group, retrieve
3021  * all statistics from each.
3022  */
3023 void
3024 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3025                                GNUNET_STATISTICS_Callback cont,
3026                                GNUNET_TESTING_STATISTICS_Iterator proc, void *cls)
3027 {
3028   struct StatsIterateContext *stats_context;
3029   struct CoreContext *core_ctx;
3030   unsigned int i;
3031   unsigned int total_count;
3032
3033   /* Allocate a single stats iteration context */
3034   stats_context = GNUNET_malloc(sizeof(struct StatsIterateContext));
3035   stats_context->cont = cont;
3036   stats_context->proc = proc;
3037   stats_context->cls = cls;
3038   total_count = 0;
3039   for (i = 0; i < pg->total; i++)
3040     {
3041       if (pg->peers[i].daemon->running == GNUNET_YES)
3042         {
3043           /* Allocate one core context per core we need to connect to */
3044           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
3045           core_ctx->daemon = pg->peers[i].daemon;
3046           /* Set back pointer to topology iteration context */
3047           core_ctx->iter_context = stats_context;
3048           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_statistics, core_ctx);
3049           total_count++;
3050         }
3051     }
3052   stats_context->total = total_count;
3053   return;
3054 }
3055
3056 /**
3057  * There are many ways to connect peers that are supported by this function.
3058  * To connect peers in the same topology that was created via the
3059  * GNUNET_TESTING_create_topology, the topology variable must be set to
3060  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3061  * a new instance of that topology will be generated and attempted to be
3062  * connected.  This could result in some connections being impossible,
3063  * because some topologies are non-deterministic.
3064  *
3065  * @param pg the peer group struct representing the running peers
3066  * @param topology which topology to connect the peers in
3067  * @param options options for connecting the topology
3068  * @param option_modifier modifier for options that take a parameter
3069  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3070  */
3071 int
3072 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3073                                  enum GNUNET_TESTING_Topology topology,
3074                                  enum GNUNET_TESTING_TopologyOption options,
3075                                  double option_modifier)
3076 {
3077   switch (topology)
3078       {
3079       case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3080 #if VERBOSE_TOPOLOGY
3081       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3082                   _("Creating clique CONNECT topology\n"));
3083 #endif
3084         create_clique (pg, &add_actual_connections);
3085         break;
3086       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3087 #if VERBOSE_TOPOLOGY
3088       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3089                   _("Creating small world (ring) CONNECT topology\n"));
3090 #endif
3091         create_small_world_ring (pg, &add_actual_connections);
3092         break;
3093       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3094 #if VERBOSE_TOPOLOGY
3095       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3096                   _("Creating small world (2d-torus) CONNECT topology\n"));
3097 #endif
3098         create_small_world (pg, &add_actual_connections);
3099         break;
3100       case GNUNET_TESTING_TOPOLOGY_RING:
3101 #if VERBOSE_TOPOLOGY
3102       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3103                   _("Creating ring CONNECT topology\n"));
3104 #endif
3105         create_ring (pg, &add_actual_connections);
3106         break;
3107       case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3108 #if VERBOSE_TOPOLOGY
3109       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3110                   _("Creating 2d torus CONNECT topology\n"));
3111 #endif
3112         create_2d_torus (pg, &add_actual_connections);
3113         break;
3114       case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3115 #if VERBOSE_TOPOLOGY
3116       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3117                   _("Creating Erdos-Renyi CONNECT topology\n"));
3118 #endif
3119         create_erdos_renyi (pg, &add_actual_connections);
3120         break;
3121       case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3122 #if VERBOSE_TOPOLOGY
3123       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3124                   _("Creating InterNAT CONNECT topology\n"));
3125 #endif
3126         create_nated_internet (pg, &add_actual_connections);
3127         break;
3128       case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3129 #if VERBOSE_TOPOLOGY
3130       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3131                   _("Creating Scale Free CONNECT topology\n"));
3132 #endif
3133         create_scale_free (pg, &add_actual_connections);
3134         break;
3135       case GNUNET_TESTING_TOPOLOGY_LINE:
3136 #if VERBOSE_TOPOLOGY
3137       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3138                   _("Creating straight line CONNECT topology\n"));
3139 #endif
3140         create_line (pg, &add_actual_connections);
3141         break;
3142       case GNUNET_TESTING_TOPOLOGY_NONE:
3143 #if VERBOSE_TOPOLOGY
3144         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3145                   _("Creating no CONNECT topology\n"));
3146 #endif
3147         copy_allowed_topology(pg);
3148         break;
3149       default:
3150         GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3151                    _("Unknown topology specification, can't connect peers!\n"));
3152         return GNUNET_SYSERR;
3153       }
3154
3155   switch (options)
3156     {
3157     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3158 #if VERBOSE_TOPOLOGY
3159       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3160                   _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
3161 #endif
3162       choose_random_connections(pg, option_modifier);
3163       break;
3164     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3165 #if VERBOSE_TOPOLOGY
3166       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3167                   _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3168 #endif
3169       choose_minimum(pg, (unsigned int)option_modifier);
3170       break;
3171     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3172 #if VERBOSE_TOPOLOGY
3173       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3174                   _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3175 #endif
3176       perform_dfs(pg, (int)option_modifier);
3177       break;
3178     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3179       break;
3180     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3181       break;
3182     default:
3183       break;
3184     }
3185
3186   return connect_topology(pg);
3187 }
3188
3189 /**
3190  * Callback that is called whenever a hostkey is generated
3191  * for a peer.  Call the real callback and decrement the
3192  * starting counter for the peergroup.
3193  *
3194  * @param cls closure
3195  * @param id identifier for the daemon, NULL on error
3196  * @param d handle for the daemon
3197  * @param emsg error message (NULL on success)
3198  */
3199 static void internal_hostkey_callback (void *cls,
3200                                        const struct GNUNET_PeerIdentity *id,
3201                                        struct GNUNET_TESTING_Daemon *d,
3202                                        const char *emsg)
3203 {
3204   struct InternalStartContext *internal_context = cls;
3205   internal_context->peer->pg->starting--;
3206   internal_context->peer->pg->started++;
3207   if (internal_context->hostkey_callback != NULL)
3208     internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
3209   else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
3210     {
3211       internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
3212       GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
3213     }
3214 }
3215
3216 /**
3217  * Callback that is called whenever a peer has finished starting.
3218  * Call the real callback and decrement the starting counter
3219  * for the peergroup.
3220  *
3221  * @param cls closure
3222  * @param id identifier for the daemon, NULL on error
3223  * @param d handle for the daemon
3224  * @param emsg error message (NULL on success)
3225  */
3226 static void internal_startup_callback (void *cls,
3227                                        const struct GNUNET_PeerIdentity *id,
3228                                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3229                                        struct GNUNET_TESTING_Daemon *d,
3230                                        const char *emsg)
3231 {
3232   struct InternalStartContext *internal_context = cls;
3233   internal_context->peer->pg->starting--;
3234   if (internal_context->start_cb != NULL)
3235     internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
3236 }
3237
3238 static void
3239 internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3240 {
3241   struct InternalStartContext *internal_context = cls;
3242
3243   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3244     {
3245       return;
3246     }
3247
3248   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3249     {
3250       internal_context->peer->pg->starting++;
3251       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3252     }
3253   else
3254     {
3255       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
3256     }
3257 }
3258
3259 static void
3260 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3261 {
3262   struct InternalStartContext *internal_context = cls;
3263
3264   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3265     {
3266       return;
3267     }
3268
3269   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
3270     {
3271       internal_context->peer->pg->starting++;
3272       internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
3273                                                                     internal_context->peer->cfg,
3274                                                                     internal_context->timeout,
3275                                                                     internal_context->hostname,
3276                                                                     &internal_hostkey_callback,
3277                                                                     internal_context,
3278                                                                     &internal_startup_callback,
3279                                                                     internal_context);
3280     }
3281   else
3282     {
3283       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
3284     }
3285 }
3286
3287 /**
3288  * Function which continues a peer group starting up
3289  * after successfully generating hostkeys for each peer.
3290  *
3291  * @param pg the peer group to continue starting
3292  *
3293  */
3294 void
3295 GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
3296 {
3297   unsigned int i;
3298
3299   pg->starting = 0;
3300   for (i = 0; i < pg->total; i++)
3301     {
3302       GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
3303       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
3304     }
3305 }
3306
3307 /**
3308  * Start count gnunet instances with the same set of transports and
3309  * applications.  The port numbers (any option called "PORT") will be
3310  * adjusted to ensure that no two peers running on the same system
3311  * have the same port(s) in their respective configurations.
3312  *
3313  * @param sched scheduler to use
3314  * @param cfg configuration template to use
3315  * @param total number of daemons to start
3316  * @param timeout total time allowed for peers to start
3317  * @param hostkey_callback function to call on each peers hostkey generation
3318  *        if NULL, peers will be started by this call, if non-null,
3319  *        GNUNET_TESTING_daemons_continue_startup must be called after
3320  *        successful hostkey generation
3321  * @param hostkey_cls closure for hostkey callback
3322  * @param cb function to call on each daemon that was started
3323  * @param cb_cls closure for cb
3324  * @param connect_callback function to call each time two hosts are connected
3325  * @param connect_callback_cls closure for connect_callback
3326  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
3327  *
3328  * @return NULL on error, otherwise handle to control peer group
3329  */
3330 struct GNUNET_TESTING_PeerGroup *
3331 GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
3332                               const struct GNUNET_CONFIGURATION_Handle *cfg,
3333                               unsigned int total,
3334                               struct GNUNET_TIME_Relative timeout,
3335                               GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
3336                               void *hostkey_cls,
3337                               GNUNET_TESTING_NotifyDaemonRunning cb,
3338                               void *cb_cls,
3339                               GNUNET_TESTING_NotifyConnection
3340                               connect_callback, void *connect_callback_cls,
3341                               const struct GNUNET_TESTING_Host *hostnames)
3342 {
3343   struct GNUNET_TESTING_PeerGroup *pg;
3344   const struct GNUNET_TESTING_Host *hostpos;
3345 #if 0
3346   char *pos;
3347   const char *rpos;
3348   char *start;
3349 #endif
3350   const char *hostname;
3351   char *baseservicehome;
3352   char *newservicehome;
3353   char *tmpdir;
3354   struct GNUNET_CONFIGURATION_Handle *pcfg;
3355   unsigned int off;
3356   unsigned int hostcnt;
3357   uint16_t minport;
3358   uint32_t upnum;
3359
3360   if (0 == total)
3361     {
3362       GNUNET_break (0);
3363       return NULL;
3364     }
3365   upnum = 0;
3366   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
3367   pg->sched = sched;
3368   pg->cfg = cfg;
3369   pg->notify_connection = connect_callback;
3370   pg->notify_connection_cls = connect_callback_cls;
3371   pg->total = total;
3372   pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
3373   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
3374   if (NULL != hostnames)
3375     {
3376       off = 2;
3377       hostpos = hostnames;
3378       while (hostpos != NULL)
3379         {
3380           hostpos = hostpos->next;
3381           off++;
3382         }
3383       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3384       off = 0;
3385
3386       hostpos = hostnames;
3387       while (hostpos != NULL)
3388         {
3389           pg->hosts[off].minport = LOW_PORT;
3390           pg->hosts[off++].hostname = GNUNET_strdup(hostpos->hostname);
3391           hostpos = hostpos->next;
3392         }
3393
3394       if (off == 0)
3395         {
3396           GNUNET_free (pg->hosts);
3397           pg->hosts = NULL;
3398         }
3399       hostcnt = off;
3400       minport = 0;
3401
3402 #if NO_LL
3403       off = 2;
3404       /* skip leading spaces */
3405       while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
3406         hostnames++;
3407       rpos = hostnames;
3408       while ('\0' != *rpos)
3409         {
3410           if (isspace ( (unsigned char) *rpos))
3411             off++;
3412           rpos++;
3413         }
3414       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3415       off = 0;
3416       start = GNUNET_strdup (hostnames);
3417       pos = start;
3418       while ('\0' != *pos)
3419         {
3420           if (isspace ( (unsigned char) *pos))
3421             {
3422               *pos = '\0';
3423               if (strlen (start) > 0)
3424                 {
3425                   pg->hosts[off].minport = LOW_PORT;
3426                   pg->hosts[off++].hostname = start;
3427                 }
3428               start = pos + 1;
3429             }
3430           pos++;
3431         }
3432       if (strlen (start) > 0)
3433         {
3434           pg->hosts[off].minport = LOW_PORT;
3435           pg->hosts[off++].hostname = start;
3436         }
3437       if (off == 0)
3438         {
3439           GNUNET_free (start);
3440           GNUNET_free (pg->hosts);
3441           pg->hosts = NULL;
3442         }
3443       hostcnt = off;
3444       minport = 0;              /* make gcc happy */
3445 #endif
3446     }
3447   else
3448     {
3449       hostcnt = 0;
3450       minport = LOW_PORT;
3451     }
3452   for (off = 0; off < total; off++)
3453     {
3454       if (hostcnt > 0)
3455         {
3456           hostname = pg->hosts[off % hostcnt].hostname;
3457           pcfg = make_config (cfg,
3458                               &pg->hosts[off % hostcnt].minport,
3459                               &upnum,
3460                               hostname);
3461         }
3462       else
3463         {
3464           hostname = NULL;
3465           pcfg = make_config (cfg,
3466                               &minport,
3467                               &upnum,
3468                               hostname);
3469         }
3470
3471       if (NULL == pcfg)
3472         {
3473           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3474                       _
3475                       ("Could not create configuration for peer number %u on `%s'!\n"),
3476                       off, hostname == NULL ? "localhost" : hostname);
3477           continue;
3478         }
3479
3480       if (GNUNET_YES ==
3481           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
3482                                                  &baseservicehome))
3483         {
3484           GNUNET_asprintf (&newservicehome,
3485                            "%s/%d/", baseservicehome, off);
3486           GNUNET_free (baseservicehome);
3487         }
3488       else
3489         {
3490           tmpdir = getenv ("TMPDIR");
3491           tmpdir = tmpdir ? tmpdir : "/tmp";
3492           GNUNET_asprintf (&newservicehome,
3493                            "%s/%s/%d/",
3494                            tmpdir,
3495                            "gnunet-testing-test-test", off);
3496         }
3497       GNUNET_CONFIGURATION_set_value_string (pcfg,
3498                                              "PATHS",
3499                                              "SERVICEHOME", newservicehome);
3500       GNUNET_free (newservicehome);
3501       pg->peers[off].cfg = pcfg;
3502       pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
3503       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
3504       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
3505       pg->peers[off].pg = pg;
3506
3507       pg->peers[off].internal_context.peer = &pg->peers[off];
3508       pg->peers[off].internal_context.timeout = timeout;
3509       pg->peers[off].internal_context.hostname = hostname;
3510       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
3511       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
3512       pg->peers[off].internal_context.start_cb = cb;
3513       pg->peers[off].internal_context.start_cb_cls = cb_cls;
3514
3515       GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
3516
3517     }
3518   return pg;
3519 }
3520
3521 /*
3522  * Get a daemon by number, so callers don't have to do nasty
3523  * offsetting operation.
3524  */
3525 struct GNUNET_TESTING_Daemon *
3526 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
3527 {
3528   if (position < pg->total)
3529     return pg->peers[position].daemon;
3530   else
3531     return NULL;
3532 }
3533
3534 /**
3535  * Prototype of a function that will be called when a
3536  * particular operation was completed the testing library.
3537  *
3538  * @param cls closure (a struct RestartContext)
3539  * @param id id of the peer that was restarted
3540  * @param cfg handle to the configuration of the peer
3541  * @param d handle to the daemon that was restarted
3542  * @param emsg NULL on success
3543  */
3544 void restart_callback (void *cls,
3545                        const struct GNUNET_PeerIdentity *id,
3546                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3547                        struct GNUNET_TESTING_Daemon *d,
3548                        const char *emsg)
3549 {
3550   struct RestartContext *restart_context = cls;
3551
3552   if (emsg == NULL)
3553     {
3554       restart_context->peers_restarted++;
3555     }
3556   else
3557     {
3558       restart_context->peers_restart_failed++;
3559     }
3560
3561   if (restart_context->peers_restarted == restart_context->peer_group->total)
3562     {
3563       restart_context->callback(restart_context->callback_cls, NULL);
3564       GNUNET_free(restart_context);
3565     }
3566   else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
3567     {
3568       restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
3569       GNUNET_free(restart_context);
3570     }
3571
3572 }
3573
3574 /**
3575  * Callback for informing us about a successful
3576  * or unsuccessful churn stop call.
3577  *
3578  * @param cls a ChurnContext
3579  * @param emsg NULL on success, non-NULL on failure
3580  *
3581  */
3582 void
3583 churn_stop_callback (void *cls, const char *emsg)
3584 {
3585   struct ChurnContext *churn_ctx = cls;
3586   unsigned int total_left;
3587   char *error_message;
3588
3589   error_message = NULL;
3590   if (emsg != NULL)
3591     {
3592       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3593                  "Churn stop callback failed with error `%s'\n", emsg);
3594       churn_ctx->num_failed_stop++;
3595     }
3596   else
3597     {
3598       churn_ctx->num_to_stop--;
3599     }
3600
3601 #if DEBUG_CHURN
3602   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3603              "Stopped peer, %d left.\n",
3604              churn_ctx->num_to_stop);
3605 #endif
3606   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3607
3608   if (total_left == 0)
3609   {
3610     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3611       {
3612         GNUNET_asprintf(&error_message,
3613                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3614                         churn_ctx->num_failed_start,
3615                         churn_ctx->num_failed_stop);
3616       }
3617     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3618     GNUNET_free_non_null(error_message);
3619     GNUNET_free(churn_ctx);
3620   }
3621 }
3622
3623 /**
3624  * Callback for informing us about a successful
3625  * or unsuccessful churn start call.
3626  *
3627  * @param cls a ChurnContext
3628  * @param id the peer identity of the started peer
3629  * @param cfg the handle to the configuration of the peer
3630  * @param d handle to the daemon for the peer
3631  * @param emsg NULL on success, non-NULL on failure
3632  *
3633  */
3634 void
3635 churn_start_callback (void *cls,
3636                       const struct GNUNET_PeerIdentity *id,
3637                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3638                       struct GNUNET_TESTING_Daemon *d,
3639                       const char *emsg)
3640 {
3641   struct ChurnContext *churn_ctx = cls;
3642   unsigned int total_left;
3643   char *error_message;
3644
3645   error_message = NULL;
3646   if (emsg != NULL)
3647     {
3648       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3649                   "Churn stop callback failed with error `%s'\n",
3650                   emsg);
3651       churn_ctx->num_failed_start++;
3652     }
3653   else
3654     {
3655       churn_ctx->num_to_start--;
3656     }
3657
3658 #if DEBUG_CHURN
3659   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3660              "Started peer, %d left.\n",
3661              churn_ctx->num_to_start);
3662 #endif
3663
3664   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3665
3666   if (total_left == 0)
3667   {
3668     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3669       GNUNET_asprintf(&error_message,
3670                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3671                       churn_ctx->num_failed_start,
3672                       churn_ctx->num_failed_stop);
3673     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3674     GNUNET_free_non_null(error_message);
3675     GNUNET_free(churn_ctx);
3676   }
3677 }
3678
3679
3680 /**
3681  * Simulate churn by stopping some peers (and possibly
3682  * re-starting others if churn is called multiple times).  This
3683  * function can only be used to create leave-join churn (peers "never"
3684  * leave for good).  First "voff" random peers that are currently
3685  * online will be taken offline; then "von" random peers that are then
3686  * offline will be put back online.  No notifications will be
3687  * generated for any of these operations except for the callback upon
3688  * completion.
3689  *
3690  * @param pg handle for the peer group
3691  * @param voff number of peers that should go offline
3692  * @param von number of peers that should come back online;
3693  *            must be zero on first call (since "testbed_start"
3694  *            always starts all of the peers)
3695  * @param timeout how long to wait for operations to finish before
3696  *        giving up
3697  * @param cb function to call at the end
3698  * @param cb_cls closure for cb
3699  */
3700 void
3701 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
3702                               unsigned int voff,
3703                               unsigned int von,
3704                               struct GNUNET_TIME_Relative timeout,
3705                               GNUNET_TESTING_NotifyCompletion cb,
3706                               void *cb_cls)
3707 {
3708   struct ChurnContext *churn_ctx;
3709   unsigned int running;
3710   unsigned int stopped;
3711   unsigned int total_running;
3712   unsigned int total_stopped;
3713   unsigned int i;
3714   unsigned int *running_arr;
3715   unsigned int *stopped_arr;
3716   unsigned int *running_permute;
3717   unsigned int *stopped_permute;
3718
3719   running = 0;
3720   stopped = 0;
3721
3722   if ((von == 0) && (voff == 0)) /* No peers at all? */
3723     {
3724       cb(cb_cls, NULL);
3725       return;
3726     }
3727
3728   for (i = 0; i < pg->total; i++)
3729   {
3730     if (pg->peers[i].daemon->running == GNUNET_YES)
3731     {
3732       GNUNET_assert(running != -1);
3733       running++;
3734     }
3735     else
3736     {
3737       GNUNET_assert(stopped != -1);
3738       stopped++;
3739     }
3740   }
3741
3742   if (voff > running)
3743   {
3744     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
3745     cb(cb_cls, "Trying to stop more peers than are currently running!");
3746     return;
3747   }
3748
3749   if (von > stopped)
3750   {
3751     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
3752     cb(cb_cls, "Trying to start more peers than are currently stopped!");
3753     return;
3754   }
3755
3756   churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3757
3758   running_arr = NULL;
3759   if (running > 0)
3760     running_arr = GNUNET_malloc(running * sizeof(unsigned int));
3761
3762   stopped_arr = NULL;
3763   if (stopped > 0)
3764     stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
3765
3766   running_permute = NULL;
3767   stopped_permute = NULL;
3768
3769   if (running > 0)
3770     running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
3771   if (stopped > 0)
3772     stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
3773
3774   total_running = running;
3775   total_stopped = stopped;
3776   running = 0;
3777   stopped = 0;
3778
3779   churn_ctx->num_to_start = von;
3780   churn_ctx->num_to_stop = voff;
3781   churn_ctx->cb = cb;
3782   churn_ctx->cb_cls = cb_cls;
3783
3784   for (i = 0; i < pg->total; i++)
3785   {
3786     if (pg->peers[i].daemon->running == GNUNET_YES)
3787     {
3788       GNUNET_assert((running_arr != NULL) && (total_running > running));
3789       running_arr[running] = i;
3790       running++;
3791     }
3792     else
3793     {
3794       GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
3795       stopped_arr[stopped] = i;
3796       stopped++;
3797     }
3798   }
3799
3800   for (i = 0; i < voff; i++)
3801   {
3802 #if DEBUG_CHURN
3803     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
3804 #endif
3805     GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
3806                                 timeout,
3807                                 &churn_stop_callback, churn_ctx,
3808                                 GNUNET_NO, GNUNET_YES);
3809   }
3810
3811   for (i = 0; i < von; i++)
3812     {
3813 #if DEBUG_CHURN
3814       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
3815 #endif
3816       GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
3817                                           timeout, &churn_start_callback, churn_ctx);
3818   }
3819
3820   GNUNET_free_non_null(running_arr);
3821   GNUNET_free_non_null(stopped_arr);
3822   GNUNET_free_non_null(running_permute);
3823   GNUNET_free_non_null(stopped_permute);
3824 }
3825
3826
3827 /**
3828  * Restart all peers in the given group.
3829  *
3830  * @param pg the handle to the peer group
3831  * @param callback function to call on completion (or failure)
3832  * @param callback_cls closure for the callback function
3833  */
3834 void
3835 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
3836                                 GNUNET_TESTING_NotifyCompletion callback,
3837                                 void *callback_cls)
3838 {
3839   struct RestartContext *restart_context;
3840   unsigned int off;
3841
3842   if (pg->total > 0)
3843     {
3844       restart_context = GNUNET_malloc(sizeof(struct RestartContext));
3845       restart_context->peer_group = pg;
3846       restart_context->peers_restarted = 0;
3847       restart_context->callback = callback;
3848       restart_context->callback_cls = callback_cls;
3849
3850       for (off = 0; off < pg->total; off++)
3851         {
3852           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
3853         }
3854     }
3855 }
3856
3857 /**
3858  * Start or stop an individual peer from the given group.
3859  *
3860  * @param pg handle to the peer group
3861  * @param offset which peer to start or stop
3862  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
3863  * @param timeout how long to wait for shutdown
3864  * @param cb function to call at the end
3865  * @param cb_cls closure for cb
3866  */
3867 void
3868 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
3869                              unsigned int offset,
3870                              int desired_status,
3871                              struct GNUNET_TIME_Relative timeout,
3872                              GNUNET_TESTING_NotifyCompletion cb,
3873                              void *cb_cls)
3874 {
3875   struct ChurnContext *churn_ctx;
3876
3877   if (GNUNET_NO == desired_status)
3878     {
3879       if (NULL != pg->peers[offset].daemon)
3880         {
3881           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3882           churn_ctx->num_to_start = 0;
3883           churn_ctx->num_to_stop = 1;
3884           churn_ctx->cb = cb;
3885           churn_ctx->cb_cls = cb_cls;
3886           GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon,
3887                                      timeout, &churn_stop_callback, churn_ctx,
3888                                      GNUNET_NO, GNUNET_YES);
3889         }
3890     }
3891   else if (GNUNET_YES == desired_status)
3892     {
3893       if (NULL == pg->peers[offset].daemon)
3894         {
3895           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3896           churn_ctx->num_to_start = 1;
3897           churn_ctx->num_to_stop = 0;
3898           churn_ctx->cb = cb;
3899           churn_ctx->cb_cls = cb_cls;
3900           GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon,
3901                                               timeout, &churn_start_callback, churn_ctx);
3902         }
3903     }
3904   else
3905     GNUNET_break (0);
3906 }
3907
3908
3909 /**
3910  * Callback for shutting down peers in a peer group.
3911  *
3912  * @param cls closure (struct ShutdownContext)
3913  * @param emsg NULL on success
3914  */
3915 void internal_shutdown_callback (void *cls,
3916                                  const char *emsg)
3917 {
3918   struct ShutdownContext *shutdown_ctx = cls;
3919
3920   shutdown_ctx->outstanding--;
3921   if (emsg == NULL)
3922     {
3923       shutdown_ctx->peers_down++;
3924     }
3925   else
3926     {
3927       shutdown_ctx->peers_failed++;
3928     }
3929
3930   if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
3931     {
3932       if (shutdown_ctx->peers_failed > 0)
3933         shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
3934       else
3935         shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
3936       GNUNET_free(shutdown_ctx);
3937     }
3938 }
3939
3940 /**
3941  * Individual shutdown context for a particular peer.
3942  */
3943 struct PeerShutdownContext
3944 {
3945   /**
3946    * Pointer to the high level shutdown context.
3947    */
3948   struct ShutdownContext *shutdown_ctx;
3949
3950   /**
3951    * The daemon handle for the peer to shut down.
3952    */
3953   struct GNUNET_TESTING_Daemon *daemon;
3954 };
3955
3956 /**
3957  * Task to rate limit the number of outstanding peer shutdown
3958  * requests.  This is necessary for making sure we don't do
3959  * too many ssh connections at once, but is generally nicer
3960  * to any system as well (graduated task starts, as opposed
3961  * to calling gnunet-arm N times all at once).
3962  */
3963 static void
3964 schedule_shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3965 {
3966   struct PeerShutdownContext *peer_shutdown_ctx = cls;
3967   struct ShutdownContext *shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
3968
3969   GNUNET_assert(peer_shutdown_ctx != NULL);
3970   GNUNET_assert(shutdown_ctx != NULL);
3971
3972   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
3973     GNUNET_SCHEDULER_add_delayed(peer_shutdown_ctx->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_shutdown_task, peer_shutdown_ctx);
3974   else
3975     {
3976       shutdown_ctx->outstanding++;
3977       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, shutdown_ctx->timeout, &internal_shutdown_callback, shutdown_ctx, GNUNET_YES, GNUNET_NO);
3978       GNUNET_free(peer_shutdown_ctx);
3979     }
3980 }
3981 /**
3982  * Shutdown all peers started in the given group.
3983  *
3984  * @param pg handle to the peer group
3985  * @param timeout how long to wait for shutdown
3986  * @param cb callback to notify upon success or failure
3987  * @param cb_cls closure for cb
3988  */
3989 void
3990 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
3991                              struct GNUNET_TIME_Relative timeout,
3992                              GNUNET_TESTING_NotifyCompletion cb,
3993                              void *cb_cls)
3994 {
3995   unsigned int off;
3996   struct ShutdownContext *shutdown_ctx;
3997   struct PeerShutdownContext *peer_shutdown_ctx;
3998
3999   GNUNET_assert(pg->total > 0);
4000
4001   shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
4002   shutdown_ctx->cb = cb;
4003   shutdown_ctx->cb_cls = cb_cls;
4004   shutdown_ctx->total_peers = pg->total;
4005   shutdown_ctx->timeout = timeout;
4006   /* shtudown_ctx->outstanding = 0; */
4007
4008   for (off = 0; off < pg->total; off++)
4009     {
4010       GNUNET_assert(NULL != pg->peers[off].daemon);
4011       peer_shutdown_ctx = GNUNET_malloc(sizeof(struct PeerShutdownContext));
4012       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4013       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4014       GNUNET_SCHEDULER_add_now(pg->peers[off].daemon->sched, &schedule_shutdown_task, peer_shutdown_ctx);
4015       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4016       if (NULL != pg->peers[off].cfg)
4017         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4018       if (pg->peers[off].allowed_peers != NULL)
4019         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
4020       if (pg->peers[off].connect_peers != NULL)
4021         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
4022       if (pg->peers[off].blacklisted_peers != NULL)
4023         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
4024     }
4025   GNUNET_free (pg->peers);
4026   if (NULL != pg->hosts)
4027     {
4028       GNUNET_free (pg->hosts[0].hostname);
4029       GNUNET_free (pg->hosts);
4030     }
4031   GNUNET_free (pg);
4032 }
4033
4034
4035 /* end of testing_group.c */