src/testing/testing_group.c

   1 /*
   2       This file is part of GNUnet
   3       (C) 2008, 2009 Christian Grothoff (and other contributing authors)
   4
   5       GNUnet is free software; you can redistribute it and/or modify
   6       it under the terms of the GNU General Public License as published
   7       by the Free Software Foundation; either version 3, or (at your
   8       option) any later version.
   9
  10       GNUnet is distributed in the hope that it will be useful, but
  11       WITHOUT ANY WARRANTY; without even the implied warranty of
  12       MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13       General Public License for more details.
  14
  15       You should have received a copy of the GNU General Public License
  16       along with GNUnet; see the file COPYING.  If not, write to the
  17       Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18       Boston, MA 02111-1307, USA.
  19  */
  20
  21 /**
  22  * @file testing/testing_group.c
  23  * @brief convenience API for writing testcases for GNUnet
  24  * @author Nathan Evans
  25  * @author Christian Grothoff
  26  *
  27  */
  28 #include "platform.h"
  29 #include "gnunet_arm_service.h"
  30 #include "gnunet_testing_lib.h"
  31 #include "gnunet_core_service.h"
  32
  33 #define VERBOSE_TESTING GNUNET_NO
  34
  35 #define DEBUG_CHURN GNUNET_NO
  36
  37 /**
  38  * Lowest port used for GNUnet testing.  Should be high enough to not
  39  * conflict with other applications running on the hosts but be low
  40  * enough to not conflict with client-ports (typically starting around
  41  * 32k).
  42  */
  43 #define LOW_PORT 10000
  44
  45 /**
  46  * Highest port used for GNUnet testing.  Should be low enough to not
  47  * conflict with the port range for "local" ports (client apps; see
  48  * /proc/sys/net/ipv4/ip_local_port_range on Linux for example).
  49  */
  50 #define HIGH_PORT 56000
  51
  52 #define MAX_OUTSTANDING_CONNECTIONS 10
  53
  54 #define MAX_CONCURRENT_HOSTKEYS 10
  55
  56 #define MAX_CONCURRENT_STARTING 10
  57
  58 #define MAX_CONCURRENT_SHUTDOWN 10
  59
  60 #define CONNECT_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 300)
  61
  62 #define CONNECT_ATTEMPTS 8
  63
  64 /**
  65  * Prototype of a function called whenever two peers would be connected
  66  * in a certain topology.
  67  */
  68 typedef int (*GNUNET_TESTING_ConnectionProcessor)(struct GNUNET_TESTING_PeerGroup *pg,
  69                                                   unsigned int first,
  70                                                   unsigned int second);
  71
  72
  73 /**
  74  * Context for handling churning a peer group
  75  */
  76 struct ChurnContext
  77 {
  78   /**
  79    * Callback used to notify of churning finished
  80    */
  81   GNUNET_TESTING_NotifyCompletion cb;
  82
  83   /**
  84    * Closure for callback
  85    */
  86   void *cb_cls;
  87
  88   /**
  89    * Number of peers that still need to be started
  90    */
  91   unsigned int num_to_start;
  92
  93   /**
  94    * Number of peers that still need to be stopped
  95    */
  96   unsigned int num_to_stop;
  97
  98   /**
  99    * Number of peers that failed to start
 100    */
 101   unsigned int num_failed_start;
 102
 103   /**
 104    * Number of peers that failed to stop
 105    */
 106   unsigned int num_failed_stop;
 107 };
 108
 109 struct RestartContext
 110 {
 111   /**
 112    * The group of peers being restarted
 113    */
 114   struct GNUNET_TESTING_PeerGroup *peer_group;
 115
 116   /**
 117    * How many peers have been restarted thus far
 118    */
 119   unsigned int peers_restarted;
 120
 121   /**
 122    * How many peers got an error when restarting
 123    */
 124   unsigned int peers_restart_failed;
 125
 126   /**
 127    * The function to call once all peers have been restarted
 128    */
 129   GNUNET_TESTING_NotifyCompletion callback;
 130
 131   /**
 132    * Closure for callback function
 133    */
 134   void *callback_cls;
 135
 136 };
 137
 138
 139 struct ShutdownContext
 140 {
 141   /**
 142    * Total peers to wait for
 143    */
 144   unsigned int total_peers;
 145
 146   /**
 147    * Number of peers successfully shut down
 148    */
 149   unsigned int peers_down;
 150
 151   /**
 152    * Number of peers failed to shut down
 153    */
 154   unsigned int peers_failed;
 155
 156   /**
 157    * Number of peers we have started shutting
 158    * down.  If too many, wait on them.
 159    */
 160   unsigned int outstanding;
 161
 162   /**
 163    * Timeout for shutdown.
 164    */
 165   struct GNUNET_TIME_Relative timeout;
 166
 167   /**
 168    * Callback to call when all peers either
 169    * shutdown or failed to shutdown
 170    */
 171   GNUNET_TESTING_NotifyCompletion cb;
 172
 173   /**
 174    * Closure for cb
 175    */
 176   void *cb_cls;
 177 };
 178
 179 struct CreateTopologyContext
 180 {
 181
 182   /**
 183    * Function to call with number of connections
 184    */
 185   GNUNET_TESTING_NotifyConnections cont;
 186
 187   /**
 188    * Closure for connection notification
 189    */
 190   void *cls;
 191 };
 192
 193 #if OLD
 194 struct PeerConnection
 195 {
 196   /*
 197    * Linked list
 198    */
 199   struct PeerConnection *next;
 200
 201   /*
 202    * Pointer to daemon handle
 203    */
 204   struct GNUNET_TESTING_Daemon *daemon;
 205
 206 };
 207 #endif
 208
 209 struct InternalStartContext
 210 {
 211   /**
 212    * Pointer to peerdata
 213    */
 214   struct PeerData *peer;
 215
 216   /**
 217    * Timeout for peer startup
 218    */
 219   struct GNUNET_TIME_Relative timeout;
 220
 221   /**
 222    * Client callback for hostkey notification
 223    */
 224   GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback;
 225
 226   /**
 227    * Closure for hostkey_callback
 228    */
 229   void *hostkey_cls;
 230
 231   /**
 232    * Client callback for peer start notification
 233    */
 234   GNUNET_TESTING_NotifyDaemonRunning start_cb;
 235
 236   /**
 237    * Closure for cb
 238    */
 239   void *start_cb_cls;
 240
 241   /**
 242    * Hostname, where to start the peer
 243    */
 244   const char *hostname;
 245
 246   /**
 247    * Username to use when connecting to the
 248    * host via ssh.
 249    */
 250   const char *username;
 251
 252   /**
 253    * Port to use for ssh.
 254    */
 255   uint16_t sshport;
 256
 257 };
 258
 259 /**
 260  * Data we keep per peer.
 261  */
 262 struct PeerData
 263 {
 264   /**
 265    * (Initial) configuration of the host.
 266    * (initial because clients could change
 267    *  it and we would not know about those
 268    *  updates).
 269    */
 270   struct GNUNET_CONFIGURATION_Handle *cfg;
 271
 272   /**
 273    * Handle for controlling the daemon.
 274    */
 275   struct GNUNET_TESTING_Daemon *daemon;
 276
 277   /**
 278    * The peergroup this peer belongs to.
 279    */
 280   struct GNUNET_TESTING_PeerGroup *pg;
 281
 282   /**
 283    * Hash map of allowed peer connections (F2F created topology)
 284    */
 285   struct GNUNET_CONTAINER_MultiHashMap *allowed_peers;
 286
 287   /**
 288    * Hash map of blacklisted peers
 289    */
 290   struct GNUNET_CONTAINER_MultiHashMap *blacklisted_peers;
 291
 292   /**
 293    * Hash map of peer connections
 294    */
 295   struct GNUNET_CONTAINER_MultiHashMap *connect_peers;
 296
 297   /**
 298    * Temporary hash map of peer connections
 299    */
 300   struct GNUNET_CONTAINER_MultiHashMap *connect_peers_working_set;
 301
 302   /**
 303    * Temporary variable for topology creation, should be reset before
 304    * creating any topology so the count is valid once finished.
 305    */
 306   int num_connections;
 307
 308   /**
 309    * Context to keep track of peers being started, to
 310    * stagger hostkey generation and peer startup.
 311    */
 312   struct InternalStartContext internal_context;
 313 };
 314
 315
 316 /**
 317  * Data we keep per host.
 318  */
 319 struct HostData
 320 {
 321   /**
 322    * Name of the host.
 323    */
 324   char *hostname;
 325
 326   /**
 327    * SSH username to use when connecting to this host.
 328    */
 329   char *username;
 330
 331   /**
 332    * SSH port to use when connecting to this host.
 333    */
 334   uint16_t sshport;
 335
 336   /**
 337    * Lowest port that we have not yet used
 338    * for GNUnet.
 339    */
 340   uint16_t minport;
 341 };
 342
 343 struct TopologyIterateContext
 344 {
 345   /**
 346    * Callback for notifying of two connected peers.
 347    */
 348   GNUNET_TESTING_NotifyTopology topology_cb;
 349
 350   /**
 351    * Closure for topology_cb
 352    */
 353   void *cls;
 354
 355   /**
 356    * Number of peers currently connected to.
 357    */
 358   unsigned int connected;
 359
 360   /**
 361    * Number of peers we have finished iterating.
 362    */
 363   unsigned int completed;
 364
 365   /**
 366    * Number of peers total.
 367    */
 368   unsigned int total;
 369 };
 370
 371 struct StatsIterateContext
 372 {
 373   /**
 374    * Continuation to call once all stats information has been retrieved.
 375    */
 376   GNUNET_STATISTICS_Callback cont;
 377
 378   /**
 379    * Proc function to call on each value received.
 380    */
 381   GNUNET_TESTING_STATISTICS_Iterator proc;
 382
 383   /**
 384    * Closure for topology_cb
 385    */
 386   void *cls;
 387
 388   /**
 389    * Number of peers currently connected to.
 390    */
 391   unsigned int connected;
 392
 393   /**
 394    * Number of peers we have finished iterating.
 395    */
 396   unsigned int completed;
 397
 398   /**
 399    * Number of peers total.
 400    */
 401   unsigned int total;
 402 };
 403
 404 struct CoreContext
 405 {
 406   void *iter_context;
 407   struct GNUNET_TESTING_Daemon *daemon;
 408 };
 409
 410 struct StatsCoreContext
 411 {
 412   void *iter_context;
 413   struct GNUNET_TESTING_Daemon *daemon;
 414   /**
 415    * Handle to the statistics service.
 416    */
 417   struct GNUNET_STATISTICS_Handle *stats_handle;
 418
 419   /**
 420    * Handle for getting statistics.
 421    */
 422   struct GNUNET_STATISTICS_GetHandle *stats_get_handle;
 423 };
 424
 425 /**
 426  * Handle to a group of GNUnet peers.
 427  */
 428 struct GNUNET_TESTING_PeerGroup
 429 {
 430   /**
 431    * Our scheduler.
 432    */
 433   struct GNUNET_SCHEDULER_Handle *sched;
 434
 435   /**
 436    * Configuration template.
 437    */
 438   const struct GNUNET_CONFIGURATION_Handle *cfg;
 439
 440   /**
 441    * Function to call on each started daemon.
 442    */
 443   //GNUNET_TESTING_NotifyDaemonRunning cb;
 444
 445   /**
 446    * Closure for cb.
 447    */
 448   //void *cb_cls;
 449
 450   /*
 451    * Function to call on each topology connection created
 452    */
 453   GNUNET_TESTING_NotifyConnection notify_connection;
 454
 455   /*
 456    * Callback for notify_connection
 457    */
 458   void *notify_connection_cls;
 459
 460   /**
 461    * NULL-terminated array of information about
 462    * hosts.
 463    */
 464   struct HostData *hosts;
 465
 466   /**
 467    * Array of "total" peers.
 468    */
 469   struct PeerData *peers;
 470
 471   /**
 472    * Number of peers in this group.
 473    */
 474   unsigned int total;
 475
 476   /**
 477    * At what time should we fail the peer startup process?
 478    */
 479   struct GNUNET_TIME_Absolute max_timeout;
 480
 481   /**
 482    * How many peers are being started right now?
 483    */
 484   unsigned int starting;
 485
 486   /**
 487    * How many peers have already been started?
 488    */
 489   unsigned int started;
 490 };
 491
 492 struct UpdateContext
 493 {
 494   struct GNUNET_CONFIGURATION_Handle *ret;
 495   const struct GNUNET_CONFIGURATION_Handle *orig;
 496   const char *hostname;
 497   unsigned int nport;
 498   unsigned int upnum;
 499   unsigned int fdnum;
 500 };
 501
 502
 503 struct ConnectContext
 504 {
 505   struct GNUNET_TESTING_Daemon *first;
 506
 507   struct GNUNET_TESTING_Daemon *second;
 508
 509   struct GNUNET_TESTING_PeerGroup *pg;
 510 };
 511
 512 /**
 513  * Convert unique ID to hash code.
 514  *
 515  * @param uid unique ID to convert
 516  * @param hash set to uid (extended with zeros)
 517  */
 518 static void
 519 hash_from_uid (uint32_t uid,
 520                GNUNET_HashCode *hash)
 521 {
 522   memset (hash, 0, sizeof(GNUNET_HashCode));
 523   *((uint32_t*)hash) = uid;
 524 }
 525
 526 /**
 527  * Convert hash code to unique ID.
 528  *
 529  * @param uid unique ID to convert
 530  * @param hash set to uid (extended with zeros)
 531  */
 532 static void
 533 uid_from_hash (const GNUNET_HashCode *hash, uint32_t *uid)
 534 {
 535   memcpy (uid, hash, sizeof(uint32_t));
 536 }
 537
 538 /**
 539  * Number of connects we are waiting on, allows us to rate limit
 540  * connect attempts.
 541  */
 542 static int outstanding_connects;
 543
 544 /**
 545  * Get a topology from a string input.
 546  *
 547  * @param topology where to write the retrieved topology
 548  * @param topology_string The string to attempt to
 549  *        get a configuration value from
 550  * @return GNUNET_YES if topology string matched a
 551  *         known topology, GNUNET_NO if not
 552  */
 553 int
 554 GNUNET_TESTING_topology_get(enum GNUNET_TESTING_Topology *topology, char * topology_string)
 555 {
 556   /**
 557    * Strings representing topologies in enum
 558    */
 559   static const char * topology_strings[] =
 560     {
 561       /**
 562        * A clique (everyone connected to everyone else).
 563        */
 564       "CLIQUE",
 565
 566       /**
 567        * Small-world network (2d torus plus random links).
 568        */
 569       "SMALL_WORLD",
 570
 571       /**
 572        * Small-world network (ring plus random links).
 573        */
 574       "SMALL_WORLD_RING",
 575
 576       /**
 577        * Ring topology.
 578        */
 579       "RING",
 580
 581       /**
 582        * 2-d torus.
 583        */
 584       "2D_TORUS",
 585
 586       /**
 587        * Random graph.
 588        */
 589       "ERDOS_RENYI",
 590
 591       /**
 592        * Certain percentage of peers are unable to communicate directly
 593        * replicating NAT conditions
 594        */
 595       "INTERNAT",
 596
 597       /**
 598        * Scale free topology.
 599        */
 600       "SCALE_FREE",
 601
 602       /**
 603        * Straight line topology.
 604        */
 605       "LINE",
 606
 607       /**
 608        * All peers are disconnected.
 609        */
 610       "NONE",
 611
 612       NULL
 613     };
 614
 615   int curr = 0;
 616   if (topology_string == NULL)
 617     return GNUNET_NO;
 618   while (topology_strings[curr] != NULL)
 619     {
 620       if (strcasecmp(topology_strings[curr], topology_string) == 0)
 621         {
 622           *topology = curr;
 623           return GNUNET_YES;
 624         }
 625       curr++;
 626     }
 627   *topology = GNUNET_TESTING_TOPOLOGY_NONE;
 628   return GNUNET_NO;
 629 }
 630
 631
 632 /**
 633  * Get connect topology option from string input.
 634  *
 635  * @param topology_option where to write the retrieved topology
 636  * @param topology_string The string to attempt to
 637  *        get a configuration value from
 638  * @return GNUNET_YES if string matched a known
 639  *         topology option, GNUNET_NO if not
 640  */
 641 int
 642 GNUNET_TESTING_topology_option_get (enum GNUNET_TESTING_TopologyOption *topology_option,
 643                                     char * topology_string)
 644 {
 645   /**
 646    * Options for connecting a topology as strings.
 647    */
 648   static const char * topology_option_strings[] =
 649     {
 650       /**
 651        * Try to connect all peers specified in the topology.
 652        */
 653       "CONNECT_ALL",
 654
 655       /**
 656        * Choose a random subset of connections to create.
 657        */
 658       "CONNECT_RANDOM_SUBSET",
 659
 660       /**
 661        * Create at least X connections for each peer.
 662        */
 663       "CONNECT_MINIMUM",
 664
 665       /**
 666        * Using a depth first search, create one connection
 667        * per peer.  If any are missed (graph disconnected)
 668        * start over at those peers until all have at least one
 669        * connection.
 670        */
 671       "CONNECT_DFS",
 672
 673       /**
 674        * No options specified.
 675        */
 676       "CONNECT_NONE",
 677
 678       NULL
 679     };
 680   int curr = 0;
 681
 682   if (topology_string == NULL)
 683     return GNUNET_NO;
 684   while (NULL != topology_option_strings[curr])
 685     {
 686       if (strcasecmp(topology_option_strings[curr], topology_string) == 0)
 687         {
 688           *topology_option = curr;
 689           return GNUNET_YES;
 690         }
 691       curr++;
 692     }
 693   *topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_NONE;
 694   return GNUNET_NO;
 695 }
 696
 697 /**
 698  * Function to iterate over options.  Copies
 699  * the options to the target configuration,
 700  * updating PORT values as needed.
 701  *
 702  * @param cls closure
 703  * @param section name of the section
 704  * @param option name of the option
 705  * @param value value of the option
 706  */
 707 static void
 708 update_config (void *cls,
 709                const char *section, const char *option, const char *value)
 710 {
 711   struct UpdateContext *ctx = cls;
 712   unsigned int ival;
 713   char cval[12];
 714   char uval[128];
 715   char *single_variable;
 716   char *per_host_variable;
 717   unsigned long long num_per_host;
 718
 719   if ((0 == strcmp (option, "PORT")) && (1 == sscanf (value, "%u", &ival)))
 720     {
 721       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 722       if ((ival != 0) && (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable)))
 723         {
 724           GNUNET_snprintf (cval, sizeof (cval), "%u", ctx->nport++);
 725           value = cval;
 726         }
 727
 728       GNUNET_free(single_variable);
 729     }
 730
 731   if (0 == strcmp (option, "UNIXPATH"))
 732     {
 733       GNUNET_asprintf(&single_variable, "single_%s_per_host", section);
 734       GNUNET_asprintf(&per_host_variable, "num_%s_per_host", section);
 735       if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_yesno(ctx->orig, "testing", single_variable))
 736         {
 737           GNUNET_snprintf (uval,
 738                            sizeof (uval),
 739                            "/tmp/test-service-%s-%u",
 740                            section,
 741                            ctx->upnum++);
 742           value = uval;
 743         }
 744       else if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number(ctx->orig, "testing", per_host_variable, &num_per_host))
 745         {
 746           GNUNET_snprintf (uval,
 747                            sizeof (uval),
 748                            "/tmp/test-service-%s-%u",
 749                            section,
 750                            ctx->fdnum % num_per_host);
 751           value = uval;
 752         }
 753       GNUNET_free(single_variable);
 754       GNUNET_free(per_host_variable);
 755
 756     }
 757
 758   if ((0 == strcmp (option, "HOSTNAME")) && (ctx->hostname != NULL))
 759     {
 760       value = ctx->hostname;
 761     }
 762
 763   GNUNET_CONFIGURATION_set_value_string (ctx->ret, section, option, value);
 764 }
 765
 766
 767 /**
 768  * Create a new configuration using the given configuration
 769  * as a template; however, each PORT in the existing cfg
 770  * must be renumbered by incrementing "*port".  If we run
 771  * out of "*port" numbers, return NULL.
 772  *
 773  * @param cfg template configuration
 774  * @param port port numbers to use, update to reflect
 775  *             port numbers that were used
 776  * @param upnum number to make unix domain socket names unique
 777  * @param hostname hostname of the controlling host, to allow control connections from
 778  * @param fdnum number used to offset the unix domain socket for grouped processes
 779  *              (such as statistics or peerinfo, which can be shared among others)
 780  *
 781  * @return new configuration, NULL on error
 782  */
 783 static struct GNUNET_CONFIGURATION_Handle *
 784 make_config (const struct GNUNET_CONFIGURATION_Handle *cfg,
 785              uint16_t * port,
 786              uint32_t * upnum,
 787              const char *hostname, uint32_t * fdnum)
 788 {
 789   struct UpdateContext uc;
 790   uint16_t orig;
 791   char *control_host;
 792   char *allowed_hosts;
 793
 794   orig = *port;
 795   uc.nport = *port;
 796   uc.upnum = *upnum;
 797   uc.fdnum = *fdnum;
 798   uc.ret = GNUNET_CONFIGURATION_create ();
 799   uc.hostname = hostname;
 800   uc.orig = cfg;
 801
 802   GNUNET_CONFIGURATION_iterate (cfg, &update_config, &uc);
 803   if (uc.nport >= HIGH_PORT)
 804     {
 805       *port = orig;
 806       GNUNET_CONFIGURATION_destroy (uc.ret);
 807       return NULL;
 808     }
 809
 810   if (GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "control_host", &control_host) == GNUNET_OK)
 811     {
 812       if (hostname != NULL)
 813         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1; %s;", control_host, hostname);
 814       else
 815         GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", control_host);
 816
 817       GNUNET_CONFIGURATION_set_value_string(uc.ret, "core", "ACCEPT_FROM", allowed_hosts);
 818       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport", "ACCEPT_FROM", allowed_hosts);
 819       GNUNET_CONFIGURATION_set_value_string(uc.ret, "dht", "ACCEPT_FROM", allowed_hosts);
 820       GNUNET_CONFIGURATION_set_value_string(uc.ret, "statistics", "ACCEPT_FROM", allowed_hosts);
 821       GNUNET_free_non_null(control_host);
 822       GNUNET_free(allowed_hosts);
 823     }
 824
 825
 826   /* arm needs to know to allow connections from the host on which it is running,
 827    * otherwise gnunet-arm is unable to connect to it in some instances */
 828   if (hostname != NULL)
 829     {
 830       GNUNET_asprintf(&allowed_hosts, "%s; 127.0.0.1;", hostname);
 831       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", hostname);
 832       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", hostname);
 833       GNUNET_CONFIGURATION_set_value_string(uc.ret, "arm", "ACCEPT_FROM", allowed_hosts);
 834       GNUNET_free(allowed_hosts);
 835     }
 836   else
 837     {
 838       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-tcp", "BINDTO", "127.0.0.1");
 839       GNUNET_CONFIGURATION_set_value_string(uc.ret, "transport-udp", "BINDTO", "127.0.0.1");
 840     }
 841
 842
 843   *port = (uint16_t) uc.nport;
 844   *upnum = uc.upnum;
 845   uc.fdnum++;
 846   *fdnum = uc.fdnum;
 847   return uc.ret;
 848 }
 849
 850
 851 /*
 852  * Add entries to the peers connect list
 853  *
 854  * @param pg the peer group we are working with
 855  * @param first index of the first peer
 856  * @param second index of the second peer
 857  *
 858  * @return the number of connections added
 859  *         technically should only be 0 or 2
 860  *
 861  */
 862 static int
 863 add_actual_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 864 {
 865   int added;
 866   int add_first;
 867   int add_second;
 868
 869   GNUNET_HashCode hash_first;
 870   GNUNET_HashCode hash_second;
 871
 872   hash_from_uid(first, &hash_first);
 873   hash_from_uid(second, &hash_second);
 874
 875   add_first = GNUNET_NO;
 876   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].connect_peers, &hash_second))
 877     {
 878       add_first = GNUNET_YES;
 879     }
 880
 881   add_second = GNUNET_NO;
 882   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].connect_peers, &hash_first))
 883     {
 884       add_second = GNUNET_YES;
 885     }
 886
 887   added = 0;
 888   if (add_first)
 889     {
 890       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].connect_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 891       pg->peers[first].num_connections++;
 892       added++;
 893     }
 894
 895   if (add_second)
 896     {
 897       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].connect_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 898       pg->peers[second].num_connections++;
 899       added++;
 900     }
 901
 902   return added;
 903 }
 904
 905
 906 /*
 907  * Add entries to the peers allowed connections list
 908  *
 909  * @param pg the peer group we are working with
 910  * @param first index of the first peer
 911  * @param second index of the second peer
 912  *
 913  * @return the number of connections added (can be 0, 1 or 2)
 914  *         technically should only be 0 or 2, but the small price
 915  *         of iterating over the lists (hashmaps in the future)
 916  *         for being sure doesn't bother me!
 917  *
 918  */
 919 static int
 920 add_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
 921 {
 922   int added;
 923 #if OLD
 924   struct PeerConnection *first_iter;
 925   struct PeerConnection *second_iter;
 926   struct PeerConnection *new_first;
 927   struct PeerConnection *new_second;
 928 #endif
 929   int add_first;
 930   int add_second;
 931
 932   GNUNET_HashCode hash_first;
 933   GNUNET_HashCode hash_second;
 934
 935   hash_from_uid(first, &hash_first);
 936   hash_from_uid(second, &hash_second);
 937
 938   add_first = GNUNET_NO;
 939   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].allowed_peers, &hash_second))
 940     {
 941       add_first = GNUNET_YES;
 942     }
 943
 944   add_second = GNUNET_NO;
 945   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].allowed_peers, &hash_first))
 946     {
 947       add_second = GNUNET_YES;
 948     }
 949 #if OLD
 950   first_iter = pg->peers[first].connected_peers;
 951   while (first_iter != NULL)
 952     {
 953       if (first_iter->daemon == pg->peers[second].daemon)
 954         add_first = GNUNET_NO;
 955       first_iter = first_iter->next;
 956     }
 957
 958   second_iter = pg->peers[second].connected_peers;
 959   add_second = GNUNET_YES;
 960   while (second_iter != NULL)
 961     {
 962       if (second_iter->daemon == pg->peers[first].daemon)
 963         add_second = GNUNET_NO;
 964       second_iter = second_iter->next;
 965     }
 966 #endif
 967
 968   added = 0;
 969   if (add_first)
 970     {
 971       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].allowed_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 972 #if OLD
 973       new_first = GNUNET_malloc(sizeof(struct PeerConnection));
 974       new_first->daemon = pg->peers[second].daemon;
 975       new_first->next = pg->peers[first].connected_peers;
 976       pg->peers[first].connected_peers = new_first;
 977 #endif
 978       pg->peers[first].num_connections++;
 979       added++;
 980     }
 981
 982   if (add_second)
 983     {
 984       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].allowed_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
 985 #if OLD
 986       new_second = GNUNET_malloc(sizeof(struct PeerConnection));
 987       new_second->daemon = pg->peers[first].daemon;
 988       new_second->next = pg->peers[second].connected_peers;
 989       pg->peers[second].connected_peers = new_second;
 990       pg->peers[first].num_connections++;
 991 #endif
 992       pg->peers[second].num_connections++;
 993       added++;
 994     }
 995
 996   return added;
 997 }
 998
 999 /*
1000  * Add entries to the peers blacklisted list
1001  *
1002  * @param pg the peer group we are working with
1003  * @param first index of the first peer
1004  * @param second index of the second peer
1005  *
1006  * @return the number of connections added (can be 0, 1 or 2)
1007  *
1008  */
1009 static int
1010 blacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1011 {
1012   int added;
1013   int add_first;
1014   int add_second;
1015   GNUNET_HashCode hash_first;
1016   GNUNET_HashCode hash_second;
1017
1018   hash_from_uid(first, &hash_first);
1019   hash_from_uid(second, &hash_second);
1020
1021   add_first = GNUNET_NO;
1022   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second))
1023     {
1024       add_first = GNUNET_YES;
1025     }
1026
1027   add_second = GNUNET_NO;
1028   if (GNUNET_NO == GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first))
1029     {
1030       add_second = GNUNET_YES;
1031     }
1032
1033   added = 0;
1034   if (add_first)
1035     {
1036       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1037       pg->peers[first].num_connections++;
1038       added++;
1039     }
1040
1041   if (add_second)
1042     {
1043       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
1044       pg->peers[second].num_connections++;
1045       added++;
1046     }
1047
1048   return added;
1049 }
1050
1051 /*
1052  * Remove entries from the peers blacklisted list
1053  *
1054  * @param pg the peer group we are working with
1055  * @param first index of the first peer
1056  * @param second index of the second peer
1057  *
1058  * @return the number of connections removed (can be 0, 1 or 2)
1059  *
1060  */
1061 static int
1062 unblacklist_connections(struct GNUNET_TESTING_PeerGroup *pg, unsigned int first, unsigned int second)
1063 {
1064   int removed;
1065   int remove_first;
1066   int remove_second;
1067   GNUNET_HashCode hash_first;
1068   GNUNET_HashCode hash_second;
1069
1070   hash_from_uid(first, &hash_first);
1071   hash_from_uid(second, &hash_second);
1072
1073   remove_first = GNUNET_CONTAINER_multihashmap_contains(pg->peers[first].blacklisted_peers, &hash_second);
1074   remove_second = GNUNET_CONTAINER_multihashmap_contains(pg->peers[second].blacklisted_peers, &hash_first);
1075
1076   removed = 0;
1077   if (remove_first)
1078     {
1079       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[first].blacklisted_peers, &hash_second, pg->peers[second].daemon));
1080       removed++;
1081     }
1082
1083   if (remove_second)
1084     {
1085       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[second].blacklisted_peers, &hash_first, pg->peers[first].daemon));
1086       removed++;
1087     }
1088
1089   return removed;
1090 }
1091
1092 /**
1093  * Scale free network construction as described in:
1094  *
1095  * "Emergence of Scaling in Random Networks." Science 286, 509-512, 1999.
1096  *
1097  * Start with a network of "one" peer, then progressively add
1098  * peers up to the total number.  At each step, iterate over
1099  * all possible peers and connect new peer based on number of
1100  * existing connections of the target peer.
1101  *
1102  * @param pg the peer group we are dealing with
1103  * @param proc the connection processor to use
1104  *
1105  * @return the number of connections created
1106  */
1107 static int
1108 create_scale_free (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1109 {
1110
1111   unsigned int total_connections;
1112   unsigned int outer_count;
1113   unsigned int i;
1114   unsigned int previous_total_connections;
1115   double random;
1116   double probability;
1117
1118   GNUNET_assert(pg->total > 1);
1119
1120   /* Add a connection between the first two nodes */
1121   total_connections = proc(pg, 0, 1);
1122
1123   for (outer_count = 1; outer_count < pg->total; outer_count++)
1124     {
1125       previous_total_connections = total_connections;
1126       for (i = 0; i < outer_count; i++)
1127         {
1128           probability = pg->peers[i].num_connections / (double)previous_total_connections;
1129           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1130                                                       UINT64_MAX)) / ( (double) UINT64_MAX);
1131 #if VERBOSE_TESTING
1132           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1133                       "Considering connecting peer %d to peer %d\n",
1134                       outer_count, i);
1135 #endif
1136           if (random < probability)
1137             {
1138 #if VERBOSE_TESTING
1139               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1140                           "Connecting peer %d to peer %d\n",
1141                           outer_count, i);
1142 #endif
1143               total_connections += proc(pg, outer_count, i);
1144             }
1145         }
1146     }
1147
1148   return total_connections;
1149 }
1150
1151 /**
1152  * Create a topology given a peer group (set of running peers)
1153  * and a connection processor.
1154  *
1155  * @param pg the peergroup to create the topology on
1156  * @param proc the connection processor to call to actually set
1157  *        up connections between two peers
1158  *
1159  * @return the number of connections that were set up
1160  *
1161  */
1162 int
1163 create_small_world_ring(struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1164 {
1165   unsigned int i, j;
1166   int nodeToConnect;
1167   unsigned int natLog;
1168   unsigned int randomPeer;
1169   double random, logNModifier, percentage;
1170   unsigned int smallWorldConnections;
1171   int connsPerPeer;
1172   char *p_string;
1173   int max;
1174   int min;
1175   unsigned int useAnd;
1176   int connect_attempts;
1177
1178   logNModifier = 0.5; /* FIXME: default value? */
1179   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1180                                                          "TESTING",
1181                                                          "LOGNMODIFIER",
1182                                                          &p_string))
1183     {
1184       if (sscanf(p_string, "%lf", &logNModifier) != 1)
1185         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1186                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1187                     p_string,
1188                     "LOGNMODIFIER",
1189                     "TESTING");
1190       GNUNET_free (p_string);
1191     }
1192   percentage = 0.5; /* FIXME: default percentage? */
1193   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1194                                                          "TESTING",
1195                                                          "PERCENTAGE",
1196                                                          &p_string))
1197     {
1198       if (sscanf(p_string, "%lf", &percentage) != 1)
1199         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1200                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1201                     p_string,
1202                     "PERCENTAGE",
1203                     "TESTING");
1204       GNUNET_free (p_string);
1205     }
1206   natLog = log (pg->total);
1207   connsPerPeer = ceil (natLog * logNModifier);
1208
1209   if (connsPerPeer % 2 == 1)
1210     connsPerPeer += 1;
1211
1212   smallWorldConnections = 0;
1213   connect_attempts = 0;
1214   for (i = 0; i < pg->total; i++)
1215     {
1216       useAnd = 0;
1217       max = i + connsPerPeer / 2;
1218       min = i - connsPerPeer / 2;
1219
1220       if (max > pg->total - 1)
1221         {
1222           max = max - pg->total;
1223           useAnd = 1;
1224         }
1225
1226       if (min < 0)
1227         {
1228           min = pg->total - 1 + min;
1229           useAnd = 1;
1230         }
1231
1232       for (j = 0; j < connsPerPeer / 2; j++)
1233         {
1234           random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1235                                                       UINT64_MAX) / ( (double) UINT64_MAX));
1236           if (random < percentage)
1237             {
1238               /* Connect to uniformly selected random peer */
1239               randomPeer =
1240                 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1241                                    pg->total);
1242               while ((((randomPeer < max) && (randomPeer > min))
1243                       && (useAnd == 0)) || (((randomPeer > min)
1244                                              || (randomPeer < max))
1245                                             && (useAnd == 1)))
1246                 {
1247                   randomPeer =
1248                       GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1249                                                          pg->total);
1250                 }
1251               smallWorldConnections +=
1252                 proc (pg, i, randomPeer);
1253             }
1254           else
1255             {
1256               nodeToConnect = i + j + 1;
1257               if (nodeToConnect > pg->total - 1)
1258                 {
1259                   nodeToConnect = nodeToConnect - pg->total;
1260                 }
1261               connect_attempts +=
1262                 proc (pg, i, nodeToConnect);
1263             }
1264         }
1265
1266     }
1267
1268   connect_attempts += smallWorldConnections;
1269
1270   return connect_attempts;
1271 }
1272
1273 /**
1274  * Create a topology given a peer group (set of running peers)
1275  * and a connection processor.
1276  *
1277  * @param pg the peergroup to create the topology on
1278  * @param proc the connection processor to call to actually set
1279  *        up connections between two peers
1280  *
1281  * @return the number of connections that were set up
1282  *
1283  */
1284 static int
1285 create_nated_internet (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1286 {
1287   unsigned int outer_count, inner_count;
1288   unsigned int cutoff;
1289   int connect_attempts;
1290   double nat_percentage;
1291   char *p_string;
1292
1293   nat_percentage = 0.6; /* FIXME: default percentage? */
1294   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1295                                                          "TESTING",
1296                                                          "NATPERCENTAGE",
1297                                                          &p_string))
1298     {
1299       if (sscanf(p_string, "%lf", &nat_percentage) != 1)
1300         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1301                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1302                     p_string,
1303                     "NATPERCENTAGE",
1304                     "TESTING");
1305       GNUNET_free (p_string);
1306     }
1307
1308
1309
1310   cutoff = (unsigned int) (nat_percentage * pg->total);
1311
1312   connect_attempts = 0;
1313
1314   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1315     {
1316       for (inner_count = outer_count + 1; inner_count < pg->total;
1317            inner_count++)
1318         {
1319           if ((outer_count > cutoff) || (inner_count > cutoff))
1320             {
1321 #if VERBOSE_TESTING
1322               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1323                           "Connecting peer %d to peer %d\n",
1324                           outer_count, inner_count);
1325 #endif
1326               connect_attempts += proc(pg, outer_count, inner_count);
1327             }
1328         }
1329     }
1330
1331   return connect_attempts;
1332
1333 }
1334
1335 /**
1336  * Create a topology given a peer group (set of running peers)
1337  * and a connection processor.
1338  *
1339  * @param pg the peergroup to create the topology on
1340  * @param proc the connection processor to call to actually set
1341  *        up connections between two peers
1342  *
1343  * @return the number of connections that were set up
1344  *
1345  */
1346 static int
1347 create_small_world (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1348 {
1349   unsigned int i, j, k;
1350   unsigned int square;
1351   unsigned int rows;
1352   unsigned int cols;
1353   unsigned int toggle = 1;
1354   unsigned int nodeToConnect;
1355   unsigned int natLog;
1356   unsigned int node1Row;
1357   unsigned int node1Col;
1358   unsigned int node2Row;
1359   unsigned int node2Col;
1360   unsigned int distance;
1361   double probability, random, percentage;
1362   unsigned int smallWorldConnections;
1363   unsigned int small_world_it;
1364   char *p_string;
1365   int connect_attempts;
1366   square = floor (sqrt (pg->total));
1367   rows = square;
1368   cols = square;
1369
1370   percentage = 0.5; /* FIXME: default percentage? */
1371   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1372                                                          "TESTING",
1373                                                          "PERCENTAGE",
1374                                                          &p_string))
1375     {
1376       if (sscanf(p_string, "%lf", &percentage) != 1)
1377         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1378                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1379                     p_string,
1380                     "PERCENTAGE",
1381                     "TESTING");
1382       GNUNET_free (p_string);
1383     }
1384   if (percentage < 0.0)
1385     {
1386       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1387                   _("Invalid value `%s' for option `%s' in section `%s': got %f, needed value greater than 0\n"),
1388                   "PERCENTAGE", "TESTING", percentage);
1389       percentage = 0.5;
1390     }
1391   probability = 0.5; /* FIXME: default percentage? */
1392   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1393                                                          "TESTING",
1394                                                          "PROBABILITY",
1395                                                          &p_string))
1396     {
1397       if (sscanf(p_string, "%lf", &probability) != 1)
1398         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1399                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1400                     p_string,
1401                     "PROBABILITY",
1402                     "TESTING");
1403       GNUNET_free (p_string);
1404     }
1405   if (square * square != pg->total)
1406     {
1407       while (rows * cols < pg->total)
1408         {
1409           if (toggle % 2 == 0)
1410             rows++;
1411           else
1412             cols++;
1413
1414           toggle++;
1415         }
1416     }
1417 #if VERBOSE_TESTING
1418       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1419                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1420                   rows, cols);
1421 #endif
1422
1423   connect_attempts = 0;
1424   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1425    * to the node to its right and above.  Once this is over, we'll have our torus!
1426    * Special case for the last node (if the rows and columns are not equal), connect
1427    * to the first in the row to maintain topology.
1428    */
1429   for (i = 0; i < pg->total; i++)
1430     {
1431       /* First connect to the node to the right */
1432       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1433         nodeToConnect = i + 1;
1434       else if (i + 1 == pg->total)
1435         nodeToConnect = rows * cols - cols;
1436       else
1437         nodeToConnect = i - cols + 1;
1438
1439       connect_attempts += proc (pg, i, nodeToConnect);
1440
1441       if (i < cols)
1442         nodeToConnect = (rows * cols) - cols + i;
1443       else
1444         nodeToConnect = i - cols;
1445
1446       if (nodeToConnect < pg->total)
1447         connect_attempts += proc (pg, i, nodeToConnect);
1448     }
1449   natLog = log (pg->total);
1450 #if VERBOSE_TESTING > 2
1451   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1452               _("natural log of %d is %d, will run %d iterations\n"),
1453              pg->total, natLog, (int) (natLog * percentage));
1454   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, _("Total connections added thus far: %u!\n"), connect_attempts);
1455 #endif
1456   smallWorldConnections = 0;
1457   small_world_it = (unsigned int)(natLog * percentage);
1458   GNUNET_assert(small_world_it > 0 && small_world_it < (unsigned int)-1);
1459   for (i = 0; i < small_world_it; i++)
1460     {
1461       for (j = 0; j < pg->total; j++)
1462         {
1463           /* Determine the row and column of node at position j on the 2d torus */
1464           node1Row = j / cols;
1465           node1Col = j - (node1Row * cols);
1466           for (k = 0; k < pg->total; k++)
1467             {
1468               /* Determine the row and column of node at position k on the 2d torus */
1469               node2Row = k / cols;
1470               node2Col = k - (node2Row * cols);
1471               /* Simple Cartesian distance */
1472               distance = abs (node1Row - node2Row) + abs (node1Col - node2Col);
1473               if (distance > 1)
1474                 {
1475                   /* Calculate probability as 1 over the square of the distance */
1476                   probability = 1.0 / (distance * distance);
1477                   /* Choose a random value between 0 and 1 */
1478                   random = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1479                                                               UINT64_MAX)) / ( (double) UINT64_MAX);
1480                   /* If random < probability, then connect the two nodes */
1481                   if (random < probability)
1482                     smallWorldConnections += proc (pg, j, k);
1483
1484                 }
1485             }
1486         }
1487     }
1488   connect_attempts += smallWorldConnections;
1489 #if VERBOSE_TESTING > 2
1490           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1491                       _("Total connections added for small world: %d!\n"),
1492                       smallWorldConnections);
1493 #endif
1494   return connect_attempts;
1495 }
1496
1497 /**
1498  * Create a topology given a peer group (set of running peers)
1499  * and a connection processor.
1500  *
1501  * @param pg the peergroup to create the topology on
1502  * @param proc the connection processor to call to actually set
1503  *        up connections between two peers
1504  *
1505  * @return the number of connections that were set up
1506  *
1507  */
1508 static int
1509 create_erdos_renyi (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1510 {
1511   double temp_rand;
1512   unsigned int outer_count;
1513   unsigned int inner_count;
1514   int connect_attempts;
1515   double probability;
1516   char *p_string;
1517
1518   probability = 0.5; /* FIXME: default percentage? */
1519   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string(pg->cfg,
1520                                                          "TESTING",
1521                                                          "PROBABILITY",
1522                                                          &p_string))
1523     {
1524       if (sscanf(p_string, "%lf", &probability) != 1)
1525         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1526                     _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
1527                     p_string,
1528                     "PROBABILITY",
1529                     "TESTING");
1530       GNUNET_free (p_string);
1531     }
1532   connect_attempts = 0;
1533   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1534     {
1535       for (inner_count = outer_count + 1; inner_count < pg->total;
1536            inner_count++)
1537         {
1538           temp_rand = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
1539                                                          UINT64_MAX)) / ( (double) UINT64_MAX);
1540 #if VERBOSE_TESTING
1541           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1542                       _("rand is %f probability is %f\n"), temp_rand,
1543                       probability);
1544 #endif
1545           if (temp_rand < probability)
1546             {
1547               connect_attempts += proc (pg, outer_count, inner_count);
1548             }
1549         }
1550     }
1551
1552   return connect_attempts;
1553 }
1554
1555 /**
1556  * Create a topology given a peer group (set of running peers)
1557  * and a connection processor.
1558  *
1559  * @param pg the peergroup to create the topology on
1560  * @param proc the connection processor to call to actually set
1561  *        up connections between two peers
1562  *
1563  * @return the number of connections that were set up
1564  *
1565  */
1566 static int
1567 create_2d_torus (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1568 {
1569   unsigned int i;
1570   unsigned int square;
1571   unsigned int rows;
1572   unsigned int cols;
1573   unsigned int toggle = 1;
1574   unsigned int nodeToConnect;
1575   int connect_attempts;
1576
1577   connect_attempts = 0;
1578
1579   square = floor (sqrt (pg->total));
1580   rows = square;
1581   cols = square;
1582
1583   if (square * square != pg->total)
1584     {
1585       while (rows * cols < pg->total)
1586         {
1587           if (toggle % 2 == 0)
1588             rows++;
1589           else
1590             cols++;
1591
1592           toggle++;
1593         }
1594     }
1595 #if VERBOSE_TESTING
1596       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1597                   _("Connecting nodes in 2d torus topology: %u rows %u columns\n"),
1598                   rows, cols);
1599 #endif
1600   /* Rows and columns are all sorted out, now iterate over all nodes and connect each
1601    * to the node to its right and above.  Once this is over, we'll have our torus!
1602    * Special case for the last node (if the rows and columns are not equal), connect
1603    * to the first in the row to maintain topology.
1604    */
1605   for (i = 0; i < pg->total; i++)
1606     {
1607       /* First connect to the node to the right */
1608       if (((i + 1) % cols != 0) && (i + 1 != pg->total))
1609         nodeToConnect = i + 1;
1610       else if (i + 1 == pg->total)
1611         nodeToConnect = rows * cols - cols;
1612       else
1613         nodeToConnect = i - cols + 1;
1614 #if VERBOSE_TESTING
1615           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1616                       "Connecting peer %d to peer %d\n",
1617                       i, nodeToConnect);
1618 #endif
1619       connect_attempts += proc(pg, i, nodeToConnect);
1620
1621       /* Second connect to the node immediately above */
1622       if (i < cols)
1623         nodeToConnect = (rows * cols) - cols + i;
1624       else
1625         nodeToConnect = i - cols;
1626
1627       if (nodeToConnect < pg->total)
1628         {
1629 #if VERBOSE_TESTING
1630           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1631                       "Connecting peer %d to peer %d\n",
1632                       i, nodeToConnect);
1633 #endif
1634           connect_attempts += proc(pg, i, nodeToConnect);
1635         }
1636
1637     }
1638
1639   return connect_attempts;
1640 }
1641
1642
1643 /**
1644  * Create a topology given a peer group (set of running peers)
1645  * and a connection processor.
1646  *
1647  * @param pg the peergroup to create the topology on
1648  * @param proc the connection processor to call to actually set
1649  *        up connections between two peers
1650  *
1651  * @return the number of connections that were set up
1652  *
1653  */
1654 static int
1655 create_clique (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1656 {
1657   unsigned int outer_count;
1658   unsigned int inner_count;
1659   int connect_attempts;
1660
1661   connect_attempts = 0;
1662
1663   for (outer_count = 0; outer_count < pg->total - 1; outer_count++)
1664     {
1665       for (inner_count = outer_count + 1; inner_count < pg->total;
1666            inner_count++)
1667         {
1668 #if VERBOSE_TESTING
1669           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1670                       "Connecting peer %d to peer %d\n",
1671                       outer_count, inner_count);
1672 #endif
1673           connect_attempts += proc(pg, outer_count, inner_count);
1674         }
1675     }
1676
1677   return connect_attempts;
1678 }
1679
1680 /**
1681  * Create a topology given a peer group (set of running peers)
1682  * and a connection processor.
1683  *
1684  * @param pg the peergroup to create the topology on
1685  * @param proc the connection processor to call to actually set
1686  *        up connections between two peers
1687  *
1688  * @return the number of connections that were set up
1689  *
1690  */
1691 static int
1692 create_line (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1693 {
1694   unsigned int count;
1695   int connect_attempts;
1696
1697   connect_attempts = 0;
1698
1699   /* Connect each peer to the next highest numbered peer */
1700   for (count = 0; count < pg->total - 1; count++)
1701     {
1702 #if VERBOSE_TESTING
1703           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1704                       "Connecting peer %d to peer %d\n",
1705                       count, count + 1);
1706 #endif
1707       connect_attempts += proc(pg, count, count + 1);
1708     }
1709
1710   return connect_attempts;
1711 }
1712
1713 /**
1714  * Create a topology given a peer group (set of running peers)
1715  * and a connection processor.
1716  *
1717  * @param pg the peergroup to create the topology on
1718  * @param proc the connection processor to call to actually set
1719  *        up connections between two peers
1720  *
1721  * @return the number of connections that were set up
1722  *
1723  */
1724 static int
1725 create_ring (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_ConnectionProcessor proc)
1726 {
1727   unsigned int count;
1728   int connect_attempts;
1729
1730   connect_attempts = 0;
1731
1732   /* Connect each peer to the next highest numbered peer */
1733   for (count = 0; count < pg->total - 1; count++)
1734     {
1735 #if VERBOSE_TESTING
1736           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1737                       "Connecting peer %d to peer %d\n",
1738                       count, count + 1);
1739 #endif
1740       connect_attempts += proc(pg, count, count + 1);
1741     }
1742
1743   /* Connect the last peer to the first peer */
1744   connect_attempts += proc(pg, pg->total - 1, 0);
1745
1746   return connect_attempts;
1747 }
1748
1749
1750 /**
1751  * Iterator for writing friends of a peer to a file.
1752  *
1753  * @param cls closure, an open writable file handle
1754  * @param key the key the daemon was stored under
1755  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1756  *
1757  * @return GNUNET_YES to continue iteration
1758  *
1759  * TODO: Could replace friend_file_iterator and blacklist_file_iterator
1760  *       with a single file_iterator that takes a closure which contains
1761  *       the prefix to write before the peer.  Then this could be used
1762  *       for blacklisting multiple transports and writing the friend
1763  *       file.  I'm sure *someone* will complain loudly about other
1764  *       things that negate these functions even existing so no point in
1765  *       "fixing" now.
1766  */
1767 static int
1768 friend_file_iterator (void *cls,
1769                   const GNUNET_HashCode * key,
1770                   void *value)
1771 {
1772   FILE *temp_friend_handle = cls;
1773   struct GNUNET_TESTING_Daemon *peer = value;
1774   struct GNUNET_PeerIdentity *temppeer;
1775   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1776
1777   temppeer = &peer->id;
1778   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1779   fprintf(temp_friend_handle, "%s\n", (char *)&peer_enc);
1780
1781   return GNUNET_YES;
1782 }
1783
1784 struct BlacklistContext
1785 {
1786   /*
1787    * The (open) file handle to write to
1788    */
1789   FILE *temp_file_handle;
1790
1791   /*
1792    * The transport that this peer will be blacklisted on.
1793    */
1794   char *transport;
1795 };
1796
1797 /**
1798  * Iterator for writing blacklist data to appropriate files.
1799  *
1800  * @param cls closure, an open writable file handle
1801  * @param key the key the daemon was stored under
1802  * @param value the GNUNET_TESTING_Daemon that needs to be written.
1803  *
1804  * @return GNUNET_YES to continue iteration
1805  */
1806 static int
1807 blacklist_file_iterator (void *cls,
1808                          const GNUNET_HashCode * key,
1809                          void *value)
1810 {
1811   struct BlacklistContext *blacklist_ctx = cls;
1812   //FILE *temp_blacklist_handle = cls;
1813   struct GNUNET_TESTING_Daemon *peer = value;
1814   struct GNUNET_PeerIdentity *temppeer;
1815   struct GNUNET_CRYPTO_HashAsciiEncoded peer_enc;
1816
1817   temppeer = &peer->id;
1818   GNUNET_CRYPTO_hash_to_enc(&temppeer->hashPubKey, &peer_enc);
1819   fprintf(blacklist_ctx->temp_file_handle, "%s:%s\n", blacklist_ctx->transport, (char *)&peer_enc);
1820
1821   return GNUNET_YES;
1822 }
1823
1824 /*
1825  * Create the friend files based on the PeerConnection's
1826  * of each peer in the peer group, and copy the files
1827  * to the appropriate place
1828  *
1829  * @param pg the peer group we are dealing with
1830  */
1831 static int
1832 create_and_copy_friend_files (struct GNUNET_TESTING_PeerGroup *pg)
1833 {
1834   FILE *temp_friend_handle;
1835   unsigned int pg_iter;
1836   char *temp_service_path;
1837   pid_t *pidarr;
1838   char *arg;
1839   char * mytemp;
1840   enum GNUNET_OS_ProcessStatusType type;
1841   unsigned long return_code;
1842   int count;
1843   int ret;
1844   int max_wait = 10;
1845
1846   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1847   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1848     {
1849       mytemp = GNUNET_DISK_mktemp("friends");
1850       GNUNET_assert(mytemp != NULL);
1851       temp_friend_handle = fopen (mytemp, "wt");
1852       GNUNET_assert(temp_friend_handle != NULL);
1853       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &friend_file_iterator, temp_friend_handle);
1854       fclose(temp_friend_handle);
1855
1856       if (GNUNET_OK !=
1857           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
1858         {
1859           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1860                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
1861                       "SERVICEHOME",
1862                       "PATHS");
1863           if (UNLINK (mytemp) != 0)
1864             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
1865           GNUNET_free (mytemp);
1866           break;
1867         }
1868
1869       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
1870         {
1871           GNUNET_asprintf (&arg, "%s/friends", temp_service_path);
1872           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
1873                                          "mv", mytemp, arg, NULL);
1874 #if VERBOSE_TESTING
1875           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1876                       _("Copying file with command cp %s %s\n"), mytemp, arg);
1877 #endif
1878
1879           GNUNET_free(arg);
1880         }
1881       else /* Remote, scp the file to the correct place */
1882         {
1883           if (NULL != pg->peers[pg_iter].daemon->username)
1884             GNUNET_asprintf (&arg, "%s@%s:%s/friends", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
1885           else
1886             GNUNET_asprintf (&arg, "%s:%s/friends", pg->peers[pg_iter].daemon->hostname, temp_service_path);
1887           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
1888                                          "scp", mytemp, arg, NULL);
1889
1890 #if VERBOSE_TESTING
1891           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1892                       _("Copying file with command scp %s %s\n"), mytemp, arg);
1893 #endif
1894           GNUNET_free(arg);
1895         }
1896       GNUNET_free (temp_service_path);
1897       GNUNET_free (mytemp);
1898     }
1899
1900   count = 0;
1901   ret = GNUNET_SYSERR;
1902   while ((count < max_wait) && (ret != GNUNET_OK))
1903     {
1904       ret = GNUNET_OK;
1905       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1906         {
1907 #if VERBOSE_TESTING
1908           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1909                       _("Checking copy status of file %d\n"), pg_iter);
1910 #endif
1911           if (pidarr[pg_iter] != 0) /* Check for already completed! */
1912             {
1913               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
1914                 {
1915                   ret = GNUNET_SYSERR;
1916                 }
1917               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
1918                 {
1919                   ret = GNUNET_SYSERR;
1920                 }
1921               else
1922                 {
1923                   pidarr[pg_iter] = 0;
1924 #if VERBOSE_TESTING
1925             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1926                       _("File %d copied\n"), pg_iter);
1927 #endif
1928                 }
1929             }
1930         }
1931       count++;
1932       if (ret == GNUNET_SYSERR)
1933         {
1934           /* FIXME: why sleep here? -CG */
1935           sleep(1);
1936         }
1937     }
1938
1939 #if VERBOSE_TESTING
1940     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1941                 _("Finished copying all friend files!\n"));
1942 #endif
1943   GNUNET_free(pidarr);
1944   return ret;
1945 }
1946
1947
1948 /*
1949  * Create the blacklist files based on the PeerConnection's
1950  * of each peer in the peer group, and copy the files
1951  * to the appropriate place.
1952  *
1953  * @param pg the peer group we are dealing with
1954  * @param transports space delimited list of transports to blacklist
1955  */
1956 static int
1957 create_and_copy_blacklist_files (struct GNUNET_TESTING_PeerGroup *pg, char *transports)
1958 {
1959   FILE *temp_file_handle;
1960   static struct BlacklistContext blacklist_ctx;
1961   unsigned int pg_iter;
1962   char *temp_service_path;
1963   pid_t *pidarr;
1964   char *arg;
1965   char *mytemp;
1966   enum GNUNET_OS_ProcessStatusType type;
1967   unsigned long return_code;
1968   int count;
1969   int ret;
1970   int max_wait = 10;
1971   int transport_len;
1972   unsigned int i;
1973   char *pos;
1974   char *temp_transports;
1975
1976   pidarr = GNUNET_malloc(sizeof(pid_t) * pg->total);
1977   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
1978     {
1979       mytemp = GNUNET_DISK_mktemp("blacklist");
1980       GNUNET_assert(mytemp != NULL);
1981       temp_file_handle = fopen (mytemp, "wt");
1982       GNUNET_assert(temp_file_handle != NULL);
1983       temp_transports = GNUNET_strdup(transports);
1984       blacklist_ctx.temp_file_handle = temp_file_handle;
1985       transport_len = strlen(temp_transports) + 1;
1986       pos = NULL;
1987
1988       for (i = 0; i < transport_len; i++)
1989       {
1990         if ((temp_transports[i] == ' ') && (pos == NULL))
1991           continue; /* At start of string (whitespace) */
1992         else if ((temp_transports[i] == ' ') || (temp_transports[i] == '\0')) /* At end of string */
1993         {
1994           temp_transports[i] = '\0';
1995           blacklist_ctx.transport = pos;
1996           GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].blacklisted_peers, &blacklist_file_iterator, &blacklist_ctx);
1997           pos = NULL;
1998         } /* At beginning of actual string */
1999         else if (pos == NULL)
2000         {
2001           pos = &temp_transports[i];
2002         }
2003       }
2004
2005       GNUNET_free (temp_transports);
2006       fclose(temp_file_handle);
2007
2008       if (GNUNET_OK !=
2009           GNUNET_CONFIGURATION_get_value_string(pg->peers[pg_iter].daemon->cfg, "PATHS", "SERVICEHOME", &temp_service_path))
2010         {
2011           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2012                       _("No `%s' specified in peer configuration in section `%s', cannot copy friends file!\n"),
2013                       "SERVICEHOME",
2014                       "PATHS");
2015           if (UNLINK (mytemp) != 0)
2016             GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_WARNING, "unlink", mytemp);
2017           GNUNET_free (mytemp);
2018           break;
2019         }
2020
2021       if (pg->peers[pg_iter].daemon->hostname == NULL) /* Local, just copy the file */
2022         {
2023           GNUNET_asprintf (&arg, "%s/blacklist", temp_service_path);
2024           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "mv",
2025                                          "mv", mytemp, arg, NULL);
2026 #if VERBOSE_TESTING
2027           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2028                       _("Copying file with command cp %s %s\n"), mytemp, arg);
2029 #endif
2030
2031           GNUNET_free(arg);
2032         }
2033       else /* Remote, scp the file to the correct place */
2034         {
2035           if (NULL != pg->peers[pg_iter].daemon->username)
2036             GNUNET_asprintf (&arg, "%s@%s:%s/blacklist", pg->peers[pg_iter].daemon->username, pg->peers[pg_iter].daemon->hostname, temp_service_path);
2037           else
2038             GNUNET_asprintf (&arg, "%s:%s/blacklist", pg->peers[pg_iter].daemon->hostname, temp_service_path);
2039           pidarr[pg_iter] = GNUNET_OS_start_process (NULL, NULL, "scp",
2040                                          "scp", mytemp, arg, NULL);
2041
2042 #if VERBOSE_TESTING
2043           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2044                       _("Copying file with command scp %s %s\n"), mytemp, arg);
2045 #endif
2046           GNUNET_free(arg);
2047         }
2048       GNUNET_free (temp_service_path);
2049       GNUNET_free (mytemp);
2050     }
2051
2052   count = 0;
2053   ret = GNUNET_SYSERR;
2054   while ((count < max_wait) && (ret != GNUNET_OK))
2055     {
2056       ret = GNUNET_OK;
2057       for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2058         {
2059 #if VERBOSE_TESTING
2060           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2061                       _("Checking copy status of file %d\n"), pg_iter);
2062 #endif
2063           if (pidarr[pg_iter] != 0) /* Check for already completed! */
2064             {
2065               if (GNUNET_OS_process_status(pidarr[pg_iter], &type, &return_code) != GNUNET_OK)
2066                 {
2067                   ret = GNUNET_SYSERR;
2068                 }
2069               else if ((type != GNUNET_OS_PROCESS_EXITED) || (return_code != 0))
2070                 {
2071                   ret = GNUNET_SYSERR;
2072                 }
2073               else
2074                 {
2075                   pidarr[pg_iter] = 0;
2076 #if VERBOSE_TESTING
2077             GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2078                       _("File %d copied\n"), pg_iter);
2079 #endif
2080                 }
2081             }
2082         }
2083       count++;
2084       if (ret == GNUNET_SYSERR)
2085         {
2086           /* FIXME: why sleep here? -CG */
2087           sleep(1);
2088         }
2089     }
2090
2091 #if VERBOSE_TESTING
2092     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2093                 _("Finished copying all blacklist files!\n"));
2094 #endif
2095   GNUNET_free(pidarr);
2096   return ret;
2097 }
2098
2099
2100 /**
2101  * Internal notification of a connection, kept so that we can ensure some connections
2102  * happen instead of flooding all testing daemons with requests to connect.
2103  */
2104 static void internal_connect_notify (void *cls,
2105                                      const struct GNUNET_PeerIdentity *first,
2106                                      const struct GNUNET_PeerIdentity *second,
2107                                      uint32_t distance,
2108                                      const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2109                                      const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2110                                      struct GNUNET_TESTING_Daemon *first_daemon,
2111                                      struct GNUNET_TESTING_Daemon *second_daemon,
2112                                      const char *emsg)
2113 {
2114   struct GNUNET_TESTING_PeerGroup *pg = cls;
2115   outstanding_connects--;
2116
2117   pg->notify_connection(pg->notify_connection_cls, first, second, distance, first_cfg, second_cfg, first_daemon, second_daemon, emsg);
2118 }
2119
2120
2121 /**
2122  * Either delay a connection (because there are too many outstanding)
2123  * or schedule it for right now.
2124  *
2125  * @param cls a connection context
2126  * @param tc the task runtime context
2127  */
2128 static void schedule_connect(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2129 {
2130   struct ConnectContext *connect_context = cls;
2131
2132   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2133     return;
2134
2135   if (outstanding_connects > MAX_OUTSTANDING_CONNECTIONS)
2136     {
2137 #if VERBOSE_TESTING > 2
2138           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2139                       _("Delaying connect, we have too many outstanding connections!\n"));
2140 #endif
2141       GNUNET_SCHEDULER_add_delayed(connect_context->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_connect, connect_context);
2142     }
2143   else
2144     {
2145 #if VERBOSE_TESTING > 2
2146           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2147                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2148 #endif
2149       outstanding_connects++;
2150       GNUNET_TESTING_daemons_connect (connect_context->first,
2151                                       connect_context->second,
2152                                       CONNECT_TIMEOUT,
2153                                       CONNECT_ATTEMPTS,
2154                                       &internal_connect_notify,
2155                                       connect_context->pg);
2156       GNUNET_free(connect_context);
2157     }
2158 }
2159
2160
2161 /**
2162  * Iterator for actually scheduling connections to be created
2163  * between two peers.
2164  *
2165  * @param cls closure, a GNUNET_TESTING_Daemon
2166  * @param key the key the second Daemon was stored under
2167  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2168  *
2169  * @return GNUNET_YES to continue iteration
2170  */
2171 static int
2172 connect_iterator (void *cls,
2173                   const GNUNET_HashCode * key,
2174                   void *value)
2175 {
2176   struct PeerData *first = cls;
2177   struct GNUNET_TESTING_Daemon *second = value;
2178   struct ConnectContext *connect_context;
2179
2180   connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2181   connect_context->pg = first->pg;
2182   connect_context->first = first->daemon;
2183   connect_context->second = second;
2184   GNUNET_SCHEDULER_add_now(first->pg->sched, &schedule_connect, connect_context);
2185
2186   return GNUNET_YES;
2187 }
2188
2189
2190 /**
2191  * Iterator for copying all entries in the allowed hashmap to the
2192  * connect hashmap.
2193  *
2194  * @param cls closure, a GNUNET_TESTING_Daemon
2195  * @param key the key the second Daemon was stored under
2196  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2197  *
2198  * @return GNUNET_YES to continue iteration
2199  */
2200 static int
2201 copy_topology_iterator (void *cls,
2202                   const GNUNET_HashCode * key,
2203                   void *value)
2204 {
2205   struct PeerData *first = cls;
2206
2207   GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(first->connect_peers, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2208
2209   return GNUNET_YES;
2210 }
2211
2212 /**
2213  * Make the peers to connect the same as those that are allowed to be
2214  * connected.
2215  *
2216  * @param pg the peer group
2217  */
2218 static int
2219 copy_allowed_topology (struct GNUNET_TESTING_PeerGroup *pg)
2220 {
2221   unsigned int pg_iter;
2222   int ret;
2223   int total;
2224
2225   total = 0;
2226   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2227     {
2228       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].allowed_peers, &copy_topology_iterator, &pg->peers[pg_iter]);
2229       if (GNUNET_SYSERR == ret)
2230         return GNUNET_SYSERR;
2231
2232       total = total + ret;
2233     }
2234
2235   return total;
2236 }
2237
2238
2239 /**
2240  * Connect the topology as specified by the PeerConnection's
2241  * of each peer in the peer group
2242  *
2243  * @param pg the peer group we are dealing with
2244  * @return the number of connections that will be attempted
2245  */
2246 static int
2247 connect_topology (struct GNUNET_TESTING_PeerGroup *pg)
2248 {
2249   unsigned int pg_iter;
2250   int ret;
2251   int total;
2252 #if OLD
2253   struct PeerConnection *connection_iter;
2254   struct ConnectContext *connect_context;
2255 #endif
2256
2257   total = 0;
2258   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2259     {
2260       ret = GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &connect_iterator, &pg->peers[pg_iter]);
2261       if (GNUNET_SYSERR == ret)
2262         return GNUNET_SYSERR;
2263
2264       total = total + ret;
2265
2266 #if OLD
2267       connection_iter = ;
2268       while (connection_iter != NULL)
2269         {
2270           connect_context = GNUNET_malloc(sizeof(struct ConnectContext));
2271           connect_context->pg = pg;
2272           connect_context->first = ;
2273           connect_context->second = connection_iter->daemon;
2274           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_connect, connect_context);
2275           connection_iter = connection_iter->next;
2276         }
2277 #endif
2278     }
2279   return total;
2280 }
2281
2282
2283 /**
2284  * Takes a peer group and creates a topology based on the
2285  * one specified.  Creates a topology means generates friend
2286  * files for the peers so they can only connect to those allowed
2287  * by the topology.  This will only have an effect once peers
2288  * are started if the FRIENDS_ONLY option is set in the base
2289  * config.  Also takes an optional restrict topology which
2290  * disallows connections based on a particular transport
2291  * UNLESS they are specified in the restricted topology.
2292  *
2293  * @param pg the peer group struct representing the running peers
2294  * @param topology which topology to connect the peers in
2295  * @param restrict_topology allow only direct TCP connections in this topology
2296  *                          use GNUNET_TESTING_TOPOLOGY_NONE for no restrictions
2297  * @param restrict_transports space delimited list of transports to blacklist
2298  *                            to create restricted topology
2299  *
2300  * @return the maximum number of connections were all allowed peers
2301  *         connected to each other
2302  */
2303 int
2304 GNUNET_TESTING_create_topology (struct GNUNET_TESTING_PeerGroup *pg,
2305                                 enum GNUNET_TESTING_Topology topology,
2306                                 enum GNUNET_TESTING_Topology restrict_topology,
2307                                 char *restrict_transports)
2308 {
2309   int ret;
2310   int num_connections;
2311   int unblacklisted_connections;
2312
2313   GNUNET_assert (pg->notify_connection != NULL);
2314
2315   switch (topology)
2316     {
2317     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2318 #if VERBOSE_TESTING
2319       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2320                   _("Creating clique topology\n"));
2321 #endif
2322       num_connections = create_clique (pg, &add_allowed_connections);
2323       break;
2324     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2325 #if VERBOSE_TESTING
2326       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2327                   _("Creating small world (ring) topology\n"));
2328 #endif
2329       num_connections = create_small_world_ring (pg, &add_allowed_connections);
2330       break;
2331     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2332 #if VERBOSE_TESTING
2333       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2334                   _("Creating small world (2d-torus) topology\n"));
2335 #endif
2336       num_connections = create_small_world (pg, &add_allowed_connections);
2337       break;
2338     case GNUNET_TESTING_TOPOLOGY_RING:
2339 #if VERBOSE_TESTING
2340       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2341                   _("Creating ring topology\n"));
2342 #endif
2343       num_connections = create_ring (pg, &add_allowed_connections);
2344       break;
2345     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2346 #if VERBOSE_TESTING
2347       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2348                   _("Creating 2d torus topology\n"));
2349 #endif
2350       num_connections = create_2d_torus (pg, &add_allowed_connections);
2351       break;
2352     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2353 #if VERBOSE_TESTING
2354       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2355                   _("Creating Erdos-Renyi topology\n"));
2356 #endif
2357       num_connections = create_erdos_renyi (pg, &add_allowed_connections);
2358       break;
2359     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2360 #if VERBOSE_TESTING
2361       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2362                   _("Creating InterNAT topology\n"));
2363 #endif
2364       num_connections = create_nated_internet (pg, &add_allowed_connections);
2365       break;
2366     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2367 #if VERBOSE_TESTING
2368       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2369                   _("Creating Scale Free topology\n"));
2370 #endif
2371       num_connections = create_scale_free (pg, &add_allowed_connections);
2372       break;
2373     case GNUNET_TESTING_TOPOLOGY_LINE:
2374 #if VERBOSE_TESTING
2375       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2376                   _("Creating straight line topology\n"));
2377 #endif
2378       num_connections = create_line (pg, &add_allowed_connections);
2379       break;
2380     case GNUNET_TESTING_TOPOLOGY_NONE:
2381 #if VERBOSE_TESTING
2382       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2383                   _("Creating no allowed topology (all peers can connect at core level)\n"));
2384 #endif
2385       num_connections = 0;
2386       break;
2387     default:
2388       num_connections = 0;
2389       break;
2390     }
2391
2392   if (num_connections < 0)
2393     return GNUNET_SYSERR;
2394
2395   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (pg->cfg, "TESTING", "F2F"))
2396     {
2397       ret = create_and_copy_friend_files(pg);
2398       if (ret != GNUNET_OK)
2399         {
2400 #if VERBOSE_TESTING
2401           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2402                       _("Failed during friend file copying!\n"));
2403 #endif
2404           return GNUNET_SYSERR;
2405         }
2406       else
2407         {
2408 #if VERBOSE_TESTING
2409               GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2410                           _("Friend files created/copied successfully!\n"));
2411 #endif
2412         }
2413     }
2414
2415   /* Use the create clique method to initially set all connections as blacklisted. */
2416   if (restrict_topology != GNUNET_TESTING_TOPOLOGY_NONE)
2417     create_clique (pg, &blacklist_connections);
2418   unblacklisted_connections = 0;
2419   /* Un-blacklist connections as per the topology specified */
2420   switch (restrict_topology)
2421     {
2422     case GNUNET_TESTING_TOPOLOGY_CLIQUE:
2423 #if VERBOSE_TESTING
2424       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2425                   _("Blacklisting all but clique topology\n"));
2426 #endif
2427       unblacklisted_connections = create_clique (pg, &unblacklist_connections);
2428       break;
2429     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
2430 #if VERBOSE_TESTING
2431       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2432                   _("Blacklisting all but small world (ring) topology\n"));
2433 #endif
2434       unblacklisted_connections = create_small_world_ring (pg, &unblacklist_connections);
2435       break;
2436     case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
2437 #if VERBOSE_TESTING
2438       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2439                   _("Blacklisting all but small world (2d-torus) topology\n"));
2440 #endif
2441       unblacklisted_connections = create_small_world (pg, &unblacklist_connections);
2442       break;
2443     case GNUNET_TESTING_TOPOLOGY_RING:
2444 #if VERBOSE_TESTING
2445       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2446                   _("Blacklisting all but ring topology\n"));
2447 #endif
2448       unblacklisted_connections = create_ring (pg, &unblacklist_connections);
2449       break;
2450     case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
2451 #if VERBOSE_TESTING
2452       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2453                   _("Blacklisting all but 2d torus topology\n"));
2454 #endif
2455       unblacklisted_connections = create_2d_torus (pg, &unblacklist_connections);
2456       break;
2457     case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
2458 #if VERBOSE_TESTING
2459       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2460                   _("Blacklisting all but Erdos-Renyi topology\n"));
2461 #endif
2462       unblacklisted_connections = create_erdos_renyi (pg, &unblacklist_connections);
2463       break;
2464     case GNUNET_TESTING_TOPOLOGY_INTERNAT:
2465 #if VERBOSE_TESTING
2466       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2467                   _("Blacklisting all but InterNAT topology\n"));
2468 #endif
2469       unblacklisted_connections = create_nated_internet (pg, &unblacklist_connections);
2470       break;
2471     case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
2472 #if VERBOSE_TESTING
2473       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2474                   _("Blacklisting all but Scale Free topology\n"));
2475 #endif
2476       unblacklisted_connections = create_scale_free (pg, &unblacklist_connections);
2477       break;
2478     case GNUNET_TESTING_TOPOLOGY_LINE:
2479 #if VERBOSE_TESTING
2480       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2481                   _("Blacklisting all but straight line topology\n"));
2482 #endif
2483       unblacklisted_connections = create_line (pg, &unblacklist_connections);
2484       break;
2485     case GNUNET_TESTING_TOPOLOGY_NONE:
2486 #if VERBOSE_TESTING
2487       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2488                   _("Creating no blacklist topology (all peers can connect at transport level)\n"));
2489 #endif
2490     default:
2491       break;
2492     }
2493
2494   if ((unblacklisted_connections > 0) && (restrict_transports != NULL))
2495   {
2496     ret = create_and_copy_blacklist_files(pg, restrict_transports);
2497     if (ret != GNUNET_OK)
2498       {
2499 #if VERBOSE_TESTING
2500         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2501                     _("Failed during blacklist file copying!\n"));
2502 #endif
2503         return GNUNET_SYSERR;
2504       }
2505     else
2506       {
2507 #if VERBOSE_TESTING
2508         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2509                     _("Blacklist files created/copied successfully!\n"));
2510 #endif
2511       }
2512   }
2513   return num_connections;
2514 }
2515
2516 struct RandomContext
2517 {
2518   /**
2519    * The peergroup
2520    */
2521   struct GNUNET_TESTING_PeerGroup *pg;
2522
2523   /**
2524    * uid of the first peer
2525    */
2526   uint32_t first_uid;
2527
2528   /**
2529    * Peer data for first peer.
2530    */
2531   struct PeerData *first;
2532
2533   /**
2534    * Random percentage to use
2535    */
2536   double percentage;
2537 };
2538
2539 struct MinimumContext
2540 {
2541   /**
2542    * The peergroup
2543    */
2544   struct GNUNET_TESTING_PeerGroup *pg;
2545
2546   /**
2547    * uid of the first peer
2548    */
2549   uint32_t first_uid;
2550
2551   /**
2552    * Peer data for first peer.
2553    */
2554   struct PeerData *first;
2555
2556   /**
2557    * Number of conns per peer
2558    */
2559   unsigned int num_to_add;
2560
2561   /**
2562    * Permuted array of all possible connections.  Only add the Nth
2563    * peer if it's in the Nth position.
2564    */
2565   unsigned int *pg_array;
2566
2567   /**
2568    * What number is the current element we are iterating over?
2569    */
2570   unsigned int current;
2571 };
2572
2573 struct DFSContext
2574 {
2575   /**
2576    * The peergroup
2577    */
2578   struct GNUNET_TESTING_PeerGroup *pg;
2579
2580   /**
2581    * uid of the first peer
2582    */
2583   uint32_t first_uid;
2584
2585   /**
2586    * uid of the second peer
2587    */
2588   uint32_t second_uid;
2589
2590   /**
2591    * Peer data for first peer.
2592    */
2593   struct PeerData *first;
2594
2595   /**
2596    * Which peer has been chosen as the one to add?
2597    */
2598   unsigned int chosen;
2599
2600   /**
2601    * What number is the current element we are iterating over?
2602    */
2603   unsigned int current;
2604 };
2605
2606 /**
2607  * Iterator for choosing random peers to connect.
2608  *
2609  * @param cls closure, a RandomContext
2610  * @param key the key the second Daemon was stored under
2611  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2612  *
2613  * @return GNUNET_YES to continue iteration
2614  */
2615 static int
2616 random_connect_iterator (void *cls,
2617                   const GNUNET_HashCode * key,
2618                   void *value)
2619 {
2620   struct RandomContext *random_ctx = cls;
2621   double random_number;
2622   uint32_t second_pos;
2623   GNUNET_HashCode first_hash;
2624   random_number = ((double) GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK,
2625                                                      UINT64_MAX)) / ( (double) UINT64_MAX);
2626   if (random_number < random_ctx->percentage)
2627   {
2628     GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(random_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2629   }
2630   /* Now we have considered this particular connection, remove it from the second peer so it's not double counted */
2631   uid_from_hash(key, &second_pos);
2632   hash_from_uid(random_ctx->first_uid, &first_hash);
2633   GNUNET_assert(random_ctx->pg->total > second_pos);
2634   GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(random_ctx->pg->peers[second_pos].connect_peers, &first_hash, random_ctx->first->daemon));
2635
2636   return GNUNET_YES;
2637 }
2638
2639 /**
2640  * Iterator for adding at least X peers to a peers connection set.
2641  *
2642  * @param cls closure, MinimumContext
2643  * @param key the key the second Daemon was stored under
2644  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2645  *
2646  * @return GNUNET_YES to continue iteration
2647  */
2648 static int
2649 minimum_connect_iterator (void *cls,
2650                   const GNUNET_HashCode * key,
2651                   void *value)
2652 {
2653   struct MinimumContext *min_ctx = cls;
2654   uint32_t second_pos;
2655   GNUNET_HashCode first_hash;
2656   unsigned int i;
2657
2658   if (GNUNET_CONTAINER_multihashmap_size(min_ctx->first->connect_peers_working_set) < min_ctx->num_to_add)
2659   {
2660     for (i = 0; i < min_ctx->num_to_add; i++)
2661     {
2662       if (min_ctx->pg_array[i] == min_ctx->current)
2663       {
2664         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2665         uid_from_hash(key, &second_pos);
2666         hash_from_uid(min_ctx->first_uid, &first_hash);
2667         GNUNET_assert(min_ctx->pg->total > second_pos);
2668         GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(min_ctx->pg->peers[second_pos].connect_peers_working_set, &first_hash, min_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2669         /* Now we have added this particular connection, remove it from the second peer's map so it's not double counted */
2670         GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(min_ctx->pg->peers[second_pos].connect_peers, &first_hash, min_ctx->first->daemon));
2671       }
2672     }
2673     min_ctx->current++;
2674     return GNUNET_YES;
2675   }
2676   else
2677     return GNUNET_NO; /* We can stop iterating, we have enough peers! */
2678
2679 }
2680
2681
2682 /**
2683  * Iterator for adding peers to a connection set based on a depth first search.
2684  *
2685  * @param cls closure, MinimumContext
2686  * @param key the key the second daemon was stored under
2687  * @param value the GNUNET_TESTING_Daemon that the first is to connect to
2688  *
2689  * @return GNUNET_YES to continue iteration
2690  */
2691 static int
2692 dfs_connect_iterator (void *cls,
2693                   const GNUNET_HashCode * key,
2694                   void *value)
2695 {
2696   struct DFSContext *dfs_ctx = cls;
2697   GNUNET_HashCode first_hash;
2698
2699   if (dfs_ctx->current == dfs_ctx->chosen)
2700     {
2701       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->first->connect_peers_working_set, key, value, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2702       uid_from_hash(key, &dfs_ctx->second_uid);
2703       hash_from_uid(dfs_ctx->first_uid, &first_hash);
2704       GNUNET_assert(GNUNET_OK == GNUNET_CONTAINER_multihashmap_put(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers_working_set, &first_hash, dfs_ctx->first->daemon, GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY));
2705       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(dfs_ctx->pg->peers[dfs_ctx->second_uid].connect_peers, &first_hash, dfs_ctx->first->daemon));
2706       /* Can't remove second from first yet because we are currently iterating, hence the return value in the DFSContext! */
2707       return GNUNET_NO; /* We have found our peer, don't iterate more */
2708     }
2709
2710   dfs_ctx->current++;
2711   return GNUNET_YES;
2712 }
2713
2714
2715 /**
2716  * From the set of connections possible, choose percentage percent of connections
2717  * to actually connect.
2718  *
2719  * @param pg the peergroup we are dealing with
2720  * @param percentage what percent of total connections to make
2721  */
2722 void
2723 choose_random_connections(struct GNUNET_TESTING_PeerGroup *pg, double percentage)
2724 {
2725   struct RandomContext random_ctx;
2726   uint32_t pg_iter;
2727
2728   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2729     {
2730       random_ctx.first_uid = pg_iter;
2731       random_ctx.first = &pg->peers[pg_iter];
2732       random_ctx.percentage = percentage;
2733       random_ctx.pg = pg;
2734       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(pg->total);
2735       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers, &random_connect_iterator, &random_ctx);
2736       /* Now remove the old connections */
2737       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2738       /* And replace with the random set */
2739       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2740     }
2741 }
2742
2743 /**
2744  * From the set of connections possible, choose at least num connections per
2745  * peer.
2746  *
2747  * @param pg the peergroup we are dealing with
2748  * @param num how many connections at least should each peer have (if possible)?
2749  */
2750 static void
2751 choose_minimum(struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2752 {
2753   struct MinimumContext minimum_ctx;
2754   uint32_t pg_iter;
2755
2756   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2757    {
2758      pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2759    }
2760
2761   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2762     {
2763       minimum_ctx.first_uid = pg_iter;
2764       minimum_ctx.pg_array = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK,
2765                                                           GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers));
2766       minimum_ctx.first = &pg->peers[pg_iter];
2767       minimum_ctx.pg = pg;
2768       minimum_ctx.num_to_add = num;
2769       minimum_ctx.current = 0;
2770       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[pg_iter].connect_peers,
2771                                             &minimum_connect_iterator,
2772                                             &minimum_ctx);
2773     }
2774
2775   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2776     {
2777       /* Remove the "old" connections */
2778       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2779       /* And replace with the working set */
2780       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2781     }
2782
2783 }
2784
2785
2786 static unsigned int
2787 count_workingset_connections(struct GNUNET_TESTING_PeerGroup *pg)
2788 {
2789   unsigned int count;
2790   unsigned int pg_iter;
2791
2792   count = 0;
2793
2794   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2795     {
2796       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2797     }
2798
2799   return count;
2800 }
2801
2802
2803 static unsigned int count_allowed_connections(struct GNUNET_TESTING_PeerGroup *pg)
2804 {
2805   unsigned int count;
2806   unsigned int pg_iter;
2807
2808   count = 0;
2809
2810   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2811     {
2812       count += GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers);
2813     }
2814
2815   return count;
2816 }
2817
2818 /**
2819  * From the set of connections possible, choose at least num connections per
2820  * peer based on depth first traversal of peer connections.  If DFS leaves
2821  * peers unconnected, ensure those peers get connections.
2822  *
2823  * @param pg the peergroup we are dealing with
2824  * @param num how many connections at least should each peer have (if possible)?
2825  */
2826 void
2827 perform_dfs (struct GNUNET_TESTING_PeerGroup *pg, unsigned int num)
2828 {
2829   struct DFSContext dfs_ctx;
2830   uint32_t pg_iter;
2831   uint32_t dfs_count;
2832   uint32_t starting_peer;
2833   uint32_t least_connections;
2834   GNUNET_HashCode second_hash;
2835
2836   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2837     {
2838       pg->peers[pg_iter].connect_peers_working_set = GNUNET_CONTAINER_multihashmap_create(num);
2839     }
2840
2841   starting_peer = 0;
2842   dfs_count = 0;
2843   while ((count_workingset_connections(pg) < num * pg->total) && (count_allowed_connections(pg) > 0))
2844     {
2845       if (dfs_count % pg->total == 0) /* Restart the DFS at some weakly connected peer */
2846         {
2847           least_connections = -1; /* Set to very high number */
2848           for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2849             {
2850               if (GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set) < least_connections)
2851                 {
2852                   starting_peer = pg_iter;
2853                   least_connections = GNUNET_CONTAINER_multihashmap_size(pg->peers[pg_iter].connect_peers_working_set);
2854                 }
2855             }
2856         }
2857
2858       if (GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers) == 0)  /* Ensure there is at least one peer left to connect! */
2859         {
2860           dfs_count = 0;
2861           continue;
2862         }
2863
2864       /* Choose a random peer from the chosen peers set of connections to add */
2865       dfs_ctx.chosen = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_CONTAINER_multihashmap_size(pg->peers[starting_peer].connect_peers));
2866       dfs_ctx.first_uid = starting_peer;
2867       dfs_ctx.first = &pg->peers[starting_peer];
2868       dfs_ctx.pg = pg;
2869       dfs_ctx.current = 0;
2870
2871       GNUNET_CONTAINER_multihashmap_iterate(pg->peers[starting_peer].connect_peers, &dfs_connect_iterator, &dfs_ctx);
2872       /* Remove the second from the first, since we will be continuing the search and may encounter the first peer again! */
2873       hash_from_uid(dfs_ctx.second_uid, &second_hash);
2874       GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(pg->peers[starting_peer].connect_peers, &second_hash, pg->peers[dfs_ctx.second_uid].daemon));
2875       starting_peer = dfs_ctx.second_uid;
2876     }
2877
2878   for (pg_iter = 0; pg_iter < pg->total; pg_iter++)
2879     {
2880       /* Remove the "old" connections */
2881       GNUNET_CONTAINER_multihashmap_destroy(pg->peers[pg_iter].connect_peers);
2882       /* And replace with the working set */
2883       pg->peers[pg_iter].connect_peers = pg->peers[pg_iter].connect_peers_working_set;
2884     }
2885 }
2886
2887 /**
2888  * Internal callback for topology information for a particular peer.
2889  */
2890 static void
2891 internal_topology_callback(void *cls,
2892                            const struct GNUNET_PeerIdentity *peer,
2893                            struct GNUNET_TIME_Relative latency, uint32_t distance)
2894 {
2895   struct CoreContext *core_ctx = cls;
2896   struct TopologyIterateContext *iter_ctx = core_ctx->iter_context;
2897
2898   if (peer == NULL) /* Either finished, or something went wrong */
2899     {
2900       iter_ctx->completed++;
2901       iter_ctx->connected--;
2902       /* One core context allocated per iteration, must free! */
2903       GNUNET_free(core_ctx);
2904     }
2905   else
2906     {
2907       iter_ctx->topology_cb(iter_ctx->cls, &core_ctx->daemon->id, peer, latency, distance, NULL);
2908     }
2909
2910   if (iter_ctx->completed == iter_ctx->total)
2911     {
2912       iter_ctx->topology_cb(iter_ctx->cls, NULL, NULL, GNUNET_TIME_relative_get_zero(), 0, NULL);
2913       /* Once all are done, free the iteration context */
2914       GNUNET_free(iter_ctx);
2915     }
2916 }
2917
2918
2919 /**
2920  * Check running topology iteration tasks, if below max start a new one, otherwise
2921  * schedule for some time in the future.
2922  */
2923 static void
2924 schedule_get_topology(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
2925 {
2926   struct CoreContext *core_context = cls;
2927   struct TopologyIterateContext *topology_context = (struct TopologyIterateContext *)core_context->iter_context;
2928   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
2929     return;
2930
2931   if (topology_context->connected > MAX_OUTSTANDING_CONNECTIONS)
2932     {
2933 #if VERBOSE_TESTING > 2
2934           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2935                       _("Delaying connect, we have too many outstanding connections!\n"));
2936 #endif
2937       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_topology, core_context);
2938     }
2939   else
2940     {
2941 #if VERBOSE_TESTING > 2
2942           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2943                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
2944 #endif
2945       topology_context->connected++;
2946       if (GNUNET_OK != GNUNET_CORE_iterate_peers (core_context->daemon->sched, core_context->daemon->cfg, &internal_topology_callback, core_context))
2947         internal_topology_callback(core_context, NULL, GNUNET_TIME_relative_get_zero(), 0);
2948
2949     }
2950 }
2951
2952 /**
2953  * Iterate over all (running) peers in the peer group, retrieve
2954  * all connections that each currently has.
2955  */
2956 void
2957 GNUNET_TESTING_get_topology (struct GNUNET_TESTING_PeerGroup *pg, GNUNET_TESTING_NotifyTopology cb, void *cls)
2958 {
2959   struct TopologyIterateContext *topology_context;
2960   struct CoreContext *core_ctx;
2961   unsigned int i;
2962   unsigned int total_count;
2963
2964   /* Allocate a single topology iteration context */
2965   topology_context = GNUNET_malloc(sizeof(struct TopologyIterateContext));
2966   topology_context->topology_cb = cb;
2967   topology_context->cls = cls;
2968   total_count = 0;
2969   for (i = 0; i < pg->total; i++)
2970     {
2971       if (pg->peers[i].daemon->running == GNUNET_YES)
2972         {
2973           /* Allocate one core context per core we need to connect to */
2974           core_ctx = GNUNET_malloc(sizeof(struct CoreContext));
2975           core_ctx->daemon = pg->peers[i].daemon;
2976           /* Set back pointer to topology iteration context */
2977           core_ctx->iter_context = topology_context;
2978           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_topology, core_ctx);
2979           total_count++;
2980         }
2981     }
2982   topology_context->total = total_count;
2983   return;
2984 }
2985
2986 /**
2987  * Callback function to process statistic values.
2988  * This handler is here only really to insert a peer
2989  * identity (or daemon) so the statistics can be uniquely
2990  * tied to a single running peer.
2991  *
2992  * @param cls closure
2993  * @param subsystem name of subsystem that created the statistic
2994  * @param name the name of the datum
2995  * @param value the current value
2996  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
2997  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
2998  */
2999 static int internal_stats_callback (void *cls,
3000                                     const char *subsystem,
3001                                     const char *name,
3002                                     uint64_t value,
3003                                     int is_persistent)
3004 {
3005   struct StatsCoreContext *core_context = cls;
3006   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
3007
3008   return stats_context->proc(stats_context->cls, &core_context->daemon->id, subsystem, name, value, is_persistent);
3009 }
3010
3011 /**
3012  * Internal continuation call for statistics iteration.
3013  *
3014  * @param cls closure, the CoreContext for this iteration
3015  * @param success whether or not the statistics iterations
3016  *        was canceled or not (we don't care)
3017  */
3018 static void internal_stats_cont (void *cls, int success)
3019 {
3020   struct StatsCoreContext *core_context = cls;
3021   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
3022
3023   stats_context->connected--;
3024   stats_context->completed++;
3025
3026   if (stats_context->completed == stats_context->total)
3027     {
3028       stats_context->cont(stats_context->cls, GNUNET_YES);
3029       GNUNET_free(stats_context);
3030     }
3031
3032   if (core_context->stats_handle != NULL)
3033     GNUNET_STATISTICS_destroy(core_context->stats_handle, GNUNET_NO);
3034
3035   GNUNET_free(core_context);
3036 }
3037
3038 /**
3039  * Check running topology iteration tasks, if below max start a new one, otherwise
3040  * schedule for some time in the future.
3041  */
3042 static void
3043 schedule_get_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
3044 {
3045   struct StatsCoreContext *core_context = cls;
3046   struct StatsIterateContext *stats_context = (struct StatsIterateContext *)core_context->iter_context;
3047
3048   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3049     return;
3050
3051   if (stats_context->connected > MAX_OUTSTANDING_CONNECTIONS)
3052     {
3053 #if VERBOSE_TESTING > 2
3054           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3055                       _("Delaying connect, we have too many outstanding connections!\n"));
3056 #endif
3057       GNUNET_SCHEDULER_add_delayed(core_context->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_get_statistics, core_context);
3058     }
3059   else
3060     {
3061 #if VERBOSE_TESTING > 2
3062           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3063                       _("Creating connection, outstanding_connections is %d\n"), outstanding_connects);
3064 #endif
3065
3066       stats_context->connected++;
3067       core_context->stats_handle = GNUNET_STATISTICS_create(core_context->daemon->sched, "testing", core_context->daemon->cfg);
3068       if (core_context->stats_handle == NULL)
3069         {
3070           internal_stats_cont (core_context, GNUNET_NO);
3071           return;
3072         }
3073
3074       core_context->stats_get_handle = GNUNET_STATISTICS_get(core_context->stats_handle, NULL, NULL, GNUNET_TIME_relative_get_forever(), &internal_stats_cont, &internal_stats_callback, core_context);
3075       if (core_context->stats_get_handle == NULL)
3076         internal_stats_cont (core_context, GNUNET_NO);
3077
3078     }
3079 }
3080
3081
3082 /**
3083  * Iterate over all (running) peers in the peer group, retrieve
3084  * all statistics from each.
3085  */
3086 void
3087 GNUNET_TESTING_get_statistics (struct GNUNET_TESTING_PeerGroup *pg,
3088                                GNUNET_STATISTICS_Callback cont,
3089                                GNUNET_TESTING_STATISTICS_Iterator proc, void *cls)
3090 {
3091   struct StatsIterateContext *stats_context;
3092   struct StatsCoreContext *core_ctx;
3093   unsigned int i;
3094   unsigned int total_count;
3095
3096   /* Allocate a single stats iteration context */
3097   stats_context = GNUNET_malloc(sizeof(struct StatsIterateContext));
3098   stats_context->cont = cont;
3099   stats_context->proc = proc;
3100   stats_context->cls = cls;
3101   total_count = 0;
3102   for (i = 0; i < pg->total; i++)
3103     {
3104       if (pg->peers[i].daemon->running == GNUNET_YES)
3105         {
3106           /* Allocate one core context per core we need to connect to */
3107           core_ctx = GNUNET_malloc(sizeof(struct StatsCoreContext));
3108           core_ctx->daemon = pg->peers[i].daemon;
3109           /* Set back pointer to topology iteration context */
3110           core_ctx->iter_context = stats_context;
3111           GNUNET_SCHEDULER_add_now(pg->sched, &schedule_get_statistics, core_ctx);
3112           total_count++;
3113         }
3114     }
3115   stats_context->total = total_count;
3116   return;
3117 }
3118
3119 /**
3120  * There are many ways to connect peers that are supported by this function.
3121  * To connect peers in the same topology that was created via the
3122  * GNUNET_TESTING_create_topology, the topology variable must be set to
3123  * GNUNET_TESTING_TOPOLOGY_NONE.  If the topology variable is specified,
3124  * a new instance of that topology will be generated and attempted to be
3125  * connected.  This could result in some connections being impossible,
3126  * because some topologies are non-deterministic.
3127  *
3128  * @param pg the peer group struct representing the running peers
3129  * @param topology which topology to connect the peers in
3130  * @param options options for connecting the topology
3131  * @param option_modifier modifier for options that take a parameter
3132  * @return the number of connections that will be attempted, GNUNET_SYSERR on error
3133  */
3134 int
3135 GNUNET_TESTING_connect_topology (struct GNUNET_TESTING_PeerGroup *pg,
3136                                  enum GNUNET_TESTING_Topology topology,
3137                                  enum GNUNET_TESTING_TopologyOption options,
3138                                  double option_modifier)
3139 {
3140   switch (topology)
3141       {
3142       case GNUNET_TESTING_TOPOLOGY_CLIQUE:
3143 #if VERBOSE_TOPOLOGY
3144       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3145                   _("Creating clique CONNECT topology\n"));
3146 #endif
3147         create_clique (pg, &add_actual_connections);
3148         break;
3149       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD_RING:
3150 #if VERBOSE_TOPOLOGY
3151       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3152                   _("Creating small world (ring) CONNECT topology\n"));
3153 #endif
3154         create_small_world_ring (pg, &add_actual_connections);
3155         break;
3156       case GNUNET_TESTING_TOPOLOGY_SMALL_WORLD:
3157 #if VERBOSE_TOPOLOGY
3158       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3159                   _("Creating small world (2d-torus) CONNECT topology\n"));
3160 #endif
3161         create_small_world (pg, &add_actual_connections);
3162         break;
3163       case GNUNET_TESTING_TOPOLOGY_RING:
3164 #if VERBOSE_TOPOLOGY
3165       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3166                   _("Creating ring CONNECT topology\n"));
3167 #endif
3168         create_ring (pg, &add_actual_connections);
3169         break;
3170       case GNUNET_TESTING_TOPOLOGY_2D_TORUS:
3171 #if VERBOSE_TOPOLOGY
3172       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3173                   _("Creating 2d torus CONNECT topology\n"));
3174 #endif
3175         create_2d_torus (pg, &add_actual_connections);
3176         break;
3177       case GNUNET_TESTING_TOPOLOGY_ERDOS_RENYI:
3178 #if VERBOSE_TOPOLOGY
3179       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3180                   _("Creating Erdos-Renyi CONNECT topology\n"));
3181 #endif
3182         create_erdos_renyi (pg, &add_actual_connections);
3183         break;
3184       case GNUNET_TESTING_TOPOLOGY_INTERNAT:
3185 #if VERBOSE_TOPOLOGY
3186       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3187                   _("Creating InterNAT CONNECT topology\n"));
3188 #endif
3189         create_nated_internet (pg, &add_actual_connections);
3190         break;
3191       case GNUNET_TESTING_TOPOLOGY_SCALE_FREE:
3192 #if VERBOSE_TOPOLOGY
3193       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3194                   _("Creating Scale Free CONNECT topology\n"));
3195 #endif
3196         create_scale_free (pg, &add_actual_connections);
3197         break;
3198       case GNUNET_TESTING_TOPOLOGY_LINE:
3199 #if VERBOSE_TOPOLOGY
3200       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3201                   _("Creating straight line CONNECT topology\n"));
3202 #endif
3203         create_line (pg, &add_actual_connections);
3204         break;
3205       case GNUNET_TESTING_TOPOLOGY_NONE:
3206 #if VERBOSE_TOPOLOGY
3207         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3208                   _("Creating no CONNECT topology\n"));
3209 #endif
3210         copy_allowed_topology(pg);
3211         break;
3212       default:
3213         GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3214                    _("Unknown topology specification, can't connect peers!\n"));
3215         return GNUNET_SYSERR;
3216       }
3217
3218   switch (options)
3219     {
3220     case GNUNET_TESTING_TOPOLOGY_OPTION_RANDOM:
3221 #if VERBOSE_TOPOLOGY
3222       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3223                   _("Connecting random subset (%'.2f percent) of possible peers\n"), 100 * option_modifier);
3224 #endif
3225       choose_random_connections(pg, option_modifier);
3226       break;
3227     case GNUNET_TESTING_TOPOLOGY_OPTION_MINIMUM:
3228 #if VERBOSE_TOPOLOGY
3229       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3230                   _("Connecting a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3231 #endif
3232       choose_minimum(pg, (unsigned int)option_modifier);
3233       break;
3234     case GNUNET_TESTING_TOPOLOGY_OPTION_DFS:
3235 #if VERBOSE_TOPOLOGY
3236       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3237                   _("Using DFS to connect a minimum of %u peers each (if possible)\n"), (unsigned int)option_modifier);
3238 #endif
3239       perform_dfs(pg, (int)option_modifier);
3240       break;
3241     case GNUNET_TESTING_TOPOLOGY_OPTION_NONE:
3242       break;
3243     case GNUNET_TESTING_TOPOLOGY_OPTION_ALL:
3244       break;
3245     default:
3246       break;
3247     }
3248
3249   return connect_topology(pg);
3250 }
3251
3252 /**
3253  * Callback that is called whenever a hostkey is generated
3254  * for a peer.  Call the real callback and decrement the
3255  * starting counter for the peergroup.
3256  *
3257  * @param cls closure
3258  * @param id identifier for the daemon, NULL on error
3259  * @param d handle for the daemon
3260  * @param emsg error message (NULL on success)
3261  */
3262 static void internal_hostkey_callback (void *cls,
3263                                        const struct GNUNET_PeerIdentity *id,
3264                                        struct GNUNET_TESTING_Daemon *d,
3265                                        const char *emsg)
3266 {
3267   struct InternalStartContext *internal_context = cls;
3268   internal_context->peer->pg->starting--;
3269   internal_context->peer->pg->started++;
3270   if (internal_context->hostkey_callback != NULL)
3271     internal_context->hostkey_callback(internal_context->hostkey_cls, id, d, emsg);
3272   else if (internal_context->peer->pg->started == internal_context->peer->pg->total)
3273     {
3274       internal_context->peer->pg->started = 0; /* Internal startup may use this counter! */
3275       GNUNET_TESTING_daemons_continue_startup(internal_context->peer->pg);
3276     }
3277 }
3278
3279 /**
3280  * Callback that is called whenever a peer has finished starting.
3281  * Call the real callback and decrement the starting counter
3282  * for the peergroup.
3283  *
3284  * @param cls closure
3285  * @param id identifier for the daemon, NULL on error
3286  * @param d handle for the daemon
3287  * @param emsg error message (NULL on success)
3288  */
3289 static void internal_startup_callback (void *cls,
3290                                        const struct GNUNET_PeerIdentity *id,
3291                                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3292                                        struct GNUNET_TESTING_Daemon *d,
3293                                        const char *emsg)
3294 {
3295   struct InternalStartContext *internal_context = cls;
3296   internal_context->peer->pg->starting--;
3297   if (internal_context->start_cb != NULL)
3298     internal_context->start_cb(internal_context->start_cb_cls, id, cfg, d, emsg);
3299 }
3300
3301 static void
3302 internal_continue_startup (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3303 {
3304   struct InternalStartContext *internal_context = cls;
3305
3306   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3307     {
3308       return;
3309     }
3310
3311   if (internal_context->peer->pg->starting < MAX_CONCURRENT_STARTING)
3312     {
3313       internal_context->peer->pg->starting++;
3314       GNUNET_TESTING_daemon_continue_startup (internal_context->peer->daemon);
3315     }
3316   else
3317     {
3318       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_continue_startup, internal_context);
3319     }
3320 }
3321
3322 static void
3323 internal_start (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
3324 {
3325   struct InternalStartContext *internal_context = cls;
3326
3327   if (tc->reason == GNUNET_SCHEDULER_REASON_SHUTDOWN)
3328     {
3329       return;
3330     }
3331
3332   if (internal_context->peer->pg->starting < MAX_CONCURRENT_HOSTKEYS)
3333     {
3334       internal_context->peer->pg->starting++;
3335       internal_context->peer->daemon = GNUNET_TESTING_daemon_start (internal_context->peer->pg->sched,
3336                                                                     internal_context->peer->cfg,
3337                                                                     internal_context->timeout,
3338                                                                     internal_context->hostname,
3339                                                                     internal_context->username,
3340                                                                     internal_context->sshport,
3341                                                                     &internal_hostkey_callback,
3342                                                                     internal_context,
3343                                                                     &internal_startup_callback,
3344                                                                     internal_context);
3345     }
3346   else
3347     {
3348       GNUNET_SCHEDULER_add_delayed(internal_context->peer->pg->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &internal_start, internal_context);
3349     }
3350 }
3351
3352 /**
3353  * Function which continues a peer group starting up
3354  * after successfully generating hostkeys for each peer.
3355  *
3356  * @param pg the peer group to continue starting
3357  *
3358  */
3359 void
3360 GNUNET_TESTING_daemons_continue_startup(struct GNUNET_TESTING_PeerGroup *pg)
3361 {
3362   unsigned int i;
3363
3364   pg->starting = 0;
3365   for (i = 0; i < pg->total; i++)
3366     {
3367       GNUNET_SCHEDULER_add_now (pg->sched, &internal_continue_startup, &pg->peers[i].internal_context);
3368       //GNUNET_TESTING_daemon_continue_startup(pg->peers[i].daemon);
3369     }
3370 }
3371
3372 /**
3373  * Start count gnunet instances with the same set of transports and
3374  * applications.  The port numbers (any option called "PORT") will be
3375  * adjusted to ensure that no two peers running on the same system
3376  * have the same port(s) in their respective configurations.
3377  *
3378  * @param sched scheduler to use
3379  * @param cfg configuration template to use
3380  * @param total number of daemons to start
3381  * @param timeout total time allowed for peers to start
3382  * @param hostkey_callback function to call on each peers hostkey generation
3383  *        if NULL, peers will be started by this call, if non-null,
3384  *        GNUNET_TESTING_daemons_continue_startup must be called after
3385  *        successful hostkey generation
3386  * @param hostkey_cls closure for hostkey callback
3387  * @param cb function to call on each daemon that was started
3388  * @param cb_cls closure for cb
3389  * @param connect_callback function to call each time two hosts are connected
3390  * @param connect_callback_cls closure for connect_callback
3391  * @param hostnames linked list of hosts to use to start peers on (NULL to run on localhost only)
3392  *
3393  * @return NULL on error, otherwise handle to control peer group
3394  */
3395 struct GNUNET_TESTING_PeerGroup *
3396 GNUNET_TESTING_daemons_start (struct GNUNET_SCHEDULER_Handle *sched,
3397                               const struct GNUNET_CONFIGURATION_Handle *cfg,
3398                               unsigned int total,
3399                               struct GNUNET_TIME_Relative timeout,
3400                               GNUNET_TESTING_NotifyHostkeyCreated hostkey_callback,
3401                               void *hostkey_cls,
3402                               GNUNET_TESTING_NotifyDaemonRunning cb,
3403                               void *cb_cls,
3404                               GNUNET_TESTING_NotifyConnection
3405                               connect_callback, void *connect_callback_cls,
3406                               const struct GNUNET_TESTING_Host *hostnames)
3407 {
3408   struct GNUNET_TESTING_PeerGroup *pg;
3409   const struct GNUNET_TESTING_Host *hostpos;
3410 #if 0
3411   char *pos;
3412   const char *rpos;
3413   char *start;
3414 #endif
3415   const char *hostname;
3416   const char *username;
3417   char *baseservicehome;
3418   char *newservicehome;
3419   char *tmpdir;
3420   struct GNUNET_CONFIGURATION_Handle *pcfg;
3421   unsigned int off;
3422   unsigned int hostcnt;
3423   uint16_t minport;
3424   uint16_t sshport;
3425   uint32_t upnum;
3426   uint32_t fdnum;
3427
3428   if (0 == total)
3429     {
3430       GNUNET_break (0);
3431       return NULL;
3432     }
3433   upnum = 0;
3434   fdnum = 0;
3435   pg = GNUNET_malloc (sizeof (struct GNUNET_TESTING_PeerGroup));
3436   pg->sched = sched;
3437   pg->cfg = cfg;
3438   pg->notify_connection = connect_callback;
3439   pg->notify_connection_cls = connect_callback_cls;
3440   pg->total = total;
3441   pg->max_timeout = GNUNET_TIME_relative_to_absolute(timeout);
3442   pg->peers = GNUNET_malloc (total * sizeof (struct PeerData));
3443   if (NULL != hostnames)
3444     {
3445       off = 2;
3446       hostpos = hostnames;
3447       while (hostpos != NULL)
3448         {
3449           hostpos = hostpos->next;
3450           off++;
3451         }
3452       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3453       off = 0;
3454
3455       hostpos = hostnames;
3456       while (hostpos != NULL)
3457         {
3458           pg->hosts[off].minport = LOW_PORT;
3459           off++;
3460           pg->hosts[off].hostname = GNUNET_strdup(hostpos->hostname);
3461           if (hostpos->username != NULL)
3462             pg->hosts[off].username = GNUNET_strdup(hostpos->username);
3463           pg->hosts[off].sshport = hostpos->port;
3464           hostpos = hostpos->next;
3465         }
3466
3467       if (off == 0)
3468         {
3469           GNUNET_free (pg->hosts);
3470           pg->hosts = NULL;
3471         }
3472       hostcnt = off;
3473       minport = 0;
3474
3475 #if NO_LL
3476       off = 2;
3477       /* skip leading spaces */
3478       while ((0 != *hostnames) && (isspace ( (unsigned char) *hostnames)))
3479         hostnames++;
3480       rpos = hostnames;
3481       while ('\0' != *rpos)
3482         {
3483           if (isspace ( (unsigned char) *rpos))
3484             off++;
3485           rpos++;
3486         }
3487       pg->hosts = GNUNET_malloc (off * sizeof (struct HostData));
3488       off = 0;
3489       start = GNUNET_strdup (hostnames);
3490       pos = start;
3491       while ('\0' != *pos)
3492         {
3493           if (isspace ( (unsigned char) *pos))
3494             {
3495               *pos = '\0';
3496               if (strlen (start) > 0)
3497                 {
3498                   pg->hosts[off].minport = LOW_PORT;
3499                   pg->hosts[off++].hostname = start;
3500                 }
3501               start = pos + 1;
3502             }
3503           pos++;
3504         }
3505       if (strlen (start) > 0)
3506         {
3507           pg->hosts[off].minport = LOW_PORT;
3508           pg->hosts[off++].hostname = start;
3509         }
3510       if (off == 0)
3511         {
3512           GNUNET_free (start);
3513           GNUNET_free (pg->hosts);
3514           pg->hosts = NULL;
3515         }
3516       hostcnt = off;
3517       minport = 0;              /* make gcc happy */
3518 #endif
3519     }
3520   else
3521     {
3522       hostcnt = 0;
3523       minport = LOW_PORT;
3524     }
3525   for (off = 0; off < total; off++)
3526     {
3527       if (hostcnt > 0)
3528         {
3529           hostname = pg->hosts[off % hostcnt].hostname;
3530           username = pg->hosts[off % hostcnt].username;
3531           sshport = pg->hosts[off % hostcnt].sshport;
3532           pcfg = make_config (cfg,
3533                               &pg->hosts[off % hostcnt].minport,
3534                               &upnum,
3535                               hostname, &fdnum);
3536         }
3537       else
3538         {
3539           hostname = NULL;
3540           username = NULL;
3541           sshport = 0;
3542           pcfg = make_config (cfg,
3543                               &minport,
3544                               &upnum,
3545                               hostname, &fdnum);
3546         }
3547
3548       if (NULL == pcfg)
3549         {
3550           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3551                       _
3552                       ("Could not create configuration for peer number %u on `%s'!\n"),
3553                       off, hostname == NULL ? "localhost" : hostname);
3554           continue;
3555         }
3556
3557       if (GNUNET_YES ==
3558           GNUNET_CONFIGURATION_get_value_string (pcfg, "PATHS", "SERVICEHOME",
3559                                                  &baseservicehome))
3560         {
3561           GNUNET_asprintf (&newservicehome,
3562                            "%s/%d/", baseservicehome, off);
3563           GNUNET_free (baseservicehome);
3564         }
3565       else
3566         {
3567           tmpdir = getenv ("TMPDIR");
3568           tmpdir = tmpdir ? tmpdir : "/tmp";
3569           GNUNET_asprintf (&newservicehome,
3570                            "%s/%s/%d/",
3571                            tmpdir,
3572                            "gnunet-testing-test-test", off);
3573         }
3574       GNUNET_CONFIGURATION_set_value_string (pcfg,
3575                                              "PATHS",
3576                                              "SERVICEHOME", newservicehome);
3577       GNUNET_free (newservicehome);
3578       pg->peers[off].cfg = pcfg;
3579       pg->peers[off].allowed_peers = GNUNET_CONTAINER_multihashmap_create(total);
3580       pg->peers[off].connect_peers = GNUNET_CONTAINER_multihashmap_create(total);
3581       pg->peers[off].blacklisted_peers = GNUNET_CONTAINER_multihashmap_create(total);
3582       pg->peers[off].pg = pg;
3583
3584       pg->peers[off].internal_context.peer = &pg->peers[off];
3585       pg->peers[off].internal_context.timeout = timeout;
3586       pg->peers[off].internal_context.hostname = hostname;
3587       pg->peers[off].internal_context.username = username;
3588       pg->peers[off].internal_context.sshport = sshport;
3589       pg->peers[off].internal_context.hostkey_callback = hostkey_callback;
3590       pg->peers[off].internal_context.hostkey_cls = hostkey_cls;
3591       pg->peers[off].internal_context.start_cb = cb;
3592       pg->peers[off].internal_context.start_cb_cls = cb_cls;
3593
3594       GNUNET_SCHEDULER_add_now (sched, &internal_start, &pg->peers[off].internal_context);
3595
3596     }
3597   return pg;
3598 }
3599
3600 /*
3601  * Get a daemon by number, so callers don't have to do nasty
3602  * offsetting operation.
3603  */
3604 struct GNUNET_TESTING_Daemon *
3605 GNUNET_TESTING_daemon_get (struct GNUNET_TESTING_PeerGroup *pg, unsigned int position)
3606 {
3607   if (position < pg->total)
3608     return pg->peers[position].daemon;
3609   else
3610     return NULL;
3611 }
3612
3613 /*
3614  * Get a daemon by peer identity, so callers can
3615  * retrieve the daemon without knowing it's offset.
3616  *
3617  * @param pg the peer group to retrieve the daemon from
3618  * @param peer_id the peer identity of the daemon to retrieve
3619  *
3620  * @return the daemon on success, or NULL if no such peer identity is found
3621  */
3622 struct GNUNET_TESTING_Daemon *
3623 GNUNET_TESTING_daemon_get_by_id (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_PeerIdentity *peer_id)
3624 {
3625   unsigned int i;
3626
3627   for (i = 0; i < pg->total; i ++)
3628     {
3629       if (0 == memcmp(&pg->peers[i].daemon->id, peer_id, sizeof(struct GNUNET_PeerIdentity)))
3630         return pg->peers[i].daemon;
3631     }
3632
3633   return NULL;
3634 }
3635
3636 /**
3637  * Prototype of a function that will be called when a
3638  * particular operation was completed the testing library.
3639  *
3640  * @param cls closure (a struct RestartContext)
3641  * @param id id of the peer that was restarted
3642  * @param cfg handle to the configuration of the peer
3643  * @param d handle to the daemon that was restarted
3644  * @param emsg NULL on success
3645  */
3646 void restart_callback (void *cls,
3647                        const struct GNUNET_PeerIdentity *id,
3648                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3649                        struct GNUNET_TESTING_Daemon *d,
3650                        const char *emsg)
3651 {
3652   struct RestartContext *restart_context = cls;
3653
3654   if (emsg == NULL)
3655     {
3656       restart_context->peers_restarted++;
3657     }
3658   else
3659     {
3660       restart_context->peers_restart_failed++;
3661     }
3662
3663   if (restart_context->peers_restarted == restart_context->peer_group->total)
3664     {
3665       restart_context->callback(restart_context->callback_cls, NULL);
3666       GNUNET_free(restart_context);
3667     }
3668   else if (restart_context->peers_restart_failed + restart_context->peers_restarted == restart_context->peer_group->total)
3669     {
3670       restart_context->callback(restart_context->callback_cls, "Failed to restart peers!");
3671       GNUNET_free(restart_context);
3672     }
3673
3674 }
3675
3676 /**
3677  * Callback for informing us about a successful
3678  * or unsuccessful churn stop call.
3679  *
3680  * @param cls a ChurnContext
3681  * @param emsg NULL on success, non-NULL on failure
3682  *
3683  */
3684 void
3685 churn_stop_callback (void *cls, const char *emsg)
3686 {
3687   struct ChurnContext *churn_ctx = cls;
3688   unsigned int total_left;
3689   char *error_message;
3690
3691   error_message = NULL;
3692   if (emsg != NULL)
3693     {
3694       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3695                  "Churn stop callback failed with error `%s'\n", emsg);
3696       churn_ctx->num_failed_stop++;
3697     }
3698   else
3699     {
3700       churn_ctx->num_to_stop--;
3701     }
3702
3703 #if DEBUG_CHURN
3704   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3705              "Stopped peer, %d left.\n",
3706              churn_ctx->num_to_stop);
3707 #endif
3708   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3709
3710   if (total_left == 0)
3711   {
3712     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3713       {
3714         GNUNET_asprintf(&error_message,
3715                         "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3716                         churn_ctx->num_failed_start,
3717                         churn_ctx->num_failed_stop);
3718       }
3719     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3720     GNUNET_free_non_null(error_message);
3721     GNUNET_free(churn_ctx);
3722   }
3723 }
3724
3725 /**
3726  * Callback for informing us about a successful
3727  * or unsuccessful churn start call.
3728  *
3729  * @param cls a ChurnContext
3730  * @param id the peer identity of the started peer
3731  * @param cfg the handle to the configuration of the peer
3732  * @param d handle to the daemon for the peer
3733  * @param emsg NULL on success, non-NULL on failure
3734  *
3735  */
3736 void
3737 churn_start_callback (void *cls,
3738                       const struct GNUNET_PeerIdentity *id,
3739                       const struct GNUNET_CONFIGURATION_Handle *cfg,
3740                       struct GNUNET_TESTING_Daemon *d,
3741                       const char *emsg)
3742 {
3743   struct ChurnContext *churn_ctx = cls;
3744   unsigned int total_left;
3745   char *error_message;
3746
3747   error_message = NULL;
3748   if (emsg != NULL)
3749     {
3750       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3751                   "Churn stop callback failed with error `%s'\n",
3752                   emsg);
3753       churn_ctx->num_failed_start++;
3754     }
3755   else
3756     {
3757       churn_ctx->num_to_start--;
3758     }
3759
3760 #if DEBUG_CHURN
3761   GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
3762              "Started peer, %d left.\n",
3763              churn_ctx->num_to_start);
3764 #endif
3765
3766   total_left = (churn_ctx->num_to_stop - churn_ctx->num_failed_stop) + (churn_ctx->num_to_start - churn_ctx->num_failed_start);
3767
3768   if (total_left == 0)
3769   {
3770     if ((churn_ctx->num_failed_stop > 0) || (churn_ctx->num_failed_start > 0))
3771       GNUNET_asprintf(&error_message,
3772                       "Churn didn't complete successfully, %u peers failed to start %u peers failed to be stopped!",
3773                       churn_ctx->num_failed_start,
3774                       churn_ctx->num_failed_stop);
3775     churn_ctx->cb(churn_ctx->cb_cls, error_message);
3776     GNUNET_free_non_null(error_message);
3777     GNUNET_free(churn_ctx);
3778   }
3779 }
3780
3781 /**
3782  * Count the number of running peers.
3783  *
3784  * @param pg handle for the peer group
3785  *
3786  * @return the number of currently running peers in the peer group
3787  */
3788 unsigned int
3789 GNUNET_TESTING_daemons_running (struct GNUNET_TESTING_PeerGroup *pg)
3790 {
3791   unsigned int i;
3792   unsigned int running = 0;
3793   for (i = 0; i < pg->total; i++)
3794   {
3795     if (pg->peers[i].daemon->running == GNUNET_YES)
3796     {
3797       GNUNET_assert(running != -1);
3798       running++;
3799     }
3800   }
3801   return running;
3802 }
3803
3804 /**
3805  * Simulate churn by stopping some peers (and possibly
3806  * re-starting others if churn is called multiple times).  This
3807  * function can only be used to create leave-join churn (peers "never"
3808  * leave for good).  First "voff" random peers that are currently
3809  * online will be taken offline; then "von" random peers that are then
3810  * offline will be put back online.  No notifications will be
3811  * generated for any of these operations except for the callback upon
3812  * completion.
3813  *
3814  * @param pg handle for the peer group
3815  * @param voff number of peers that should go offline
3816  * @param von number of peers that should come back online;
3817  *            must be zero on first call (since "testbed_start"
3818  *            always starts all of the peers)
3819  * @param timeout how long to wait for operations to finish before
3820  *        giving up
3821  * @param cb function to call at the end
3822  * @param cb_cls closure for cb
3823  */
3824 void
3825 GNUNET_TESTING_daemons_churn (struct GNUNET_TESTING_PeerGroup *pg,
3826                               unsigned int voff,
3827                               unsigned int von,
3828                               struct GNUNET_TIME_Relative timeout,
3829                               GNUNET_TESTING_NotifyCompletion cb,
3830                               void *cb_cls)
3831 {
3832   struct ChurnContext *churn_ctx;
3833   unsigned int running;
3834   unsigned int stopped;
3835   unsigned int total_running;
3836   unsigned int total_stopped;
3837   unsigned int i;
3838   unsigned int *running_arr;
3839   unsigned int *stopped_arr;
3840   unsigned int *running_permute;
3841   unsigned int *stopped_permute;
3842
3843   running = 0;
3844   stopped = 0;
3845
3846   if ((von == 0) && (voff == 0)) /* No peers at all? */
3847     {
3848       cb(cb_cls, NULL);
3849       return;
3850     }
3851
3852   for (i = 0; i < pg->total; i++)
3853   {
3854     if (pg->peers[i].daemon->running == GNUNET_YES)
3855     {
3856       GNUNET_assert(running != -1);
3857       running++;
3858     }
3859     else
3860     {
3861       GNUNET_assert(stopped != -1);
3862       stopped++;
3863     }
3864   }
3865
3866   if (voff > running)
3867   {
3868     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to stop more peers than are currently running!\n");
3869     cb(cb_cls, "Trying to stop more peers than are currently running!");
3870     return;
3871   }
3872
3873   if (von > stopped)
3874   {
3875     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Trying to start more peers than are currently stopped!\n");
3876     cb(cb_cls, "Trying to start more peers than are currently stopped!");
3877     return;
3878   }
3879
3880   churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
3881
3882   running_arr = NULL;
3883   if (running > 0)
3884     running_arr = GNUNET_malloc(running * sizeof(unsigned int));
3885
3886   stopped_arr = NULL;
3887   if (stopped > 0)
3888     stopped_arr = GNUNET_malloc(stopped * sizeof(unsigned int));
3889
3890   running_permute = NULL;
3891   stopped_permute = NULL;
3892
3893   if (running > 0)
3894     running_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, running);
3895   if (stopped > 0)
3896     stopped_permute = GNUNET_CRYPTO_random_permute(GNUNET_CRYPTO_QUALITY_WEAK, stopped);
3897
3898   total_running = running;
3899   total_stopped = stopped;
3900   running = 0;
3901   stopped = 0;
3902
3903   churn_ctx->num_to_start = von;
3904   churn_ctx->num_to_stop = voff;
3905   churn_ctx->cb = cb;
3906   churn_ctx->cb_cls = cb_cls;
3907
3908   for (i = 0; i < pg->total; i++)
3909   {
3910     if (pg->peers[i].daemon->running == GNUNET_YES)
3911     {
3912       GNUNET_assert((running_arr != NULL) && (total_running > running));
3913       running_arr[running] = i;
3914       running++;
3915     }
3916     else
3917     {
3918       GNUNET_assert((stopped_arr != NULL) && (total_stopped > stopped));
3919       stopped_arr[stopped] = i;
3920       stopped++;
3921     }
3922   }
3923
3924   GNUNET_assert(running >= voff);
3925   for (i = 0; i < voff; i++)
3926   {
3927 #if DEBUG_CHURN
3928     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping peer %d!\n", running_permute[i]);
3929 #endif
3930     GNUNET_assert(running_arr != NULL);
3931     GNUNET_TESTING_daemon_stop (pg->peers[running_arr[running_permute[i]]].daemon,
3932                                 timeout,
3933                                 &churn_stop_callback, churn_ctx,
3934                                 GNUNET_NO, GNUNET_YES);
3935   }
3936
3937   GNUNET_assert(stopped >= von);
3938   for (i = 0; i < von; i++)
3939     {
3940 #if DEBUG_CHURN
3941       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Starting up peer %d!\n", stopped_permute[i]);
3942 #endif
3943       GNUNET_assert(stopped_arr != NULL);
3944       GNUNET_TESTING_daemon_start_stopped(pg->peers[stopped_arr[stopped_permute[i]]].daemon,
3945                                           timeout, &churn_start_callback, churn_ctx);
3946   }
3947
3948   GNUNET_free_non_null(running_arr);
3949   GNUNET_free_non_null(stopped_arr);
3950   GNUNET_free_non_null(running_permute);
3951   GNUNET_free_non_null(stopped_permute);
3952 }
3953
3954
3955 /**
3956  * Restart all peers in the given group.
3957  *
3958  * @param pg the handle to the peer group
3959  * @param callback function to call on completion (or failure)
3960  * @param callback_cls closure for the callback function
3961  */
3962 void
3963 GNUNET_TESTING_daemons_restart (struct GNUNET_TESTING_PeerGroup *pg,
3964                                 GNUNET_TESTING_NotifyCompletion callback,
3965                                 void *callback_cls)
3966 {
3967   struct RestartContext *restart_context;
3968   unsigned int off;
3969
3970   if (pg->total > 0)
3971     {
3972       restart_context = GNUNET_malloc(sizeof(struct RestartContext));
3973       restart_context->peer_group = pg;
3974       restart_context->peers_restarted = 0;
3975       restart_context->callback = callback;
3976       restart_context->callback_cls = callback_cls;
3977
3978       for (off = 0; off < pg->total; off++)
3979         {
3980           GNUNET_TESTING_daemon_restart (pg->peers[off].daemon, &restart_callback, restart_context);
3981         }
3982     }
3983 }
3984
3985 /**
3986  * Start or stop an individual peer from the given group.
3987  *
3988  * @param pg handle to the peer group
3989  * @param offset which peer to start or stop
3990  * @param desired_status GNUNET_YES to have it running, GNUNET_NO to stop it
3991  * @param timeout how long to wait for shutdown
3992  * @param cb function to call at the end
3993  * @param cb_cls closure for cb
3994  */
3995 void
3996 GNUNET_TESTING_daemons_vary (struct GNUNET_TESTING_PeerGroup *pg,
3997                              unsigned int offset,
3998                              int desired_status,
3999                              struct GNUNET_TIME_Relative timeout,
4000                              GNUNET_TESTING_NotifyCompletion cb,
4001                              void *cb_cls)
4002 {
4003   struct ChurnContext *churn_ctx;
4004
4005   if (GNUNET_NO == desired_status)
4006     {
4007       if (NULL != pg->peers[offset].daemon)
4008         {
4009           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
4010           churn_ctx->num_to_start = 0;
4011           churn_ctx->num_to_stop = 1;
4012           churn_ctx->cb = cb;
4013           churn_ctx->cb_cls = cb_cls;
4014           GNUNET_TESTING_daemon_stop(pg->peers[offset].daemon,
4015                                      timeout, &churn_stop_callback, churn_ctx,
4016                                      GNUNET_NO, GNUNET_YES);
4017         }
4018     }
4019   else if (GNUNET_YES == desired_status)
4020     {
4021       if (NULL == pg->peers[offset].daemon)
4022         {
4023           churn_ctx = GNUNET_malloc(sizeof(struct ChurnContext));
4024           churn_ctx->num_to_start = 1;
4025           churn_ctx->num_to_stop = 0;
4026           churn_ctx->cb = cb;
4027           churn_ctx->cb_cls = cb_cls;
4028           GNUNET_TESTING_daemon_start_stopped(pg->peers[offset].daemon,
4029                                               timeout, &churn_start_callback, churn_ctx);
4030         }
4031     }
4032   else
4033     GNUNET_break (0);
4034 }
4035
4036
4037 /**
4038  * Callback for shutting down peers in a peer group.
4039  *
4040  * @param cls closure (struct ShutdownContext)
4041  * @param emsg NULL on success
4042  */
4043 void internal_shutdown_callback (void *cls,
4044                                  const char *emsg)
4045 {
4046   struct ShutdownContext *shutdown_ctx = cls;
4047
4048   shutdown_ctx->outstanding--;
4049   if (emsg == NULL)
4050     {
4051       shutdown_ctx->peers_down++;
4052     }
4053   else
4054     {
4055       shutdown_ctx->peers_failed++;
4056     }
4057
4058   if ((shutdown_ctx->cb != NULL) && (shutdown_ctx->peers_down + shutdown_ctx->peers_failed == shutdown_ctx->total_peers))
4059     {
4060       if (shutdown_ctx->peers_failed > 0)
4061         shutdown_ctx->cb(shutdown_ctx->cb_cls, "Not all peers successfully shut down!");
4062       else
4063         shutdown_ctx->cb(shutdown_ctx->cb_cls, NULL);
4064       GNUNET_free(shutdown_ctx);
4065     }
4066 }
4067
4068 /**
4069  * Individual shutdown context for a particular peer.
4070  */
4071 struct PeerShutdownContext
4072 {
4073   /**
4074    * Pointer to the high level shutdown context.
4075    */
4076   struct ShutdownContext *shutdown_ctx;
4077
4078   /**
4079    * The daemon handle for the peer to shut down.
4080    */
4081   struct GNUNET_TESTING_Daemon *daemon;
4082 };
4083
4084 /**
4085  * Task to rate limit the number of outstanding peer shutdown
4086  * requests.  This is necessary for making sure we don't do
4087  * too many ssh connections at once, but is generally nicer
4088  * to any system as well (graduated task starts, as opposed
4089  * to calling gnunet-arm N times all at once).
4090  */
4091 static void
4092 schedule_shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
4093 {
4094   struct PeerShutdownContext *peer_shutdown_ctx = cls;
4095   struct ShutdownContext *shutdown_ctx;
4096
4097   GNUNET_assert(peer_shutdown_ctx != NULL);
4098   shutdown_ctx = peer_shutdown_ctx->shutdown_ctx;
4099   GNUNET_assert(shutdown_ctx != NULL);
4100
4101   if (shutdown_ctx->outstanding > MAX_CONCURRENT_SHUTDOWN)
4102     GNUNET_SCHEDULER_add_delayed(peer_shutdown_ctx->daemon->sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MILLISECONDS, 100), &schedule_shutdown_task, peer_shutdown_ctx);
4103   else
4104     {
4105       shutdown_ctx->outstanding++;
4106       GNUNET_TESTING_daemon_stop (peer_shutdown_ctx->daemon, shutdown_ctx->timeout, &internal_shutdown_callback, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4107       GNUNET_free(peer_shutdown_ctx);
4108     }
4109 }
4110 /**
4111  * Shutdown all peers started in the given group.
4112  *
4113  * @param pg handle to the peer group
4114  * @param timeout how long to wait for shutdown
4115  * @param cb callback to notify upon success or failure
4116  * @param cb_cls closure for cb
4117  */
4118 void
4119 GNUNET_TESTING_daemons_stop (struct GNUNET_TESTING_PeerGroup *pg,
4120                              struct GNUNET_TIME_Relative timeout,
4121                              GNUNET_TESTING_NotifyCompletion cb,
4122                              void *cb_cls)
4123 {
4124   unsigned int off;
4125   struct ShutdownContext *shutdown_ctx;
4126   struct PeerShutdownContext *peer_shutdown_ctx;
4127
4128   GNUNET_assert(pg->total > 0);
4129
4130   shutdown_ctx = GNUNET_malloc(sizeof(struct ShutdownContext));
4131   shutdown_ctx->cb = cb;
4132   shutdown_ctx->cb_cls = cb_cls;
4133   shutdown_ctx->total_peers = pg->total;
4134   shutdown_ctx->timeout = timeout;
4135   /* shtudown_ctx->outstanding = 0; */
4136
4137   for (off = 0; off < pg->total; off++)
4138     {
4139       GNUNET_assert(NULL != pg->peers[off].daemon);
4140       peer_shutdown_ctx = GNUNET_malloc(sizeof(struct PeerShutdownContext));
4141       peer_shutdown_ctx->daemon = pg->peers[off].daemon;
4142       peer_shutdown_ctx->shutdown_ctx = shutdown_ctx;
4143       GNUNET_SCHEDULER_add_now(pg->peers[off].daemon->sched, &schedule_shutdown_task, peer_shutdown_ctx);
4144       //GNUNET_TESTING_daemon_stop (pg->peers[off].daemon, timeout, shutdown_cb, shutdown_ctx, GNUNET_YES, GNUNET_NO);
4145       if (NULL != pg->peers[off].cfg)
4146         GNUNET_CONFIGURATION_destroy (pg->peers[off].cfg);
4147       if (pg->peers[off].allowed_peers != NULL)
4148         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].allowed_peers);
4149       if (pg->peers[off].connect_peers != NULL)
4150         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].connect_peers);
4151       if (pg->peers[off].blacklisted_peers != NULL)
4152         GNUNET_CONTAINER_multihashmap_destroy(pg->peers[off].blacklisted_peers);
4153     }
4154   GNUNET_free (pg->peers);
4155   if (NULL != pg->hosts)
4156     {
4157       GNUNET_free (pg->hosts[0].hostname);
4158       GNUNET_free (pg->hosts);
4159     }
4160   GNUNET_free (pg);
4161 }
4162
4163
4164 /* end of testing_group.c */