src/dht/gnunet-dht-driver.c

   1 /*
   2  This file is part of GNUnet.
   3  (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5  GNUnet is free software; you can redistribute it and/or modify
   6  it under the terms of the GNU General Public License as published
   7  by the Free Software Foundation; either version 3, or (at your
   8  option) any later version.
   9
  10  GNUnet is distributed in the hope that it will be useful, but
  11  WITHOUT ANY WARRANTY; without even the implied warranty of
  12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  General Public License for more details.
  14
  15  You should have received a copy of the GNU General Public License
  16  along with GNUnet; see the file COPYING.  If not, write to the
  17  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18  Boston, MA 02111-1307, USA.
  19  */
  20 /**
  21  * @file dht/gnunet-dht-driver.c
  22  * @brief Driver for setting up a group of gnunet peers and
  23  *        then issuing GETS and PUTS on the DHT.  Coarse results
  24  *        are reported, fine grained results (if requested) are
  25  *        logged to a (mysql) database, or to file.
  26  * @author Nathan Evans (who to blame)
  27  */
  28 #include "platform.h"
  29 #ifndef HAVE_MALICIOUS
  30 #error foo
  31 #endif
  32 #include "gnunet_testing_lib.h"
  33 #include "gnunet_core_service.h"
  34 #include "gnunet_dht_service.h"
  35 #include "dhtlog.h"
  36 #include "dht.h"
  37 #include "gauger.h"
  38
  39 /* Specific DEBUG hack, do not use normally (may leak memory, segfault, or eat children.) */
  40 #define ONLY_TESTING GNUNET_NO
  41
  42 /* DEFINES */
  43 #define VERBOSE GNUNET_NO
  44
  45 /* Timeout for entire driver to run */
  46 #define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  47
  48 /* Timeout for waiting for (individual) replies to get requests */
  49 #define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  50
  51 #define DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  52
  53 /* Timeout for waiting for gets to be sent to the service */
  54 #define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  55
  56 /* Timeout for waiting for puts to be sent to the service */
  57 #define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  58
  59 /* Time to allow a find peer request to take */
  60 #define DEFAULT_FIND_PEER_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40)
  61
  62 /* Time to wait for all peers disconnected due to to churn to actually be removed from system */
  63 #define DEFAULT_PEER_DISCONNECT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  64
  65 #define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45)
  66
  67 #define DEFAULT_TEST_DATA_SIZE 8
  68
  69 #define DEFAULT_BUCKET_SIZE 4
  70
  71 /* If more than this many peers are added, slow down sending */
  72 #define MAX_FIND_PEER_CUTOFF 2000
  73
  74 /* If less than this many peers are added, speed up sending */
  75 #define MIN_FIND_PEER_CUTOFF 500
  76
  77 /* How often (in seconds) to print out connection information */
  78 #define CONN_UPDATE_DURATION 10
  79
  80 #define DEFAULT_MAX_OUTSTANDING_PUTS 10
  81
  82 #define DEFAULT_MAX_OUTSTANDING_FIND_PEERS 64
  83
  84 #define DEFAULT_FIND_PEER_OFFSET GNUNET_TIME_relative_divide (DEFAULT_FIND_PEER_DELAY, DEFAULT_MAX_OUTSTANDING_FIND_PEERS)
  85
  86 #define DEFAULT_MAX_OUTSTANDING_GETS 10
  87
  88 #define DEFAULT_CONNECT_TIMEOUT 60
  89
  90 #define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8)
  91
  92 #define DEFAULT_RECONNECT_ATTEMPTS 8
  93
  94 /*
  95  * Default frequency for sending malicious get messages
  96  */
  97 #define DEFAULT_MALICIOUS_GET_FREQUENCY GNUNET_TIME_UNIT_SECONDS
  98
  99 /*
 100  * Default frequency for sending malicious put messages
 101  */
 102 #define DEFAULT_MALICIOUS_PUT_FREQUENCY GNUNET_TIME_UNIT_SECONDS
 103
 104 /* Structs */
 105
 106 struct MaliciousContext
 107 {
 108   /**
 109    * Handle to DHT service (via the API)
 110    */
 111   struct GNUNET_DHT_Handle *dht_handle;
 112
 113   /**
 114    *  Handle to the peer daemon
 115    */
 116   struct GNUNET_TESTING_Daemon *daemon;
 117
 118   /**
 119    * Task for disconnecting DHT handles
 120    */
 121   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 122
 123   /**
 124    * What type of malicious to set this peer to.
 125    */
 126   int malicious_type;
 127 };
 128
 129 struct TestFindPeer
 130 {
 131   /* This is a linked list */
 132   struct TestFindPeer *next;
 133
 134   /* Handle to the bigger context */
 135   struct FindPeerContext *find_peer_context;
 136
 137   /**
 138    * Handle to the peer's DHT service (via the API)
 139    */
 140   struct GNUNET_DHT_Handle *dht_handle;
 141
 142   /**
 143    *  Handle to the peer daemon
 144    */
 145   struct GNUNET_TESTING_Daemon *daemon;
 146
 147   /**
 148    * Task for disconnecting DHT handles
 149    */
 150   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 151 };
 152
 153 struct TestPutContext
 154 {
 155   /* This is a linked list */
 156   struct TestPutContext *next;
 157
 158   /**
 159    * Handle to the first peers DHT service (via the API)
 160    */
 161   struct GNUNET_DHT_Handle *dht_handle;
 162
 163   /**
 164    *  Handle to the PUT peer daemon
 165    */
 166   struct GNUNET_TESTING_Daemon *daemon;
 167
 168   /**
 169    *  Identifier for this PUT
 170    */
 171   uint32_t uid;
 172
 173   /**
 174    * Task for disconnecting DHT handles
 175    */
 176   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 177 };
 178
 179 struct TestGetContext
 180 {
 181   /* This is a linked list */
 182   struct TestGetContext *next;
 183
 184   /**
 185    * Handle to the first peers DHT service (via the API)
 186    */
 187   struct GNUNET_DHT_Handle *dht_handle;
 188
 189   /**
 190    * Handle for the DHT get request
 191    */
 192   struct GNUNET_DHT_GetHandle *get_handle;
 193
 194   /**
 195    *  Handle to the GET peer daemon
 196    */
 197   struct GNUNET_TESTING_Daemon *daemon;
 198
 199   /**
 200    *  Identifier for this GET
 201    */
 202   uint32_t uid;
 203
 204   /**
 205    * Task for disconnecting DHT handles (and stopping GET)
 206    */
 207   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 208
 209   /**
 210    * Whether or not this request has been fulfilled already.
 211    */
 212   int succeeded;
 213 };
 214
 215 /**
 216  * Simple struct to keep track of progress, and print a
 217  * nice little percentage meter for long running tasks.
 218  */
 219 struct ProgressMeter
 220 {
 221   unsigned int total;
 222
 223   unsigned int modnum;
 224
 225   unsigned int dotnum;
 226
 227   unsigned int completed;
 228
 229   int print;
 230
 231   char *startup_string;
 232 };
 233
 234 /**
 235  * Linked list of information for populating statistics
 236  * before ending trial.
 237  */
 238 struct StatisticsIteratorContext
 239 {
 240   const struct GNUNET_PeerIdentity *peer;
 241   unsigned int stat_routes;
 242   unsigned int stat_route_forwards;
 243   unsigned int stat_results;
 244   unsigned int stat_results_to_client;
 245   unsigned int stat_result_forwards;
 246   unsigned int stat_gets;
 247   unsigned int stat_puts;
 248   unsigned int stat_puts_inserted;
 249   unsigned int stat_find_peer;
 250   unsigned int stat_find_peer_start;
 251   unsigned int stat_get_start;
 252   unsigned int stat_put_start;
 253   unsigned int stat_find_peer_reply;
 254   unsigned int stat_get_reply;
 255   unsigned int stat_find_peer_answer;
 256   unsigned int stat_get_response_start;
 257 };
 258
 259 /**
 260  * Context for getting a topology, logging it, and continuing
 261  * on with some next operation.
 262  */
 263 struct TopologyIteratorContext
 264 {
 265   unsigned int total_iterations;
 266   unsigned int current_iteration;
 267   unsigned int total_connections;
 268   unsigned int total_peers;
 269   struct GNUNET_CONTAINER_MultiHashMap *peers_seen;
 270   struct GNUNET_PeerIdentity *peer;
 271   GNUNET_SCHEDULER_Task cont;
 272   void *cls;
 273   struct GNUNET_TIME_Relative timeout;
 274 };
 275
 276 struct PeerCount
 277 {
 278   /** Node in the heap */
 279   struct GNUNET_CONTAINER_HeapNode *heap_node;
 280
 281   /** Peer the count refers to */
 282   struct GNUNET_PeerIdentity peer_id;
 283
 284   /** Count of connections this peer has */
 285   unsigned int count;
 286 };
 287
 288 /**
 289  * Context for sending out find peer requests.
 290  */
 291 struct FindPeerContext
 292 {
 293   /**
 294    * How long to send find peer requests, once the settle time
 295    * is over don't send any more out!
 296    */
 297   struct GNUNET_TIME_Absolute endtime;
 298
 299   /**
 300    * Number of connections in the current topology
 301    * (after this round of find peer requests has ended).
 302    */
 303   unsigned int current_peers;
 304
 305   /**
 306    * Number of connections in the current topology
 307    * (before this round of find peer requests started).
 308    */
 309   unsigned int previous_peers;
 310
 311   /**
 312    * Number of find peer requests we have currently
 313    * outstanding.
 314    */
 315   unsigned int outstanding;
 316
 317   /**
 318    * Number of find peer requests to send in this round.
 319    */
 320   unsigned int total;
 321
 322   /**
 323    * Number of find peer requests sent last time around.
 324    */
 325   unsigned int last_sent;
 326
 327   /**
 328    * Hashmap of peers in the current topology, value
 329    * is a PeerCount, with the number of connections
 330    * this peer has.
 331    */
 332   struct GNUNET_CONTAINER_MultiHashMap *peer_hash;
 333
 334   /**
 335    * Min heap which orders values in the peer_hash for
 336    * easy lookup.
 337    */
 338   struct GNUNET_CONTAINER_Heap *peer_min_heap;
 339
 340   /**
 341    * Callback for counting the peers in the current topology.
 342    */
 343   GNUNET_TESTING_NotifyTopology count_peers_cb;
 344 };
 345
 346 enum DHT_ROUND_TYPES
 347 {
 348   /**
 349    * Next full round (puts + gets).
 350    */
 351   DHT_ROUND_NORMAL,
 352
 353   /**
 354    * Next round of gets.
 355    */
 356   DHT_ROUND_GET,
 357
 358   /**
 359    * Next round of puts.
 360    */
 361   DHT_ROUND_PUT,
 362
 363   /**
 364    * Next round of churn.
 365    */
 366   DHT_ROUND_CHURN
 367 };
 368
 369 /* Globals */
 370
 371 /**
 372  * How long to try to connect two peers.
 373  */
 374 struct GNUNET_TIME_Relative connect_timeout;
 375
 376 /**
 377  * How many times to re-attempt connecting two peers.
 378  */
 379 static unsigned long long connect_attempts;
 380
 381 /**
 382  * Timeout to let all GET requests happen.
 383  */
 384 static struct GNUNET_TIME_Relative all_get_timeout;
 385
 386 /**
 387  * Per get timeout
 388  */
 389 static struct GNUNET_TIME_Relative get_timeout;
 390
 391 /**
 392  * Time to allow for GET requests to be sent to service.
 393  */
 394 static struct GNUNET_TIME_Relative get_delay;
 395
 396 /**
 397  * Time to allow for PUT requests to be sent to service.
 398  */
 399 static struct GNUNET_TIME_Relative put_delay;
 400
 401 /**
 402  * Delay between sending find peer requests (if
 403  * handled by the driver, no effect if sent by service).
 404  */
 405 static struct GNUNET_TIME_Relative find_peer_delay;
 406
 407 /**
 408  * Time between find peer requests
 409  * (find_peer_delay / max_outstanding_find_peer)
 410  */
 411 static struct GNUNET_TIME_Relative find_peer_offset;
 412
 413 /**
 414  * How many seconds to allow each peer to start.
 415  */
 416 static struct GNUNET_TIME_Relative seconds_per_peer_start;
 417
 418 /**
 419  * At what time did we start the connection process.
 420  */
 421 static struct GNUNET_TIME_Absolute connect_start_time;
 422
 423 /**
 424  * What was the last time we updated connection/second information.
 425  */
 426 static struct GNUNET_TIME_Absolute connect_last_time;
 427
 428 /**
 429  * Boolean value, should the driver issue find peer requests
 430  * (GNUNET_YES) or should it be left to the service (GNUNET_NO)
 431  */
 432 static unsigned int do_find_peer;
 433
 434 #if ONLY_TESTING
 435 /**
 436  * Are we currently trying to connect two peers repeatedly?
 437  */
 438 static unsigned int repeat_connect_mode;
 439
 440 /**
 441  * Task for repeating connects.
 442  */
 443 GNUNET_SCHEDULER_TaskIdentifier repeat_connect_task;
 444
 445 struct GNUNET_TESTING_Daemon *repeat_connect_peer1;
 446 struct GNUNET_TESTING_Daemon *repeat_connect_peer2;
 447 #endif
 448
 449 /**
 450  * Boolean value, should replication be done by the dht
 451  * service (GNUNET_YES) or by the driver (GNUNET_NO)
 452  */
 453 static unsigned int in_dht_replication;
 454
 455 /**
 456  * Size of test data to insert/retrieve during testing.
 457  */
 458 static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE;
 459
 460 /**
 461  * Maximum number of concurrent connections to peers.
 462  */
 463 static unsigned long long max_outstanding_connections;
 464
 465 /**
 466  * Maximum number of concurrent ssh instances to peers.
 467  */
 468 static unsigned long long max_concurrent_ssh;
 469
 470 /**
 471  * Maximum number of concurrent PUT requests.
 472  */
 473 static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
 474
 475 /**
 476  * Maximum number of concurrent GET requests.
 477  */
 478 static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
 479
 480 /**
 481  * Number of nodes issuing malicious GET messages.
 482  */
 483 static unsigned long long malicious_getters;
 484
 485 /**
 486  * Maximum number of concurrent find peer messages being sent.
 487  */
 488 static unsigned long long max_outstanding_find_peers;
 489
 490 /**
 491  * Number of nodes issuing malicious PUT messages.
 492  */
 493 static unsigned long long malicious_putters;
 494
 495 /**
 496  * Time (in seconds) to delay between rounds.
 497  */
 498 static unsigned long long round_delay;
 499
 500 /**
 501  * The identifier for this trial (if we have one)
 502  * for external data collection.
 503  */
 504 static unsigned long long trial_to_run;
 505
 506 /**
 507  * How many malicious droppers to seed in the network.
 508  */
 509 static unsigned long long malicious_droppers;
 510
 511 /**
 512  * Bloom filter to restrict malicious nodes chosen.
 513  */
 514 struct GNUNET_CONTAINER_BloomFilter *malicious_bloom;
 515
 516 /**
 517  * Whether malicious droppers should be chosen based on proximity to a key.
 518  */
 519 static int malicious_sybil;
 520
 521 /**
 522  * Target for the malicious sybil nodes (choose the closest to this key).
 523  */
 524 static GNUNET_HashCode sybil_target;
 525
 526 /**
 527  * How often to send malicious GET messages.
 528  */
 529 static struct GNUNET_TIME_Relative malicious_get_frequency;
 530
 531 /**
 532  * How often to send malicious PUT messages.
 533  */
 534 static struct GNUNET_TIME_Relative malicious_put_frequency;
 535
 536 /**
 537  * How long to send find peer requests.
 538  */
 539 static unsigned long long settle_time;
 540
 541 /**
 542  * Handle to the dhtlog service.
 543  */
 544 static struct GNUNET_DHTLOG_Handle *dhtlog_handle;
 545
 546 /**
 547  * Replication value for GET requests.
 548  */
 549 static unsigned long long get_replication;
 550
 551 /**
 552  * Replication value for PUT requests.
 553  */
 554 static unsigned long long put_replication;
 555
 556 /**
 557  * If GNUNET_YES, insert data at the same peers every time.
 558  * Otherwise, choose a new random peer to insert at each time.
 559  */
 560 static unsigned int replicate_same;
 561
 562 /**
 563  * If GNUNET_YES, issue GET requests at the same peers every time.
 564  * Otherwise, choose a new random peer/data combination to search
 565  * each time.
 566  */
 567 static unsigned int get_from_same;
 568
 569 /**
 570  * Should malicious peers be set after allowing for settle time?
 571  * Default is to set them malicious after initial connection setup.
 572  */
 573 static unsigned int malicious_after_settle;
 574
 575 /**
 576  * Number of rounds for testing (PUTS + GETS)
 577  */
 578 static unsigned long long total_rounds;
 579
 580 /**
 581  * Target number of connections (will stop sending find peer
 582  * messages when this number is exceeded)
 583  */
 584 static unsigned long long target_total_connections;
 585
 586 /**
 587  * Number of rounds already run
 588  */
 589 static unsigned int rounds_finished;
 590
 591 /**
 592  * Number of rounds of churn to read from the file (first line, should be a single number).
 593  */
 594 static unsigned int churn_rounds;
 595
 596 /**
 597  * Current round we are in for churn, tells us how many peers to connect/disconnect.
 598  */
 599 static unsigned int current_churn_round;
 600
 601 /**
 602  * Number of times to churn per round
 603  */
 604 static unsigned long long churns_per_round;
 605
 606 /**
 607  * Array of churn values.
 608  */
 609 static unsigned int *churn_array;
 610
 611 /**
 612  * Hash map of stats contexts.
 613  */
 614 static struct GNUNET_CONTAINER_MultiHashMap *stats_map;
 615
 616 /**
 617  * LL of malicious settings.
 618  */
 619 struct MaliciousContext *all_malicious;
 620
 621 /**
 622  * List of GETS to perform
 623  */
 624 struct TestGetContext *all_gets;
 625
 626 /**
 627  * List of PUTS to perform
 628  */
 629 struct TestPutContext *all_puts;
 630
 631 /**
 632  * Directory to store temporary data in, defined in config file
 633  */
 634 static char *test_directory;
 635
 636 /**
 637  * Variable used to store the number of connections we should wait for.
 638  */
 639 static unsigned int expected_connections;
 640
 641 /**
 642  * Variable used to keep track of how many peers aren't yet started.
 643  */
 644 static unsigned long long peers_left;
 645
 646 /**
 647  * Handle to the set of all peers run for this test.
 648  */
 649 static struct GNUNET_TESTING_PeerGroup *pg;
 650
 651 /**
 652  * Global config handle.
 653  */
 654 static const struct GNUNET_CONFIGURATION_Handle *config;
 655
 656 /**
 657  * Total number of peers to run, set based on config file.
 658  */
 659 static unsigned long long num_peers;
 660
 661 /**
 662  * Total number of items to insert.
 663  */
 664 static unsigned long long num_puts;
 665
 666 /**
 667  * How many puts do we currently have in flight?
 668  */
 669 static unsigned long long outstanding_puts;
 670
 671 /**
 672  * How many puts are done?
 673  */
 674 static unsigned long long puts_completed;
 675
 676 /**
 677  * Total number of items to attempt to get.
 678  */
 679 static unsigned long long num_gets;
 680
 681 /**
 682  * How many puts do we currently have in flight?
 683  */
 684 static unsigned long long outstanding_gets;
 685
 686 /**
 687  * How many gets are done?
 688  */
 689 static unsigned long long gets_completed;
 690
 691 /**
 692  * Total number of items to attempt to get.
 693  */
 694 static unsigned long long cumulative_num_gets;
 695
 696 /**
 697  * How many gets are done?
 698  */
 699 static unsigned long long cumulative_successful_gets;
 700
 701 /**
 702  * How many gets failed?
 703  */
 704 static unsigned long long gets_failed;
 705
 706 /**
 707  * How many malicious control messages do
 708  * we currently have in flight?
 709  */
 710 static unsigned long long outstanding_malicious;
 711
 712 /**
 713  * How many set malicious peers are done?
 714  */
 715 static unsigned int malicious_completed;
 716
 717 /**
 718  * For gauger logging, what specific identifier (svn revision)
 719  * should be used?
 720  */
 721 static unsigned long long revision;
 722
 723 /**
 724  * Global used to count how many connections we have currently
 725  * been notified about (how many times has topology_callback been called
 726  * with success?)
 727  */
 728 static unsigned int total_connections;
 729
 730 /**
 731  * Previous connections, for counting new connections during some duration.
 732  */
 733 static unsigned int previous_connections;
 734
 735 /**
 736  * For counting failed connections during some duration.
 737  */
 738 static unsigned int previous_failed_connections;
 739
 740 /**
 741  * Global used to count how many failed connections we have
 742  * been notified about (how many times has topology_callback
 743  * been called with failure?)
 744  */
 745 static unsigned int failed_connections;
 746
 747 /**
 748  * If GNUNET_YES, only log PUT/GET round data to mysql, otherwise
 749  * log everything (including each dht service logging).
 750  */
 751 static unsigned int dhtlog_minimal;
 752
 753 /* Task handle to use to schedule shutdown if something goes wrong */
 754 GNUNET_SCHEDULER_TaskIdentifier die_task;
 755
 756 static char *blacklist_transports;
 757
 758 static enum GNUNET_TESTING_Topology topology;
 759
 760 static enum GNUNET_TESTING_Topology blacklist_topology =
 761     GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 762
 763 static enum GNUNET_TESTING_Topology connect_topology =
 764     GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 765
 766 static enum GNUNET_TESTING_TopologyOption connect_topology_option =
 767     GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 768
 769 static double connect_topology_option_modifier = 0.0;
 770
 771 static struct ProgressMeter *hostkey_meter;
 772
 773 static struct ProgressMeter *peer_start_meter;
 774
 775 static struct ProgressMeter *peer_connect_meter;
 776
 777 static struct ProgressMeter *put_meter;
 778
 779 static struct ProgressMeter *get_meter;
 780
 781 static GNUNET_HashCode *known_keys;
 782
 783 /* Global return value (0 for success, anything else for failure) */
 784 static int ok;
 785
 786 /**
 787  * Create a meter to keep track of the progress of some task.
 788  *
 789  * @param total the total number of items to complete
 790  * @param start_string a string to prefix the meter with (if printing)
 791  * @param print GNUNET_YES to print the meter, GNUNET_NO to count
 792  *              internally only
 793  *
 794  * @return the progress meter
 795  */
 796 static struct ProgressMeter *
 797 create_meter(unsigned int total, char * start_string, int print)
 798 {
 799   struct ProgressMeter *ret;
 800   ret = GNUNET_malloc(sizeof(struct ProgressMeter));
 801   ret->print = print;
 802   ret->total = total;
 803   ret->modnum = total / 4;
 804   ret->dotnum = (total / 50) + 1;
 805   if (start_string != NULL)
 806     ret->startup_string = GNUNET_strdup(start_string);
 807   else
 808     ret->startup_string = GNUNET_strdup("");
 809
 810   return ret;
 811 }
 812
 813 /**
 814  * Update progress meter (increment by one).
 815  *
 816  * @param meter the meter to update and print info for
 817  *
 818  * @return GNUNET_YES if called the total requested,
 819  *         GNUNET_NO if more items expected
 820  */
 821 static int
 822 update_meter(struct ProgressMeter *meter)
 823 {
 824   if (meter->print == GNUNET_YES)
 825     {
 826       if (meter->completed % meter->modnum == 0)
 827         {
 828           if (meter->completed == 0)
 829             {
 830               fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
 831             }
 832           else
 833             fprintf (stdout, "%d%%", (int) (((float) meter->completed
 834                 / meter->total) * 100));
 835         }
 836       else if (meter->completed % meter->dotnum == 0)
 837         fprintf (stdout, ".");
 838
 839       if (meter->completed + 1 == meter->total)
 840         fprintf (stdout, "%d%%]\n", 100);
 841       fflush (stdout);
 842     }
 843   meter->completed++;
 844
 845   if (meter->completed == meter->total)
 846     return GNUNET_YES;
 847   return GNUNET_NO;
 848 }
 849
 850 /**
 851  * Reset progress meter.
 852  *
 853  * @param meter the meter to reset
 854  *
 855  * @return GNUNET_YES if meter reset,
 856  *         GNUNET_SYSERR on error
 857  */
 858 static int
 859 reset_meter(struct ProgressMeter *meter)
 860 {
 861   if (meter == NULL)
 862     return GNUNET_SYSERR;
 863
 864   meter->completed = 0;
 865   return GNUNET_YES;
 866 }
 867
 868 /**
 869  * Release resources for meter
 870  *
 871  * @param meter the meter to free
 872  */
 873 static void
 874 free_meter(struct ProgressMeter *meter)
 875 {
 876   GNUNET_free_non_null (meter->startup_string);
 877   GNUNET_free (meter);
 878 }
 879
 880 /**
 881  * Check whether peers successfully shut down.
 882  */
 883 static void
 884 shutdown_callback(void *cls, const char *emsg)
 885 {
 886   if (emsg != NULL)
 887     {
 888       if (ok == 0)
 889         ok = 2;
 890     }
 891 }
 892
 893 /**
 894  * Task to release DHT handles for PUT
 895  */
 896 static void
 897 put_disconnect_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 898 {
 899   struct TestPutContext *test_put = cls;
 900   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 901   GNUNET_DHT_disconnect (test_put->dht_handle);
 902   test_put->dht_handle = NULL;
 903   if (replicate_same == GNUNET_NO)
 904     test_put->daemon
 905         = GNUNET_TESTING_daemon_get (
 906                                      pg,
 907                                      GNUNET_CRYPTO_random_u32 (
 908                                                                GNUNET_CRYPTO_QUALITY_WEAK,
 909                                                                num_peers));
 910 }
 911
 912 /**
 913  * Function scheduled to be run on the successful completion of this
 914  * testcase.
 915  */
 916 static void
 917 finish_testing(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 918 {
 919   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ending test normally!\n",
 920               (char *) cls);
 921   GNUNET_assert (pg != NULL);
 922   struct TestPutContext *test_put = all_puts;
 923   struct TestGetContext *test_get = all_gets;
 924   char *temp_get_string;
 925   char *revision_str;
 926
 927   while (test_put != NULL)
 928     {
 929       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 930         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
 931       if (test_put->dht_handle != NULL)
 932         GNUNET_DHT_disconnect (test_put->dht_handle);
 933       test_put = test_put->next;
 934     }
 935
 936   while (test_get != NULL)
 937     {
 938       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 939         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
 940       if (test_get->get_handle != NULL)
 941         GNUNET_DHT_get_stop (test_get->get_handle);
 942       if (test_get->dht_handle != NULL)
 943         GNUNET_DHT_disconnect (test_get->dht_handle);
 944       test_get = test_get->next;
 945     }
 946
 947   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
 948
 949   if (dhtlog_handle != NULL)
 950     {
 951       fprintf (stderr, "Update trial endtime\n");
 952       dhtlog_handle->update_trial (cumulative_successful_gets);
 953       GNUNET_DHTLOG_disconnect (dhtlog_handle);
 954       dhtlog_handle = NULL;
 955     }
 956
 957   if (hostkey_meter != NULL)
 958     free_meter (hostkey_meter);
 959   if (peer_start_meter != NULL)
 960     free_meter (peer_start_meter);
 961   if (peer_connect_meter != NULL)
 962     free_meter (peer_connect_meter);
 963   if (put_meter != NULL)
 964     free_meter (put_meter);
 965   if (get_meter != NULL)
 966     free_meter (get_meter);
 967
 968   GNUNET_asprintf (&temp_get_string,
 969                    "DHT Successful GETs (trial %d)", trial_to_run);
 970   GNUNET_asprintf (&revision_str, "%llu", revision);
 971   GAUGER_ID("DHT_TESTING", temp_get_string, cumulative_successful_gets / (double)cumulative_num_gets, "percent successful", revision_str);
 972   fprintf (
 973            stderr,
 974            "Finished trial, had %llu successful gets out of %llu total, %.2f percent succeeded\n",
 975            cumulative_successful_gets, cumulative_num_gets,
 976            cumulative_successful_gets / (double) cumulative_num_gets);
 977   GNUNET_free(temp_get_string);
 978
 979   ok = 0;
 980 }
 981
 982 /**
 983  * Callback for iterating over all the peer connections of a peer group.
 984  */
 985 static void
 986 log_topology_cb(void *cls, const struct GNUNET_PeerIdentity *first,
 987     const struct GNUNET_PeerIdentity *second, const char *emsg)
 988 {
 989   struct TopologyIteratorContext *topo_ctx = cls;
 990   if ((first != NULL) && (second != NULL))
 991     {
 992       if ((topo_ctx->peers_seen != NULL) && (GNUNET_NO
 993           == GNUNET_CONTAINER_multihashmap_contains (topo_ctx->peers_seen,
 994                                                      &first->hashPubKey)))
 995         {
 996           GNUNET_CONTAINER_multihashmap_put (topo_ctx->peers_seen,
 997                                              &first->hashPubKey, NULL,
 998                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
 999           topo_ctx->total_peers++;
1000         }
1001       topo_ctx->total_connections++;
1002       if ((GNUNET_NO == dhtlog_minimal) && (dhtlog_handle != NULL))
1003         dhtlog_handle->insert_extended_topology (first, second);
1004     }
1005   else
1006     {
1007       GNUNET_assert(dhtlog_handle != NULL);
1008       GNUNET_log (
1009                   GNUNET_ERROR_TYPE_WARNING,
1010                   "Topology iteration (%u/%u) finished (%u connections, %u peers)\n",
1011                   topo_ctx->current_iteration, topo_ctx->total_iterations,
1012                   topo_ctx->total_connections, topo_ctx->total_peers);
1013       dhtlog_handle->update_topology (topo_ctx->total_connections);
1014       if (topo_ctx->cont != NULL)
1015         GNUNET_SCHEDULER_add_now (topo_ctx->cont, topo_ctx->cls);
1016       if (topo_ctx->peers_seen != NULL)
1017         GNUNET_CONTAINER_multihashmap_destroy (topo_ctx->peers_seen);
1018       GNUNET_free(topo_ctx);
1019     }
1020 }
1021
1022 /**
1023  * Iterator over hash map entries.
1024  *
1025  * @param cls closure - always NULL
1026  * @param key current key code
1027  * @param value value in the hash map, a stats context
1028  * @return GNUNET_YES if we should continue to
1029  *         iterate,
1030  *         GNUNET_NO if not.
1031  */
1032 static int
1033 stats_iterate(void *cls, const GNUNET_HashCode * key, void *value)
1034 {
1035   struct StatisticsIteratorContext *stats_ctx;
1036   if (value == NULL)
1037     return GNUNET_NO;
1038   stats_ctx = value;
1039   dhtlog_handle->insert_stat (stats_ctx->peer, stats_ctx->stat_routes,
1040                               stats_ctx->stat_route_forwards,
1041                               stats_ctx->stat_results,
1042                               stats_ctx->stat_results_to_client,
1043                               stats_ctx->stat_result_forwards,
1044                               stats_ctx->stat_gets, stats_ctx->stat_puts,
1045                               stats_ctx->stat_puts_inserted,
1046                               stats_ctx->stat_find_peer,
1047                               stats_ctx->stat_find_peer_start,
1048                               stats_ctx->stat_get_start,
1049                               stats_ctx->stat_put_start,
1050                               stats_ctx->stat_find_peer_reply,
1051                               stats_ctx->stat_get_reply,
1052                               stats_ctx->stat_find_peer_answer,
1053                               stats_ctx->stat_get_response_start);
1054   GNUNET_free(stats_ctx);
1055   return GNUNET_YES;
1056 }
1057
1058 static void
1059 stats_finished(void *cls, int result)
1060 {
1061   fprintf (stderr, "Finished getting all peers statistics, iterating!\n");
1062   GNUNET_CONTAINER_multihashmap_iterate (stats_map, &stats_iterate, NULL);
1063   GNUNET_CONTAINER_multihashmap_destroy (stats_map);
1064   GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1065 }
1066
1067 /**
1068  * Callback function to process statistic values.
1069  *
1070  * @param cls closure
1071  * @param peer the peer the statistics belong to
1072  * @param subsystem name of subsystem that created the statistic
1073  * @param name the name of the datum
1074  * @param value the current value
1075  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1076  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1077  */
1078 static int
1079 stats_handle(void *cls, const struct GNUNET_PeerIdentity *peer,
1080     const char *subsystem, const char *name, uint64_t value, int is_persistent)
1081 {
1082   struct StatisticsIteratorContext *stats_ctx;
1083
1084   if (dhtlog_handle != NULL)
1085     dhtlog_handle->add_generic_stat (peer, name, subsystem, value);
1086   if (GNUNET_CONTAINER_multihashmap_contains (stats_map, &peer->hashPubKey))
1087     {
1088       stats_ctx = GNUNET_CONTAINER_multihashmap_get (stats_map,
1089                                                      &peer->hashPubKey);
1090     }
1091   else
1092     {
1093       stats_ctx = GNUNET_malloc(sizeof(struct StatisticsIteratorContext));
1094       stats_ctx->peer = peer;
1095       GNUNET_CONTAINER_multihashmap_put (stats_map, &peer->hashPubKey,
1096                                          stats_ctx,
1097                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1098     }
1099   GNUNET_assert(stats_ctx != NULL);
1100
1101   if (strcmp (name, STAT_ROUTES) == 0)
1102     stats_ctx->stat_routes = value;
1103   else if (strcmp (name, STAT_ROUTE_FORWARDS) == 0)
1104     stats_ctx->stat_route_forwards = value;
1105   else if (strcmp (name, STAT_RESULTS) == 0)
1106     stats_ctx->stat_results = value;
1107   else if (strcmp (name, STAT_RESULTS_TO_CLIENT) == 0)
1108     stats_ctx->stat_results_to_client = value;
1109   else if (strcmp (name, STAT_RESULT_FORWARDS) == 0)
1110     stats_ctx->stat_result_forwards = value;
1111   else if (strcmp (name, STAT_GETS) == 0)
1112     stats_ctx->stat_gets = value;
1113   else if (strcmp (name, STAT_PUTS) == 0)
1114     stats_ctx->stat_puts = value;
1115   else if (strcmp (name, STAT_PUTS_INSERTED) == 0)
1116     stats_ctx->stat_puts_inserted = value;
1117   else if (strcmp (name, STAT_FIND_PEER) == 0)
1118     stats_ctx->stat_find_peer = value;
1119   else if (strcmp (name, STAT_FIND_PEER_START) == 0)
1120     stats_ctx->stat_find_peer_start = value;
1121   else if (strcmp (name, STAT_GET_START) == 0)
1122     stats_ctx->stat_get_start = value;
1123   else if (strcmp (name, STAT_PUT_START) == 0)
1124     stats_ctx->stat_put_start = value;
1125   else if (strcmp (name, STAT_FIND_PEER_REPLY) == 0)
1126     stats_ctx->stat_find_peer_reply = value;
1127   else if (strcmp (name, STAT_GET_REPLY) == 0)
1128     stats_ctx->stat_get_reply = value;
1129   else if (strcmp (name, STAT_FIND_PEER_ANSWER) == 0)
1130     stats_ctx->stat_find_peer_answer = value;
1131   else if (strcmp (name, STAT_GET_RESPONSE_START) == 0)
1132     stats_ctx->stat_get_response_start = value;
1133
1134   return GNUNET_OK;
1135 }
1136
1137 /**
1138  * Connect to statistics service for each peer and get the appropriate
1139  * dht statistics for safe keeping.
1140  */
1141 static void
1142 log_dht_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1143 {
1144   stats_map = GNUNET_CONTAINER_multihashmap_create (num_peers);
1145   fprintf (stderr, "Starting statistics logging\n");
1146   GNUNET_TESTING_get_statistics (pg, &stats_finished, &stats_handle, NULL);
1147 }
1148
1149 /**
1150  * Connect to all peers in the peer group and iterate over their
1151  * connections.
1152  */
1153 static void
1154 capture_current_topology(void *cls,
1155     const struct GNUNET_SCHEDULER_TaskContext * tc)
1156 {
1157   struct TopologyIteratorContext *topo_ctx = cls;
1158   dhtlog_handle->insert_topology (0);
1159   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Called capture_current_topology\n");
1160   GNUNET_TESTING_get_topology (pg, &log_topology_cb, topo_ctx);
1161 }
1162
1163 /**
1164  * Check if the get_handle is being used, if so stop the request.  Either
1165  * way, schedule the end_badly_cont function which actually shuts down the
1166  * test.
1167  */
1168 static void
1169 end_badly(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1170 {
1171   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failing test with error: `%s'!\n",
1172               (char *) cls);
1173
1174   struct TestPutContext *test_put = all_puts;
1175   struct TestGetContext *test_get = all_gets;
1176
1177   while (test_put != NULL)
1178     {
1179       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1180         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
1181       if (test_put->dht_handle != NULL)
1182         GNUNET_DHT_disconnect (test_put->dht_handle);
1183       test_put = test_put->next;
1184     }
1185
1186   while (test_get != NULL)
1187     {
1188       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1189         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
1190       if (test_get->get_handle != NULL)
1191         GNUNET_DHT_get_stop (test_get->get_handle);
1192       if (test_get->dht_handle != NULL)
1193         GNUNET_DHT_disconnect (test_get->dht_handle);
1194       test_get = test_get->next;
1195     }
1196
1197   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
1198
1199   if (dhtlog_handle != NULL)
1200     {
1201       fprintf (stderr, "Update trial endtime\n");
1202       dhtlog_handle->update_trial (gets_completed);
1203       GNUNET_DHTLOG_disconnect (dhtlog_handle);
1204       dhtlog_handle = NULL;
1205     }
1206
1207   if (hostkey_meter != NULL)
1208     free_meter (hostkey_meter);
1209   if (peer_start_meter != NULL)
1210     free_meter (peer_start_meter);
1211   if (peer_connect_meter != NULL)
1212     free_meter (peer_connect_meter);
1213   if (put_meter != NULL)
1214     free_meter (put_meter);
1215   if (get_meter != NULL)
1216     free_meter (get_meter);
1217
1218   ok = 1;
1219 }
1220
1221 /**
1222  * Forward declaration.
1223  */
1224 static void
1225 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1226
1227 /**
1228  * Forward declaration.
1229  */
1230 static void
1231 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1232
1233 /**
1234  * Iterator over hash map entries.
1235  *
1236  * @param cls closure
1237  * @param key current key code
1238  * @param value value in the hash map
1239  * @return GNUNET_YES if we should continue to
1240  *         iterate,
1241  *         GNUNET_NO if not.
1242  */
1243 static int
1244 remove_peer_count(void *cls, const GNUNET_HashCode * key, void *value)
1245 {
1246   struct PeerCount *peer_count = value;
1247   GNUNET_CONTAINER_heap_remove_node (peer_count->heap_node);
1248   GNUNET_free(peer_count);
1249
1250   return GNUNET_YES;
1251 }
1252
1253 /**
1254  * Connect to all peers in the peer group and iterate over their
1255  * connections.
1256  */
1257 static void
1258 count_new_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1259 {
1260   struct FindPeerContext *find_peer_context = cls;
1261   find_peer_context->previous_peers = find_peer_context->current_peers;
1262   find_peer_context->current_peers = 0;
1263   GNUNET_TESTING_get_topology (pg, find_peer_context->count_peers_cb,
1264                                find_peer_context);
1265 }
1266
1267 static void
1268 decrement_find_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1269 {
1270   struct TestFindPeer *test_find_peer = cls;
1271   GNUNET_assert(test_find_peer->find_peer_context->outstanding > 0);
1272   test_find_peer->find_peer_context->outstanding--;
1273   test_find_peer->find_peer_context->total--;
1274   if (0 == test_find_peer->find_peer_context->total)
1275     {
1276       GNUNET_SCHEDULER_add_now (&count_new_peers,
1277                                 test_find_peer->find_peer_context);
1278     }
1279   GNUNET_free(test_find_peer);
1280 }
1281
1282 /**
1283  * A find peer request has been sent to the server, now we will schedule a task
1284  * to wait the appropriate time to allow the request to go out and back.
1285  *
1286  * @param cls closure - a TestFindPeer struct
1287  * @param tc context the task is being called with
1288  */
1289 static void
1290 handle_find_peer_sent(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1291 {
1292   struct TestFindPeer *test_find_peer = cls;
1293
1294   GNUNET_DHT_disconnect (test_find_peer->dht_handle);
1295   GNUNET_SCHEDULER_add_delayed (
1296                                 GNUNET_TIME_relative_divide (find_peer_delay, 2),
1297                                 &decrement_find_peers, test_find_peer);
1298 }
1299
1300 static void
1301 send_find_peer_request(void *cls,
1302     const struct GNUNET_SCHEDULER_TaskContext * tc)
1303 {
1304   struct TestFindPeer *test_find_peer = cls;
1305
1306   if (test_find_peer->find_peer_context->outstanding
1307       > max_outstanding_find_peers)
1308     {
1309       GNUNET_SCHEDULER_add_delayed (find_peer_offset, &send_find_peer_request,
1310                                     test_find_peer);
1311       return;
1312     }
1313
1314   test_find_peer->find_peer_context->outstanding++;
1315   if (GNUNET_TIME_absolute_get_remaining (
1316                                           test_find_peer->find_peer_context->endtime).rel_value
1317       == 0)
1318     {
1319       GNUNET_SCHEDULER_add_now (&decrement_find_peers, test_find_peer);
1320       return;
1321     }
1322
1323   test_find_peer->dht_handle = GNUNET_DHT_connect (test_find_peer->daemon->cfg,
1324                                                    1);
1325   GNUNET_assert(test_find_peer->dht_handle != NULL);
1326   GNUNET_DHT_find_peers (test_find_peer->dht_handle, &handle_find_peer_sent,
1327                          test_find_peer);
1328 }
1329
1330 /**
1331  * Add a connection to the find_peer_context given.  This may
1332  * be complete overkill, but allows us to choose the peers with
1333  * the least connections to initiate find peer requests from.
1334  */
1335 static void
1336 add_new_connection(struct FindPeerContext *find_peer_context,
1337     const struct GNUNET_PeerIdentity *first,
1338     const struct GNUNET_PeerIdentity *second)
1339 {
1340   struct PeerCount *first_count;
1341   struct PeerCount *second_count;
1342
1343   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1344                                               &first->hashPubKey))
1345     {
1346       first_count
1347           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1348                                                &first->hashPubKey);
1349       GNUNET_assert(first_count != NULL);
1350       first_count->count++;
1351       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1352                                          first_count->heap_node,
1353                                          first_count->count);
1354     }
1355   else
1356     {
1357       first_count = GNUNET_malloc(sizeof(struct PeerCount));
1358       first_count->count = 1;
1359       memcpy (&first_count->peer_id, first, sizeof(struct GNUNET_PeerIdentity));
1360       first_count->heap_node
1361           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1362                                           first_count, first_count->count);
1363       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1364                                          &first->hashPubKey, first_count,
1365                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1366     }
1367
1368   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1369                                               &second->hashPubKey))
1370     {
1371       second_count
1372           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1373                                                &second->hashPubKey);
1374       GNUNET_assert(second_count != NULL);
1375       second_count->count++;
1376       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1377                                          second_count->heap_node,
1378                                          second_count->count);
1379     }
1380   else
1381     {
1382       second_count = GNUNET_malloc(sizeof(struct PeerCount));
1383       second_count->count = 1;
1384       memcpy (&second_count->peer_id, second,
1385               sizeof(struct GNUNET_PeerIdentity));
1386       second_count->heap_node
1387           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1388                                           second_count, second_count->count);
1389       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1390                                          &second->hashPubKey, second_count,
1391                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1392     }
1393 }
1394
1395 /**
1396  * Iterate over min heap of connections per peer.  For any
1397  * peer that has 0 connections, attempt to connect them to
1398  * some random peer.
1399  *
1400  * @param cls closure a struct FindPeerContext
1401  * @param node internal node of the heap
1402  * @param element value stored, a struct PeerCount
1403  * @param cost cost associated with the node
1404  * @return GNUNET_YES if we should continue to iterate,
1405  *         GNUNET_NO if not.
1406  */
1407 static int
1408 iterate_min_heap_peers(void *cls, struct GNUNET_CONTAINER_HeapNode *node,
1409     void *element, GNUNET_CONTAINER_HeapCostType cost)
1410 {
1411   struct FindPeerContext *find_peer_context = cls;
1412   struct PeerCount *peer_count = element;
1413   struct GNUNET_TESTING_Daemon *d1;
1414   struct GNUNET_TESTING_Daemon *d2;
1415   struct GNUNET_TIME_Relative timeout;
1416   if (cost == 0)
1417     {
1418       d1 = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1419       GNUNET_assert(d1 != NULL);
1420       d2 = d1;
1421       while ((d2 == d1) || (GNUNET_YES != GNUNET_TESTING_daemon_running (d2)))
1422         {
1423           d2
1424               = GNUNET_TESTING_daemon_get (
1425                                            pg,
1426                                            GNUNET_CRYPTO_random_u32 (
1427                                                                      GNUNET_CRYPTO_QUALITY_WEAK,
1428                                                                      num_peers));
1429           GNUNET_assert(d2 != NULL);
1430         }
1431
1432       /** Just try to connect the peers, don't worry about callbacks, etc. **/
1433       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1434                   "Peer %s has 0 connections.  Trying to connect to %s...\n",
1435                   GNUNET_i2s (&peer_count->peer_id), d2->shortname);
1436       timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
1437                                                DEFAULT_CONNECT_TIMEOUT);
1438       if (GNUNET_TIME_relative_to_absolute (timeout).abs_value
1439           > find_peer_context->endtime.abs_value)
1440         {
1441           timeout
1442               = GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime);
1443         }
1444       GNUNET_TESTING_daemons_connect (d1, d2, timeout,
1445                                       DEFAULT_RECONNECT_ATTEMPTS, GNUNET_YES,
1446                                       NULL, NULL);
1447     }
1448   if (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1449       > 0)
1450     return GNUNET_YES;
1451   else
1452     return GNUNET_NO;
1453 }
1454
1455 /**
1456  * Forward declaration.
1457  */
1458 static void
1459 schedule_churn_find_peer_requests(void *cls,
1460     const struct GNUNET_SCHEDULER_TaskContext * tc);
1461
1462 /**
1463  * Callback for iterating over all the peer connections of a peer group.
1464  * Used after we have churned on some peers to find which ones have zero
1465  * connections so we can make them issue find peer requests.
1466  */
1467 static void
1468 count_peers_churn_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1469     const struct GNUNET_PeerIdentity *second, const char *emsg)
1470 {
1471   struct FindPeerContext *find_peer_context = cls;
1472   struct TopologyIteratorContext *topo_ctx;
1473   struct PeerCount *peer_count;
1474
1475   if ((first != NULL) && (second != NULL))
1476     {
1477       add_new_connection (find_peer_context, first, second);
1478       find_peer_context->current_peers++;
1479     }
1480   else
1481     {
1482       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1483                   "Peer count finished (%u connections)\n",
1484                   find_peer_context->current_peers);
1485       peer_count
1486           = GNUNET_CONTAINER_heap_peek (find_peer_context->peer_min_heap);
1487       GNUNET_assert(peer_count != NULL);
1488       /* WAIT. When peers are churned they will come back with their peers (at least in peerinfo), because the HOSTS file doesn't likely get removed. CRAP. */
1489       /* NO they won't, because we have disabled peerinfo writing to disk (remember?) so we WILL have to give them new connections */
1490       /* Best course of action: have DHT automatically try to add peers from peerinfo on startup. This way IF peerinfo writes to file
1491        * then some peers will end up connected.
1492        *
1493        * Also, find any peers that have zero connections here and set up a task to choose at random another peer in the network to
1494        * connect to.  Of course, if they are blacklisted from that peer they won't be able to connect, so we will have to keep trying
1495        * until they get a peer.
1496        */
1497       /* However, they won't automatically be connected to any of their previous peers... How can we handle that? */
1498       /* So now we have choices: do we want them to come back with all their connections?  Probably not, but it solves this mess. */
1499
1500       /* Second problem, which is still a problem, is that a FIND_PEER request won't work when a peer has no connections */
1501
1502       /**
1503        * Okay, so here's how this *should* work now.
1504        *
1505        * 1. We check the min heap for any peers that have 0 connections.
1506        *    a. If any are found, we iterate over the heap and just randomly
1507        *       choose another peer and ask testing to please connect the two.
1508        *       This takes care of the case that a peer just randomly joins the
1509        *       network.  However, if there are strict topology restrictions
1510        *       (imagine a ring) choosing randomly most likely won't help.
1511        *       We make sure the connection attempt doesn't take longer than
1512        *       the total timeout, but don't care too much about the result.
1513        *    b. After that, we still schedule the find peer requests (concurrently
1514        *       with the connect attempts most likely).  This handles the case
1515        *       that the DHT iterates over peerinfo and just needs to try to send
1516        *       a message to get connected.  This should handle the case that the
1517        *       topology is very strict.
1518        *
1519        * 2. If all peers have > 0 connections, we still send find peer requests
1520        *    as long as possible (until timeout is reached) to help out those
1521        *    peers that were newly churned and need more connections.  This is because
1522        *    once all new peers have established a single connection, they won't be
1523        *    well connected.
1524        *
1525        * 3. Once we reach the timeout, we can do no more.  We must schedule the
1526        *    next iteration of get requests regardless of connections that peers
1527        *    may or may not have.
1528        *
1529        * Caveat: it would be nice to get peers to take data offline with them and
1530        *         come back with it (or not) based on the testing framework.  The
1531        *         same goes for remembering previous connections, but putting either
1532        *         into the general testing churn options seems like overkill because
1533        *         these are very specialized cases.
1534        */
1535       GNUNET_log (
1536                   GNUNET_ERROR_TYPE_WARNING,
1537                   "Out of %u peers, fewest connections is %d\n",
1538                   GNUNET_CONTAINER_heap_get_size (
1539                                                   find_peer_context->peer_min_heap),
1540                   peer_count->count);
1541       if ((peer_count->count == 0)
1542           && (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1543               > 0))
1544         {
1545           GNUNET_log (
1546                       GNUNET_ERROR_TYPE_WARNING,
1547                       "Found peer with no connections, will choose some peer(s) at random to connect to!\n");
1548           GNUNET_CONTAINER_heap_iterate (find_peer_context->peer_min_heap,
1549                                          &iterate_min_heap_peers,
1550                                          find_peer_context);
1551           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1552                                     find_peer_context);
1553         }
1554       else if ((GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1555           > 0) && (find_peer_context->last_sent != 0))
1556         {
1557           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1558                                     find_peer_context);
1559         }
1560       else
1561         {
1562           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
1563                                                  &remove_peer_count,
1564                                                  find_peer_context);
1565           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
1566           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
1567           GNUNET_free(find_peer_context);
1568           GNUNET_log (
1569                       GNUNET_ERROR_TYPE_WARNING,
1570                       "Churn round %u of %llu finished, scheduling next GET round.\n",
1571                       current_churn_round, churn_rounds);
1572           if (dhtlog_handle != NULL)
1573             {
1574               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1575               topo_ctx->cont = &do_get;
1576               topo_ctx->cls = all_gets;
1577               topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1578               topo_ctx->peers_seen
1579                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1580               die_task
1581                   = GNUNET_SCHEDULER_add_delayed (
1582                                                   GNUNET_TIME_relative_add (
1583                                                                             GNUNET_TIME_relative_add (
1584                                                                                                       DEFAULT_GET_TIMEOUT,
1585                                                                                                       all_get_timeout),
1586                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1587                                                   &end_badly,
1588                                                   "from do gets (count_peers_churn_cb)");
1589               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1590             }
1591           else
1592             {
1593               die_task
1594                   = GNUNET_SCHEDULER_add_delayed (
1595                                                   GNUNET_TIME_relative_add (
1596                                                                             GNUNET_TIME_relative_add (
1597                                                                                                       DEFAULT_GET_TIMEOUT,
1598                                                                                                       all_get_timeout),
1599                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1600                                                   &end_badly,
1601                                                   "from do gets (count_peers_churn_cb)");
1602               GNUNET_SCHEDULER_add_now (&do_get, all_gets);
1603             }
1604         }
1605     }
1606 }
1607
1608 /**
1609  * Set up a single find peer request for each peer in the topology.  Do this
1610  * until the settle time is over, limited by the number of outstanding requests
1611  * and the time allowed for each one!
1612  */
1613 static void
1614 schedule_churn_find_peer_requests(void *cls,
1615     const struct GNUNET_SCHEDULER_TaskContext * tc)
1616 {
1617   struct FindPeerContext *find_peer_ctx = cls;
1618   struct TestFindPeer *test_find_peer;
1619   struct PeerCount *peer_count;
1620   uint32_t i;
1621
1622   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
1623     find_peer_ctx->total = 1;
1624   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1625       < MIN_FIND_PEER_CUTOFF)
1626     find_peer_ctx->total = find_peer_ctx->total / 2;
1627   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1628       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
1629     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
1630         / 4);
1631   else
1632     find_peer_ctx->total = find_peer_ctx->last_sent * 4;
1633
1634   if (find_peer_ctx->total > max_outstanding_find_peers)
1635     find_peer_ctx->total = max_outstanding_find_peers;
1636
1637   find_peer_ctx->last_sent = find_peer_ctx->total;
1638   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1639               "Sending %u find peer messages (after churn)\n",
1640               find_peer_ctx->total);
1641
1642   if (find_peer_ctx->total > 0)
1643     find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
1644                                                     find_peer_ctx->total);
1645   else
1646     {
1647       find_peer_ctx->previous_peers = find_peer_ctx->current_peers;
1648       find_peer_ctx->current_peers = 0;
1649       GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_ctx);
1650     }
1651
1652   for (i = 0; i < find_peer_ctx->total; i++)
1653     {
1654       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
1655       /* If we have sent requests, choose peers with a low number of connections to send requests from */
1656       peer_count
1657           = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
1658       GNUNET_assert(peer_count != NULL);
1659       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1660                   "Sending find peer request from peer with %u connections\n",
1661                   peer_count->count);
1662       GNUNET_CONTAINER_multihashmap_remove (find_peer_ctx->peer_hash,
1663                                             &peer_count->peer_id.hashPubKey,
1664                                             peer_count);
1665       test_find_peer->daemon
1666           = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1667       GNUNET_assert(test_find_peer->daemon != NULL);
1668       test_find_peer->find_peer_context = find_peer_ctx;
1669       GNUNET_SCHEDULER_add_delayed (
1670                                     GNUNET_TIME_relative_multiply (
1671                                                                    find_peer_offset,
1672                                                                    i),
1673                                     &send_find_peer_request, test_find_peer);
1674     }
1675
1676   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
1677       == NULL))
1678     {
1679       find_peer_ctx->peer_hash
1680           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1681       find_peer_ctx->peer_min_heap
1682           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1683     }
1684   else
1685     {
1686       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
1687                                              &remove_peer_count, find_peer_ctx);
1688       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
1689       find_peer_ctx->peer_hash
1690           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1691     }
1692
1693   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
1694   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
1695 }
1696
1697 static void
1698 schedule_churn_get_topology(void *cls,
1699     const struct GNUNET_SCHEDULER_TaskContext * tc)
1700 {
1701   struct FindPeerContext *find_peer_context = cls;
1702   GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_context);
1703 }
1704
1705 /**
1706  * Called when churning of the topology has finished.
1707  *
1708  * @param cls closure unused
1709  * @param emsg NULL on success, or a printable error on failure
1710  */
1711 static void
1712 churn_complete(void *cls, const char *emsg)
1713 {
1714   struct FindPeerContext *find_peer_context = cls;
1715   struct PeerCount *peer_count;
1716   unsigned int i;
1717   struct GNUNET_TESTING_Daemon *temp_daemon;
1718   struct TopologyIteratorContext *topo_ctx;
1719   struct GNUNET_TIME_Relative calc_timeout;
1720   int count_added;
1721
1722   if (emsg != NULL)
1723     {
1724       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1725                   "Ending test, churning of peers failed with error `%s'", emsg);
1726       GNUNET_SCHEDULER_add_now (&end_badly, (void *) emsg);
1727       return;
1728     }
1729
1730   /**
1731    * If we switched any peers on, we have to somehow force connect the new peer to
1732    * SOME bootstrap peer in the network.  First schedule a task to find all peers
1733    * with no connections, then choose a random peer for each and connect them.
1734    */
1735   if (find_peer_context != NULL)
1736     {
1737       GNUNET_log (
1738                   GNUNET_ERROR_TYPE_WARNING,
1739                   "We have churned on some peers, so we must schedule find peer requests for them!\n");
1740       count_added = 0;
1741       for (i = 0; i < num_peers; i++)
1742         {
1743           temp_daemon = GNUNET_TESTING_daemon_get (pg, i);
1744           if (GNUNET_YES == GNUNET_TESTING_daemon_running (temp_daemon))
1745             {
1746               peer_count = GNUNET_malloc (sizeof(struct PeerCount));
1747               memcpy (&peer_count->peer_id, &temp_daemon->id,
1748                       sizeof(struct GNUNET_PeerIdentity));
1749               GNUNET_assert(peer_count->count == 0);
1750               peer_count->heap_node
1751                   = GNUNET_CONTAINER_heap_insert (
1752                                                   find_peer_context->peer_min_heap,
1753                                                   peer_count, peer_count->count);
1754               GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1755                                                  &temp_daemon->id.hashPubKey,
1756                                                  peer_count,
1757                                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1758               count_added++;
1759             }
1760         }
1761       GNUNET_log (
1762                   GNUNET_ERROR_TYPE_WARNING,
1763                   "Added %d peers to heap, total size %d\n",
1764                   count_added,
1765                   GNUNET_CONTAINER_heap_get_size (
1766                                                   find_peer_context->peer_min_heap));
1767       GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1768                                     &schedule_churn_get_topology,
1769                                     find_peer_context);
1770     }
1771   else
1772     {
1773       GNUNET_log (
1774                   GNUNET_ERROR_TYPE_WARNING,
1775                   "Only churned off peers, no find peer requests, scheduling more gets (after allowing time for peers to disconnect properly!)...\n");
1776       if (dhtlog_handle != NULL)
1777         {
1778           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1779           topo_ctx->cont = &do_get;
1780           topo_ctx->cls = all_gets;
1781           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1782           topo_ctx->peers_seen
1783               = GNUNET_CONTAINER_multihashmap_create (num_peers);
1784           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1785                                                    all_get_timeout);
1786           calc_timeout
1787               = GNUNET_TIME_relative_add (calc_timeout,
1788                                           DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT);
1789           calc_timeout
1790               = GNUNET_TIME_relative_add (calc_timeout,
1791                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1792           die_task
1793               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1794                                               "from do gets (churn_complete)");
1795           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1796                                         &capture_current_topology, topo_ctx);
1797           dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1798         }
1799       else
1800         {
1801           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1802                                                    all_get_timeout);
1803           calc_timeout
1804               = GNUNET_TIME_relative_add (calc_timeout,
1805                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1806           die_task
1807               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1808                                               "from do gets (churn_complete)");
1809           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1810                                         &do_get, all_gets);
1811         }
1812     }
1813 }
1814
1815 /**
1816  * Decide how many peers to turn on or off in this round, make sure the
1817  * numbers actually make sense, then do so.  This function sets in motion
1818  * churn, find peer requests for newly joined peers, and issuing get
1819  * requests once the new peers have done so.
1820  *
1821  * @param cls closure (unused)
1822  * @param tc task context (unused)
1823  */
1824 static void
1825 churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1826 {
1827   unsigned int count_running;
1828   unsigned int churn_up;
1829   unsigned int churn_down;
1830   struct GNUNET_TIME_Relative timeout;
1831   struct FindPeerContext *find_peer_context;
1832
1833   churn_up = churn_down = 0;
1834   count_running = GNUNET_TESTING_daemons_running (pg);
1835   if (count_running > churn_array[current_churn_round])
1836     churn_down = count_running - churn_array[current_churn_round];
1837   else if (count_running < churn_array[current_churn_round])
1838     churn_up = churn_array[current_churn_round] - count_running;
1839   else
1840     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1841                 "Not churning any peers, topology unchanged.\n");
1842
1843   if (churn_up > num_peers - count_running)
1844     {
1845       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1846                   "Churn file specified %u peers (up); only have %u!",
1847                   churn_array[current_churn_round], num_peers);
1848       churn_up = num_peers - count_running;
1849     }
1850   else if (churn_down > count_running)
1851     {
1852       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1853                   "Churn file specified %u peers (down); only have %u!",
1854                   churn_array[current_churn_round], count_running);
1855       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1856                   "This will leave NO peers running (mistake in churn configuration?)!");
1857       churn_down = count_running;
1858     }
1859   //timeout = GNUNET_TIME_relative_multiply(seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1860   //timeout = GNUNET_TIME_relative_multiply (seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1861   timeout = GNUNET_TIME_relative_multiply (DEFAULT_TIMEOUT, 2); /* FIXME: Lack of intelligent choice here */
1862   find_peer_context = NULL;
1863   if (churn_up > 0) /* Only need to do find peer requests if we turned new peers on */
1864     {
1865       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
1866       find_peer_context->count_peers_cb = &count_peers_churn_cb;
1867       find_peer_context->previous_peers = 0;
1868       find_peer_context->current_peers = 0;
1869       find_peer_context->endtime = GNUNET_TIME_relative_to_absolute (timeout);
1870       find_peer_context->peer_hash
1871           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1872       find_peer_context->peer_min_heap
1873           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1874     }
1875   GNUNET_log (
1876               GNUNET_ERROR_TYPE_WARNING,
1877               "churn_peers: want %u total, %u running, starting %u, stopping %u\n",
1878               churn_array[current_churn_round], count_running, churn_up,
1879               churn_down);
1880   GNUNET_TESTING_daemons_churn (pg, churn_down, churn_up, timeout,
1881                                 &churn_complete, find_peer_context);
1882   current_churn_round++;
1883 }
1884
1885 /**
1886  * Task to release DHT handle associated with GET request.
1887  */
1888 static void
1889 get_stop_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1890 {
1891   struct TestGetContext *test_get = cls;
1892   struct TopologyIteratorContext *topo_ctx;
1893
1894   /* The dht_handle may be null if this get was scheduled from a down peer */
1895   if (test_get->dht_handle != NULL)
1896     {
1897       GNUNET_DHT_disconnect (test_get->dht_handle);
1898       outstanding_gets--; /* GET is really finished */
1899       test_get->dht_handle = NULL;
1900     }
1901
1902   /* Reset the uid (which item to search for) and the daemon (which peer to search from) for later get request iterations */
1903   if (get_from_same == GNUNET_NO)
1904     {
1905       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1906                                                 num_puts);
1907       test_get->daemon
1908           = GNUNET_TESTING_daemon_get (
1909                                        pg,
1910                                        GNUNET_CRYPTO_random_u32 (
1911                                                                  GNUNET_CRYPTO_QUALITY_WEAK,
1912                                                                  num_peers));
1913     }
1914
1915 #if VERBOSE > 1
1916   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
1917 #endif
1918   update_meter (get_meter);
1919   if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0))
1920     {
1921       fprintf (
1922                stderr,
1923                "Canceling die task (get_stop_finished) %llu gets completed, %llu gets failed\n",
1924                gets_completed, gets_failed);
1925       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
1926         dhtlog_handle->insert_round_details (DHT_ROUND_GET, rounds_finished,
1927                                              num_gets, gets_completed);
1928       GNUNET_SCHEDULER_cancel (die_task);
1929       reset_meter (put_meter);
1930       reset_meter (get_meter);
1931       /**
1932        *  Handle all cases:
1933        *    1) Testing is completely finished, call the topology iteration dealy and die
1934        *    2) Testing is not finished, churn the network and do gets again (current_churn_round < churn_rounds)
1935        *    3) Testing is not finished, reschedule all the PUTS *and* GETS again (num_rounds > 1)
1936        */
1937       if (rounds_finished == total_rounds - 1) /* Everything is finished, end testing */
1938         {
1939           if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
1940             {
1941               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1942               topo_ctx->cont = &log_dht_statistics;
1943               topo_ctx->peers_seen
1944                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1945               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1946             }
1947           else
1948             GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1949         }
1950       else if (current_churn_round < churns_per_round * (rounds_finished + 1)) /* Do next round of churn */
1951         {
1952           GNUNET_log (
1953                       GNUNET_ERROR_TYPE_WARNING,
1954                       "Current churn round %u, real round %u, scheduling next round of churn.\n",
1955                       current_churn_round, rounds_finished + 1);
1956           gets_completed = 0;
1957           gets_failed = 0;
1958
1959           if (dhtlog_handle != NULL)
1960             dhtlog_handle->insert_round (DHT_ROUND_CHURN, rounds_finished);
1961
1962           GNUNET_SCHEDULER_add_now (&churn_peers, NULL);
1963         }
1964       else if (rounds_finished < total_rounds - 1) /* Start a new complete round */
1965         {
1966           rounds_finished++;
1967           gets_completed = 0;
1968           gets_failed = 0;
1969           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1970                       "Round %u of %llu finished, scheduling next round.\n",
1971                       rounds_finished, total_rounds);
1972
1973           /** We reset the peer daemon for puts and gets on each disconnect, so all we need to do is start another round! */
1974           if (GNUNET_YES == in_dht_replication) /* Replication done in DHT, don't redo puts! */
1975             {
1976               if (dhtlog_handle != NULL)
1977                 dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1978
1979               die_task
1980                   = GNUNET_SCHEDULER_add_delayed (
1981                                                   GNUNET_TIME_relative_add (
1982                                                                             GNUNET_TIME_relative_add (
1983                                                                                                       GNUNET_TIME_relative_multiply (
1984                                                                                                                                      GNUNET_TIME_UNIT_SECONDS,
1985                                                                                                                                      round_delay),
1986                                                                                                       all_get_timeout),
1987                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1988                                                   &end_badly,
1989                                                   "from do gets (next round)");
1990               GNUNET_SCHEDULER_add_delayed (
1991                                             GNUNET_TIME_relative_multiply (
1992                                                                            GNUNET_TIME_UNIT_SECONDS,
1993                                                                            round_delay),
1994                                             &do_get, all_gets);
1995             }
1996           else
1997             {
1998               if (dhtlog_handle != NULL)
1999                 dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2000               die_task
2001                   = GNUNET_SCHEDULER_add_delayed (
2002                                                   GNUNET_TIME_relative_add (
2003                                                                             GNUNET_TIME_relative_multiply (
2004                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2005                                                                                                            round_delay),
2006                                                                             GNUNET_TIME_relative_multiply (
2007                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2008                                                                                                            num_puts
2009                                                                                                                * 2)),
2010                                                   &end_badly, "from do puts");
2011               GNUNET_SCHEDULER_add_delayed (
2012                                             GNUNET_TIME_relative_multiply (
2013                                                                            GNUNET_TIME_UNIT_SECONDS,
2014                                                                            round_delay),
2015                                             &do_put, all_puts);
2016             }
2017         }
2018     }
2019 }
2020
2021 /**
2022  * Task to release get handle.
2023  */
2024 static void
2025 get_stop_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2026 {
2027   struct TestGetContext *test_get = cls;
2028
2029   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2030     gets_failed++;
2031   else
2032     cumulative_successful_gets++;
2033
2034   GNUNET_assert(test_get->get_handle != NULL);
2035   GNUNET_DHT_get_stop (test_get->get_handle);
2036   test_get->get_handle = NULL;
2037   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2038   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2039 }
2040
2041 /**
2042  * Iterator called if the GET request initiated returns a response.
2043  *
2044  * @param cls closure
2045  * @param exp when will this value expire
2046  * @param key key of the result
2047  * @param get_path NULL-terminated array of pointers
2048  *                 to the peers on reverse GET path (or NULL if not recorded)
2049  * @param put_path NULL-terminated array of pointers
2050  *                 to the peers on the PUT path (or NULL if not recorded)
2051  * @param type type of the result
2052  * @param size number of bytes in data
2053  * @param data pointer to the result data
2054  */
2055 static void
2056 get_result_iterator(void *cls, struct GNUNET_TIME_Absolute exp,
2057     const GNUNET_HashCode * key,
2058     const struct GNUNET_PeerIdentity * const *get_path,
2059     const struct GNUNET_PeerIdentity * const *put_path,
2060     enum GNUNET_BLOCK_Type type, size_t size, const void *data)
2061 {
2062   struct TestGetContext *test_get = cls;
2063
2064   if (test_get->succeeded == GNUNET_YES)
2065     return; /* Get has already been successful, probably ending now */
2066
2067   if (0 != memcmp (&known_keys[test_get->uid], key, sizeof(GNUNET_HashCode))) /* || (0 != memcmp(original_data, data, sizeof(original_data))))*/
2068     {
2069       gets_completed++;
2070       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2071                   "Key or data is not the same as was inserted!\n");
2072     }
2073   else
2074     {
2075       gets_completed++;
2076       test_get->succeeded = GNUNET_YES;
2077     }
2078 #if VERBOSE > 1
2079   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
2080 #endif
2081   GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
2082   GNUNET_SCHEDULER_add_continuation (&get_stop_task, test_get,
2083                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
2084 }
2085
2086 /**
2087  * Set up some data, and call API PUT function
2088  */
2089 static void
2090 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2091 {
2092   struct TestGetContext *test_get = cls;
2093
2094   if (num_gets == 0)
2095     {
2096       GNUNET_SCHEDULER_cancel (die_task);
2097       GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
2098     }
2099
2100   if (test_get == NULL)
2101     return; /* End of the list */
2102
2103   /* Set this here in case we are re-running gets */
2104   test_get->succeeded = GNUNET_NO;
2105
2106   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_get->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2107     {
2108       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2109                   "Peer we should issue get request from is down, skipping.\n");
2110       gets_failed++;
2111       GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2112       GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2113       return;
2114     }
2115
2116   /* Check if more gets are outstanding than should be */
2117   if (outstanding_gets > max_outstanding_gets)
2118     {
2119       GNUNET_SCHEDULER_add_delayed (
2120                                     GNUNET_TIME_relative_multiply (
2121                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2122                                                                    200),
2123                                     &do_get, test_get);
2124       return;
2125     }
2126
2127   /* Connect to the first peer's DHT */
2128   test_get->dht_handle = GNUNET_DHT_connect (test_get->daemon->cfg, 10);
2129   GNUNET_assert(test_get->dht_handle != NULL);
2130   outstanding_gets++;
2131
2132   cumulative_num_gets++;
2133   /* Insert the data at the first peer */
2134   test_get->get_handle = GNUNET_DHT_get_start (test_get->dht_handle, get_delay,
2135                                                GNUNET_BLOCK_TYPE_TEST,
2136                                                &known_keys[test_get->uid],
2137                                                get_replication,
2138                                                GNUNET_DHT_RO_NONE, NULL, 0,
2139                                                NULL, 0, &get_result_iterator,
2140                                                test_get);
2141
2142 #if VERBOSE > 1
2143   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
2144       test_get->uid,
2145       test_get->daemon->shortname);
2146 #endif
2147   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed (get_timeout,
2148                                                             &get_stop_task,
2149                                                             test_get);
2150
2151   /* Schedule the next request in the linked list of get requests */
2152   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2153 }
2154
2155 /**
2156  * Called when the PUT request has been transmitted to the DHT service.
2157  * Schedule the GET request for some time in the future.
2158  */
2159 static void
2160 put_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2161 {
2162   struct TestPutContext *test_put = cls;
2163   struct TopologyIteratorContext *topo_ctx;
2164   outstanding_puts--;
2165   puts_completed++;
2166
2167   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2168     fprintf (stderr, "PUT Request failed!\n");
2169
2170   /* Reset the daemon (which peer to insert at) for later put request iterations */
2171   if (replicate_same == GNUNET_NO)
2172     test_put->daemon
2173         = GNUNET_TESTING_daemon_get (
2174                                      pg,
2175                                      GNUNET_CRYPTO_random_u32 (
2176                                                                GNUNET_CRYPTO_QUALITY_WEAK,
2177                                                                num_peers));
2178
2179   GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
2180   test_put->disconnect_task = GNUNET_SCHEDULER_add_now (&put_disconnect_task,
2181                                                         test_put);
2182   if (GNUNET_YES == update_meter (put_meter))
2183     {
2184       GNUNET_assert(outstanding_puts == 0);
2185       GNUNET_SCHEDULER_cancel (die_task);
2186       if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2187         {
2188           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2189           topo_ctx->cont = &do_get;
2190           topo_ctx->cls = all_gets;
2191           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
2192           topo_ctx->peers_seen
2193               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2194           die_task
2195               = GNUNET_SCHEDULER_add_delayed (
2196                                               GNUNET_TIME_relative_add (
2197                                                                         GNUNET_TIME_relative_add (
2198                                                                                                   DEFAULT_GET_TIMEOUT,
2199                                                                                                   all_get_timeout),
2200                                                                         DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2201                                               &end_badly,
2202                                               "from do gets (put finished)");
2203           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
2204         }
2205       else
2206         {
2207           fprintf (stderr, "Scheduling die task (put finished)\n");
2208           die_task
2209               = GNUNET_SCHEDULER_add_delayed (
2210                                               GNUNET_TIME_relative_add (
2211                                                                         DEFAULT_GET_TIMEOUT,
2212                                                                         all_get_timeout),
2213                                               &end_badly,
2214                                               "from do gets (put finished)");
2215           GNUNET_SCHEDULER_add_delayed (DEFAULT_GET_TIMEOUT, &do_get, all_gets);
2216         }
2217       return;
2218     }
2219 }
2220
2221 /**
2222  * Set up some data, and call API PUT function
2223  */
2224 static void
2225 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2226 {
2227   struct TestPutContext *test_put = cls;
2228   char data[test_data_size]; /* Made up data to store */
2229   uint32_t rand;
2230   int i;
2231
2232   if (test_put == NULL)
2233     return; /* End of list */
2234
2235   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_put->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2236     {
2237       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2238                   "Peer we should issue put request at is down, skipping.\n");
2239       update_meter (put_meter);
2240       GNUNET_SCHEDULER_add_now (&do_put, test_put->next);
2241       return;
2242     }
2243
2244   for (i = 0; i < sizeof(data); i++)
2245     {
2246       memset (&data[i], GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2247                                                   UINT32_MAX), 1);
2248     }
2249
2250   if (outstanding_puts > max_outstanding_puts)
2251     {
2252       GNUNET_SCHEDULER_add_delayed (
2253                                     GNUNET_TIME_relative_multiply (
2254                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2255                                                                    200),
2256                                     &do_put, test_put);
2257       return;
2258     }
2259
2260 #if VERBOSE > 1
2261   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
2262       test_put->uid,
2263       test_put->daemon->shortname);
2264 #endif
2265   test_put->dht_handle = GNUNET_DHT_connect (test_put->daemon->cfg, 10);
2266
2267   GNUNET_assert(test_put->dht_handle != NULL);
2268   outstanding_puts++;
2269   GNUNET_DHT_put (test_put->dht_handle, &known_keys[test_put->uid],
2270                   put_replication, GNUNET_DHT_RO_NONE, GNUNET_BLOCK_TYPE_TEST,
2271                   sizeof(data), data, GNUNET_TIME_UNIT_FOREVER_ABS, put_delay,
2272                   &put_finished, test_put);
2273   test_put->disconnect_task
2274       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever (),
2275                                       &put_disconnect_task, test_put);
2276   rand = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2);
2277   GNUNET_SCHEDULER_add_delayed (
2278                                 GNUNET_TIME_relative_multiply (
2279                                                                GNUNET_TIME_UNIT_SECONDS,
2280                                                                rand), &do_put,
2281                                 test_put->next);
2282 }
2283
2284 static void
2285 schedule_find_peer_requests(void *cls,
2286     const struct GNUNET_SCHEDULER_TaskContext * tc);
2287
2288 static void
2289     setup_malicious_peers(void *cls,
2290         const struct GNUNET_SCHEDULER_TaskContext * tc);
2291
2292 /**
2293  * Given a number of total peers and a bucket size, estimate the number of
2294  * connections in a perfect kademlia topology.
2295  */
2296 static unsigned int
2297 connection_estimate(unsigned int peer_count, unsigned int bucket_size)
2298 {
2299   unsigned int i;
2300   unsigned int filled;
2301   i = num_peers;
2302
2303   filled = 0;
2304   while (i >= bucket_size)
2305     {
2306       filled++;
2307       i = i / 2;
2308     }
2309   filled++; /* Add one filled bucket to account for one "half full" and some miscellaneous */
2310   return filled * bucket_size * peer_count;
2311
2312 }
2313
2314 /**
2315  * Callback for iterating over all the peer connections of a peer group.
2316  */
2317 static void
2318 count_peers_cb(void *cls, const struct GNUNET_PeerIdentity *first,
2319     const struct GNUNET_PeerIdentity *second, const char *emsg)
2320 {
2321   struct FindPeerContext *find_peer_context = cls;
2322   if ((first != NULL) && (second != NULL))
2323     {
2324       add_new_connection (find_peer_context, first, second);
2325       find_peer_context->current_peers++;
2326     }
2327   else
2328     {
2329       GNUNET_log (
2330                   GNUNET_ERROR_TYPE_WARNING,
2331                   "Peer count finished (%u connections), %u new peers, connection estimate %u (target %u)\n",
2332                   find_peer_context->current_peers,
2333                   find_peer_context->current_peers
2334                       - find_peer_context->previous_peers,
2335                   connection_estimate (num_peers, DEFAULT_BUCKET_SIZE),
2336                   target_total_connections);
2337
2338       if ((find_peer_context->last_sent < 8)
2339           || ((find_peer_context->current_peers < 2
2340               * connection_estimate (num_peers, DEFAULT_BUCKET_SIZE))
2341               && (GNUNET_TIME_absolute_get_remaining (
2342                                                       find_peer_context->endtime).rel_value
2343                   > 0) && (find_peer_context->current_peers
2344               < target_total_connections)))
2345         {
2346           GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests,
2347                                     find_peer_context);
2348         }
2349       else
2350         {
2351           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
2352                                                  &remove_peer_count,
2353                                                  find_peer_context);
2354           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
2355           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
2356           GNUNET_free(find_peer_context);
2357           fprintf (stderr, "Not sending any more find peer requests.\n");
2358
2359 #if HAVE_MALICIOUS
2360           if (GNUNET_YES == malicious_after_settle)
2361             {
2362               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "calling setup_malicious_peers\n");
2363               GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2364             }
2365 #endif
2366         }
2367     }
2368 }
2369
2370 /**
2371  * Set up a single find peer request for each peer in the topology.  Do this
2372  * until the settle time is over, limited by the number of outstanding requests
2373  * and the time allowed for each one!
2374  */
2375 static void
2376 schedule_find_peer_requests(void *cls,
2377     const struct GNUNET_SCHEDULER_TaskContext * tc)
2378 {
2379   struct FindPeerContext *find_peer_ctx = cls;
2380   struct TestFindPeer *test_find_peer;
2381   struct PeerCount *peer_count;
2382   uint32_t i;
2383   uint32_t random;
2384
2385   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
2386     find_peer_ctx->total = 1;
2387   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
2388       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
2389     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
2390         / 8);
2391   else
2392     find_peer_ctx->total = find_peer_ctx->last_sent * 2;
2393
2394   if (find_peer_ctx->total > max_outstanding_find_peers)
2395     find_peer_ctx->total = max_outstanding_find_peers;
2396
2397   if (find_peer_ctx->total > num_peers) /* Don't try to send more messages than we have peers! */
2398     find_peer_ctx->total = num_peers;
2399
2400   find_peer_ctx->last_sent = find_peer_ctx->total;
2401   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2402               "Sending %u find peer messages (goal at least %u connections)\n",
2403               find_peer_ctx->total, target_total_connections);
2404
2405   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
2406                                                   find_peer_ctx->total);
2407   for (i = 0; i < find_peer_ctx->total; i++)
2408     {
2409       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
2410       /* If we haven't sent any requests yet, choose random peers */
2411       /* Also choose random in _half_ of all cases, so we don't
2412        * get stuck choosing topologically restricted peers with
2413        * few connections that will never be able to find any new
2414        * peers! */
2415       if ((find_peer_ctx->previous_peers == 0) || (i % 2 == 0))
2416         {
2417           /**
2418            * Attempt to spread find peer requests across even sections of the peer address
2419            * space.  Choose basically 1 peer in every num_peers / max_outstanding_requests
2420            * each time, then offset it by a randomish value.
2421            *
2422            * For instance, if num_peers is 100 and max_outstanding is 10, first chosen peer
2423            * will be between 0 - 10, second between 10 - 20, etc.
2424            */
2425           random = (num_peers / find_peer_ctx->total) * i;
2426           random = random
2427               + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2428                                           (num_peers / find_peer_ctx->total));
2429           if (random >= num_peers)
2430             {
2431               random = random - num_peers;
2432             }
2433 #if REAL_RANDOM
2434           random = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2435 #endif
2436           test_find_peer->daemon = GNUNET_TESTING_daemon_get (pg, random);
2437         }
2438       else /* If we have sent requests, choose peers with a low number of connections to send requests from */
2439         {
2440           peer_count
2441               = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
2442           GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(find_peer_ctx->peer_hash, &peer_count->peer_id.hashPubKey, peer_count));
2443           test_find_peer->daemon
2444               = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
2445           GNUNET_assert(test_find_peer->daemon != NULL);
2446         }
2447
2448       test_find_peer->find_peer_context = find_peer_ctx;
2449       GNUNET_SCHEDULER_add_delayed (
2450                                     GNUNET_TIME_relative_multiply (
2451                                                                    find_peer_offset,
2452                                                                    i),
2453                                     &send_find_peer_request, test_find_peer);
2454     }
2455
2456   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
2457       == NULL))
2458     {
2459       find_peer_ctx->peer_hash
2460           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2461       find_peer_ctx->peer_min_heap
2462           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2463     }
2464   else
2465     {
2466       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
2467                                              &remove_peer_count, find_peer_ctx);
2468       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
2469       find_peer_ctx->peer_hash
2470           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2471     }
2472
2473   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
2474   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
2475
2476 }
2477
2478 /**
2479  * Convert unique ID to hash code.
2480  *
2481  * @param uid unique ID to convert
2482  * @param hash set to uid (extended with zeros)
2483  */
2484 static void
2485 hash_from_uid(uint32_t uid, GNUNET_HashCode *hash)
2486 {
2487   memset (hash, 0, sizeof(GNUNET_HashCode));
2488   *((uint32_t *) hash) = uid;
2489 }
2490
2491 /**
2492  * Set up all of the put and get operations we want to do
2493  * in the current round.  Allocate data structure for each,
2494  * add to list, then schedule the actual PUT operations.
2495  */
2496 static void
2497 setup_puts_and_gets(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2498 {
2499   int i;
2500   struct TestPutContext *test_put;
2501   struct TestGetContext *test_get;
2502   uint32_t temp_peer;
2503   GNUNET_HashCode uid_hash;
2504   int count;
2505 #if REMEMBER
2506   int remember[num_puts][num_peers];
2507   memset(&remember, 0, sizeof(int) * num_puts * num_peers);
2508 #endif
2509   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "in setup_puts_and_gets\n");
2510   known_keys = GNUNET_malloc(sizeof(GNUNET_HashCode) * num_puts);
2511   for (i = 0; i < num_puts; i++)
2512     {
2513       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
2514       test_put->uid = i;
2515       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
2516                                         &known_keys[i]);
2517       /* Set first X bits to match the chosen sybil location if we want to do the sybil attack! */
2518       if (GNUNET_YES == malicious_sybil)
2519         {
2520           memcpy (&known_keys[i], &sybil_target, sizeof(GNUNET_HashCode) / 2);
2521           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2522                       "Distance between sybil location and key is %d\n",
2523                       GNUNET_CRYPTO_hash_matching_bits (&known_keys[i],
2524                                                         &sybil_target));
2525         }
2526       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2527                                             num_peers);
2528       test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2529       /* Don't start PUTs at malicious peers! */
2530       if (malicious_bloom != NULL)
2531         {
2532           count = 0;
2533           hash_from_uid (temp_peer, &uid_hash);
2534           while ((GNUNET_YES
2535               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2536               && (count < num_peers))
2537             {
2538               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2539                                                     num_peers);
2540               hash_from_uid (temp_peer, &uid_hash);
2541               test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2542               count++;
2543             }
2544           if (count == num_peers)
2545             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2546                         "Couldn't find peer not in malicious bloom to select!\n");
2547         }
2548
2549       test_put->next = all_puts;
2550       all_puts = test_put;
2551     }
2552
2553   for (i = 0; i < num_gets; i++)
2554     {
2555       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
2556       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2557                                                 num_puts);
2558 #if REMEMBER
2559       while (remember[test_get->uid][temp_daemon] == 1)
2560       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2561       remember[test_get->uid][temp_daemon] = 1;
2562 #endif
2563       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2564                                             num_peers);
2565       test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2566       /* Don't start GETs at malicious peers! */
2567       if (malicious_bloom != NULL)
2568         {
2569           hash_from_uid (temp_peer, &uid_hash);
2570           count = 0;
2571           while ((GNUNET_YES
2572               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2573               && (count < num_peers))
2574             {
2575               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2576                                                     num_peers);
2577               hash_from_uid (temp_peer, &uid_hash);
2578               test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2579               count++;
2580             }
2581           if (count == num_peers)
2582             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2583                         "Couldn't find peer not in malicious bloom to select!\n");
2584         }
2585       test_get->next = all_gets;
2586       all_gets = test_get;
2587     }
2588
2589   /*GNUNET_SCHEDULER_cancel (die_task);*/
2590   die_task
2591       = GNUNET_SCHEDULER_add_delayed (
2592                                       GNUNET_TIME_relative_multiply (
2593                                                                      GNUNET_TIME_UNIT_SECONDS,
2594                                                                      num_puts
2595                                                                          * 2),
2596                                       &end_badly, "from do puts");
2597   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
2598
2599 }
2600
2601 /**
2602  * Set up some all of the put and get operations we want
2603  * to do.  Allocate data structure for each, add to list,
2604  * then call actual insert functions.
2605  */
2606 static void
2607 continue_puts_and_gets(void *cls,
2608     const struct GNUNET_SCHEDULER_TaskContext * tc)
2609 {
2610   int i;
2611   int max;
2612   struct TopologyIteratorContext *topo_ctx;
2613   struct FindPeerContext *find_peer_context;
2614   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "In continue_puts_and_gets\n");
2615   if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2616     {
2617       if (settle_time >= 180 * 2)
2618         max = (settle_time / 180) - 2;
2619       else
2620         max = 1;
2621       for (i = 1; i < max; i++)
2622         {
2623           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2624           topo_ctx->current_iteration = i;
2625           topo_ctx->total_iterations = max;
2626           topo_ctx->peers_seen
2627               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2628           //fprintf(stderr, "scheduled topology iteration in %d minutes\n", i);
2629           GNUNET_SCHEDULER_add_delayed (
2630                                         GNUNET_TIME_relative_multiply (
2631                                                                        GNUNET_TIME_UNIT_MINUTES,
2632                                                                        i * 3),
2633                                         &capture_current_topology, topo_ctx);
2634         }
2635       topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2636       topo_ctx->cont = &setup_puts_and_gets;
2637       topo_ctx->peers_seen = GNUNET_CONTAINER_multihashmap_create (num_peers);
2638       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2639                   "setting setup_puts_and_gets for %d seconds in the future\n",
2640                   settle_time + 10);
2641       GNUNET_SCHEDULER_add_delayed (
2642                                     GNUNET_TIME_relative_multiply (
2643                                                                    GNUNET_TIME_UNIT_SECONDS,
2644                                                                    (settle_time
2645                                                                        + 10)),
2646                                     &capture_current_topology, topo_ctx);
2647     }
2648   else
2649     GNUNET_SCHEDULER_add_delayed (
2650                                   GNUNET_TIME_relative_multiply (
2651                                                                  GNUNET_TIME_UNIT_SECONDS,
2652                                                                  (settle_time
2653                                                                      + 10)),
2654                                   &setup_puts_and_gets, NULL);
2655
2656   if (dhtlog_handle != NULL)
2657     dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2658
2659 #if HAVE_MALICIOUS
2660   if ((GNUNET_YES != malicious_after_settle) || (settle_time == 0))
2661     {
2662       GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2663     }
2664 #endif
2665
2666   if ((GNUNET_YES == do_find_peer) && (settle_time > 0))
2667     {
2668       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2669                   "Scheduling find peer requests during \"settle\" time.\n");
2670       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
2671       find_peer_context->count_peers_cb = &count_peers_cb;
2672       find_peer_context->endtime
2673           = GNUNET_TIME_relative_to_absolute (
2674                                               GNUNET_TIME_relative_multiply (
2675                                                                              GNUNET_TIME_UNIT_SECONDS,
2676                                                                              settle_time));
2677       GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests, find_peer_context);
2678     }
2679   else
2680     {
2681       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2682                   "Assuming automatic DHT find peer requests.\n");
2683     }
2684 }
2685
2686 /**
2687  * Task to release DHT handles
2688  */
2689 static void
2690 malicious_disconnect_task(void *cls,
2691                           const struct GNUNET_SCHEDULER_TaskContext * tc)
2692 {
2693   struct MaliciousContext *ctx = cls;
2694   outstanding_malicious--;
2695   malicious_completed++;
2696   ctx->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2697   GNUNET_DHT_disconnect (ctx->dht_handle);
2698   ctx->dht_handle = NULL;
2699   GNUNET_free(ctx);
2700
2701   if (malicious_completed == malicious_getters + malicious_putters
2702       + malicious_droppers)
2703     {
2704       fprintf (stderr, "Finished setting all malicious peers up!\n");
2705     }
2706 }
2707
2708 /**
2709  * Task to release DHT handles
2710  */
2711 static void
2712 malicious_done_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2713 {
2714   struct MaliciousContext *ctx = cls;
2715   GNUNET_SCHEDULER_cancel (ctx->disconnect_task);
2716   GNUNET_SCHEDULER_add_now (&malicious_disconnect_task, ctx);
2717 }
2718
2719 /**
2720  * Set up some data, and call API PUT function
2721  */
2722 static void
2723 set_malicious(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2724 {
2725   struct MaliciousContext *ctx = cls;
2726
2727   if (outstanding_malicious > DEFAULT_MAX_OUTSTANDING_GETS)
2728     {
2729       GNUNET_SCHEDULER_add_delayed (
2730                                     GNUNET_TIME_relative_multiply (
2731                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2732                                                                    100),
2733                                     &set_malicious, ctx);
2734       return;
2735     }
2736
2737   if (ctx->dht_handle == NULL)
2738     {
2739       ctx->dht_handle = GNUNET_DHT_connect (ctx->daemon->cfg, 1);
2740       outstanding_malicious++;
2741     }
2742
2743   GNUNET_assert(ctx->dht_handle != NULL);
2744
2745 #if VERBOSE > 1
2746   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Setting peer %s malicious type %d\n",
2747       ctx->daemon->shortname, ctx->malicious_type);
2748 #endif
2749
2750   switch (ctx->malicious_type)
2751     {
2752   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET:
2753     GNUNET_DHT_set_malicious_getter (ctx->dht_handle, malicious_get_frequency, &malicious_done_task, ctx);
2754     break;
2755   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT:
2756     GNUNET_DHT_set_malicious_putter (ctx->dht_handle, malicious_put_frequency, &malicious_done_task, ctx);
2757     break;
2758   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP:
2759     GNUNET_DHT_set_malicious_dropper (ctx->dht_handle, &malicious_done_task, ctx);
2760     break;
2761   default:
2762     break;
2763     }
2764
2765   ctx->disconnect_task
2766       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
2767                                       &malicious_disconnect_task, ctx);
2768 }
2769
2770 #if HAVE_MALICIOUS
2771 /**
2772  * Choose the next peer from the peer group to set as malicious.
2773  * If we are doing a sybil attack, find the nearest peer to the
2774  * sybil location that has not already been set malicious.  Otherwise
2775  * just choose a random not already chosen peer.
2776  *
2777  * @param pg the peer group
2778  * @param bloom the bloomfilter which contains all peer already
2779  *        chosen to be malicious
2780  */
2781 static uint32_t
2782 choose_next_malicious (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_CONTAINER_BloomFilter *bloom)
2783   {
2784     int i;
2785     int nearest;
2786     int bits_match;
2787     int curr_distance;
2788     int count;
2789     struct GNUNET_TESTING_Daemon *temp_daemon;
2790     GNUNET_HashCode uid_hash;
2791
2792     curr_distance = 0;
2793     nearest = 0;
2794     GNUNET_assert (bloom != NULL);
2795
2796     if (GNUNET_YES == malicious_sybil)
2797       {
2798         for (i = 0; i < num_peers; i++)
2799           {
2800             temp_daemon = GNUNET_TESTING_daemon_get(pg, i);
2801             hash_from_uid(i, &uid_hash);
2802             /* Check if this peer matches the bloomfilter */
2803             if ((GNUNET_NO == GNUNET_TESTING_daemon_running(temp_daemon)) || (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)))
2804             continue;
2805
2806             bits_match = GNUNET_CRYPTO_hash_matching_bits (&temp_daemon->id.hashPubKey, &sybil_target);
2807             if (bits_match >= curr_distance)
2808               {
2809                 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found nearer peer %s to %s, old matching bits %d, new %d\n", GNUNET_i2s(&temp_daemon->id), GNUNET_h2s(&sybil_target), curr_distance, bits_match);
2810                 nearest = i;
2811                 curr_distance = bits_match;
2812               }
2813           }
2814       }
2815     else
2816       {
2817         nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2818         hash_from_uid(nearest, &uid_hash);
2819         count = 0;
2820         while ((GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)) && (count < num_peers))
2821           {
2822             GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Peer %d already in bloom (tried %d times)\n", nearest, count);
2823             nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2824             hash_from_uid(nearest, &uid_hash);
2825             count++;
2826           }
2827         if (count == num_peers)
2828         GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Tried %d times to find a peer, selecting %d at random!!\n", count, nearest);
2829       }
2830
2831     return nearest;
2832   }
2833
2834 /**
2835  * Select randomly from set of known peers,
2836  * set the desired number of peers to the
2837  * proper malicious types.
2838  */
2839 static void
2840 setup_malicious_peers (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2841   {
2842     struct MaliciousContext *ctx;
2843     int i;
2844     uint32_t temp_daemon;
2845     GNUNET_HashCode uid_hash;
2846
2847     for (i = 0; i < malicious_getters; i++)
2848       {
2849         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2850         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2851         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2852         hash_from_uid(temp_daemon, &uid_hash);
2853         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2854         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET;
2855         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2856
2857       }
2858
2859     for (i = 0; i < malicious_putters; i++)
2860       {
2861         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2862         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2863         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2864         hash_from_uid(temp_daemon, &uid_hash);
2865         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2866         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT;
2867         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2868
2869       }
2870
2871     for (i = 0; i < malicious_droppers; i++)
2872       {
2873         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2874         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2875         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2876         hash_from_uid(temp_daemon, &uid_hash);
2877         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2878         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP;
2879         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2880       }
2881   }
2882 #endif
2883
2884 #if ONLY_TESTING
2885 /* Forward declaration */
2886 static void
2887 topology_callback (void *cls,
2888     const struct GNUNET_PeerIdentity *first,
2889     const struct GNUNET_PeerIdentity *second,
2890     uint32_t distance,
2891     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2892     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2893     struct GNUNET_TESTING_Daemon *first_daemon,
2894     struct GNUNET_TESTING_Daemon *second_daemon,
2895     const char *emsg);
2896
2897 /**
2898  * Retry connecting two specific peers until they connect,
2899  * at a specific interval.  These two peers previously failed
2900  * to connect, and we hope they continue to so that we can
2901  * debug the reason they are having issues.
2902  */
2903 static void
2904 repeat_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2905   {
2906
2907     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Repeating connect attempt between %s and %s.\n", repeat_connect_peer1->shortname, repeat_connect_peer2->shortname);
2908     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 1 configuration `%s'\n", repeat_connect_peer1->cfgfile);
2909     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 2 configuration `%s'\n", repeat_connect_peer2->cfgfile);
2910
2911     repeat_connect_task = GNUNET_SCHEDULER_NO_TASK;
2912     GNUNET_TESTING_daemons_connect(repeat_connect_peer1,
2913         repeat_connect_peer2,
2914         GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60),
2915         2, &topology_callback, NULL);
2916   }
2917 #endif
2918
2919 /**
2920  * This function is called whenever a connection attempt is finished between two of
2921  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
2922  * number of times this function is called should equal the number returned
2923  * from the GNUNET_TESTING_connect_topology call.
2924  *
2925  * The emsg variable is NULL on success (peers connected), and non-NULL on
2926  * failure (peers failed to connect).
2927  */
2928 static void
2929 topology_callback(void *cls, const struct GNUNET_PeerIdentity *first,
2930     const struct GNUNET_PeerIdentity *second, uint32_t distance,
2931     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2932     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2933     struct GNUNET_TESTING_Daemon *first_daemon,
2934     struct GNUNET_TESTING_Daemon *second_daemon, const char *emsg)
2935 {
2936   struct TopologyIteratorContext *topo_ctx;
2937   unsigned long long duration;
2938   unsigned long long total_duration;
2939   unsigned int new_connections;
2940   unsigned int new_failed_connections;
2941   double conns_per_sec_recent;
2942   double conns_per_sec_total;
2943   double failed_conns_per_sec_recent;
2944   double failed_conns_per_sec_total;
2945   char *temp_conn_string;
2946   char *temp_conn_failed_string;
2947   char *revision_str;
2948
2949 #if ONLY_TESTING
2950   if (repeat_connect_mode == GNUNET_YES)
2951     {
2952       if ((first_daemon == repeat_connect_peer1) &&
2953           (second_daemon == repeat_connect_peer2))
2954         {
2955           if (emsg != NULL) /* Peers failed to connect again! */
2956             {
2957               GNUNET_assert(repeat_connect_task == GNUNET_SCHEDULER_NO_TASK);
2958               repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
2959               return;
2960             }
2961           else /* Repeat peers actually connected! */
2962             {
2963               if (repeat_connect_task != GNUNET_SCHEDULER_NO_TASK)
2964               GNUNET_SCHEDULER_cancel(repeat_connect_task);
2965               repeat_connect_peer1 = NULL;
2966               repeat_connect_peer2 = NULL;
2967               repeat_connect_mode = GNUNET_NO;
2968               GNUNET_TESTING_resume_connections(pg);
2969               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Resuming normal connection mode, debug connection was successful!\n");
2970             }
2971         }
2972     }
2973 #endif
2974
2975   if (GNUNET_TIME_absolute_get_difference (connect_last_time,
2976                                            GNUNET_TIME_absolute_get ()).rel_value
2977       > GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
2978                                        CONN_UPDATE_DURATION).rel_value)
2979     {
2980       /* Get number of new connections */
2981       new_connections = total_connections - previous_connections;
2982
2983       /* Get number of new FAILED connections */
2984       new_failed_connections = failed_connections - previous_failed_connections;
2985
2986       /* Get duration in seconds */
2987       duration
2988           = GNUNET_TIME_absolute_get_difference (connect_last_time,
2989                                                  GNUNET_TIME_absolute_get ()).rel_value
2990               / 1000;
2991       total_duration
2992           = GNUNET_TIME_absolute_get_difference (connect_start_time,
2993                                                  GNUNET_TIME_absolute_get ()).rel_value
2994               / 1000;
2995
2996       failed_conns_per_sec_recent = (double) new_failed_connections / duration;
2997       failed_conns_per_sec_total = (double) failed_connections / total_duration;
2998       conns_per_sec_recent = (double) new_connections / duration;
2999       conns_per_sec_total = (double) total_connections / total_duration;
3000       GNUNET_log (
3001                   GNUNET_ERROR_TYPE_WARNING,
3002                   "Recent: %.2f/s, Total: %.2f/s, Recent failed: %.2f/s, total failed %.2f/s\n",
3003                   conns_per_sec_recent, CONN_UPDATE_DURATION,
3004                   conns_per_sec_total, failed_conns_per_sec_recent,
3005                   failed_conns_per_sec_total);
3006       connect_last_time = GNUNET_TIME_absolute_get ();
3007       previous_connections = total_connections;
3008       previous_failed_connections = failed_connections;
3009       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3010                   "have %u total_connections, %u failed\n", total_connections,
3011                   failed_connections);
3012 #if ONLY_TESTING
3013       /* These conditions likely mean we've entered the death spiral of doom */
3014       if ((total_connections > 20000) &&
3015           (conns_per_sec_recent < 5.0) &&
3016           (conns_per_sec_total > 10.0) &&
3017           (emsg != NULL) &&
3018           (repeat_connect_mode == GNUNET_NO))
3019         {
3020           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Entering repeat connection attempt mode!\n");
3021           repeat_connect_peer1 = first_daemon;
3022           repeat_connect_peer2 = second_daemon;
3023           repeat_connect_mode = GNUNET_YES;
3024           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping NEW connections from being scheduled!\n");
3025           GNUNET_TESTING_stop_connections(pg);
3026           repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
3027         }
3028 #endif
3029     }
3030
3031   if (emsg == NULL)
3032     {
3033       total_connections++;
3034 #if VERBOSE > 1
3035       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
3036           first_daemon->shortname,
3037           second_daemon->shortname,
3038           distance);
3039 #endif
3040     }
3041   else
3042     {
3043       failed_connections++;
3044 #if VERBOSE
3045       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peer %s to peer %s with error :\n%s\n",
3046           first_daemon->shortname,
3047           second_daemon->shortname, emsg);
3048
3049       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
3050           first_daemon->shortname,
3051           second_daemon->shortname, emsg);
3052 #endif
3053     }
3054
3055 #if ONLY_TESTING
3056   if ((repeat_connect_mode == GNUNET_YES) )
3057   return;
3058 #endif
3059
3060   GNUNET_assert(peer_connect_meter != NULL);
3061   if (GNUNET_YES == update_meter (peer_connect_meter))
3062     {
3063 #if VERBOSE
3064       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3065           "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
3066           total_connections);
3067 #endif
3068       if (failed_connections > 0)
3069         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3070                     "While connecting, had %u failed connections.\n",
3071                     failed_connections);
3072       if (dhtlog_handle != NULL)
3073         {
3074           dhtlog_handle->update_connections (total_connections);
3075           dhtlog_handle->insert_topology (expected_connections);
3076         }
3077
3078       total_duration
3079           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3080                                                  GNUNET_TIME_absolute_get ()).rel_value
3081               / 1000;
3082       failed_conns_per_sec_total = (double) failed_connections / total_duration;
3083       conns_per_sec_total = (double) total_connections / total_duration;
3084       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3085                   "Overall connection info --- Total: %u, Total Failed %u/s\n",
3086                   total_connections, failed_connections);
3087       GNUNET_log (
3088                   GNUNET_ERROR_TYPE_WARNING,
3089                   "Overall connection info --- Total: %.2f/s, Total Failed %.2f/s\n",
3090                   conns_per_sec_total, failed_conns_per_sec_total);
3091
3092       GNUNET_asprintf (&temp_conn_string,
3093                        "DHT Profiler Connection (trial %d)",
3094                        trial_to_run);
3095       GNUNET_asprintf (&temp_conn_failed_string,
3096                        "DHT Profiler Connection failed (trial %d)",
3097                        trial_to_run);
3098       GNUNET_asprintf (&revision_str, "%llu", revision);
3099       GAUGER_ID("DHT_TESTING", temp_conn_string, conns_per_sec_total, "conns/s", revision_str);
3100       GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_conns_per_sec_total, "failed_conns", revision_str);
3101       GNUNET_free(temp_conn_string);
3102       GNUNET_free(temp_conn_failed_string);
3103       GNUNET_asprintf (&temp_conn_string,
3104                        "DHT Profiler Total Connections (trial %d)",
3105                        trial_to_run);
3106       GNUNET_asprintf (
3107                        &temp_conn_failed_string,
3108                        "DHT Profiler Total Connections failed (trial %d)",
3109                        trial_to_run);
3110       GAUGER_ID("DHT_TESTING", temp_conn_string, total_connections, "conns", revision_str);
3111       GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_connections, "failed conns", revision_str);
3112       GNUNET_free(temp_conn_string);
3113       GNUNET_free(temp_conn_failed_string);
3114       GNUNET_free(revision_str);
3115
3116       GNUNET_SCHEDULER_cancel (die_task);
3117
3118       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
3119         {
3120           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
3121           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3122                       "Setting continue gets and puts as topo_cont\n");
3123           topo_ctx->cont = &continue_puts_and_gets;
3124           topo_ctx->peers_seen
3125               = GNUNET_CONTAINER_multihashmap_create (num_peers);
3126           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
3127         }
3128       else
3129         {
3130           GNUNET_log (
3131                       GNUNET_ERROR_TYPE_WARNING,
3132                       "For some reason, NOT scheduling final topology capture (settle_time %d, dhtlog_handle %s)!\n",
3133                       settle_time, dhtlog_handle);
3134           GNUNET_SCHEDULER_add_now (&continue_puts_and_gets, NULL);
3135         }
3136     }
3137   else if (total_connections + failed_connections == expected_connections)
3138     {
3139       GNUNET_SCHEDULER_cancel (die_task);
3140       die_task
3141           = GNUNET_SCHEDULER_add_now (&end_badly,
3142                                       "from topology_callback (too many failed connections)");
3143     }
3144 }
3145
3146 static void
3147 peers_started_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3148     const struct GNUNET_CONFIGURATION_Handle *cfg,
3149     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3150 {
3151   if (emsg != NULL)
3152     {
3153       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3154                   "Failed to start daemon with error: `%s'\n", emsg);
3155       return;
3156     }
3157   GNUNET_assert (id != NULL);
3158
3159 #if VERBOSE > 1
3160   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
3161       (num_peers - peers_left) + 1, num_peers);
3162 #endif
3163
3164   peers_left--;
3165
3166   if (GNUNET_YES == update_meter (peer_start_meter))
3167     {
3168 #if VERBOSE
3169       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3170           "All %d daemons started, now connecting peers!\n",
3171           num_peers);
3172 #endif
3173       GNUNET_SCHEDULER_cancel (die_task);
3174
3175       expected_connections = UINT_MAX;
3176       if ((pg != NULL) && (peers_left == 0))
3177         {
3178           connect_start_time = GNUNET_TIME_absolute_get ();
3179           expected_connections
3180               = GNUNET_TESTING_connect_topology (
3181                                                  pg,
3182                                                  connect_topology,
3183                                                  connect_topology_option,
3184                                                  connect_topology_option_modifier,
3185                                                  connect_timeout,
3186                                                  connect_attempts, NULL, NULL);
3187
3188           peer_connect_meter = create_meter (expected_connections,
3189                                              "Peer connection ", GNUNET_YES);
3190           fprintf (stderr, "Have %d expected connections\n",
3191                    expected_connections);
3192         }
3193
3194       if (expected_connections == GNUNET_SYSERR)
3195         {
3196           die_task
3197               = GNUNET_SCHEDULER_add_now (&end_badly,
3198                                           "from connect topology (bad return)");
3199         }
3200
3201       die_task
3202           = GNUNET_SCHEDULER_add_delayed (
3203                                           GNUNET_TIME_relative_multiply (
3204                                                                          GNUNET_TIME_UNIT_SECONDS,
3205                                                                          DEFAULT_CONNECT_TIMEOUT
3206                                                                              * expected_connections),
3207                                           &end_badly,
3208                                           "from connect topology (timeout)");
3209
3210       ok = 0;
3211     }
3212 }
3213
3214 static void
3215 create_topology()
3216 {
3217   peers_left = num_peers; /* Reset counter */
3218   if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology,
3219                                       blacklist_transports) != GNUNET_SYSERR)
3220     {
3221       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3222                   "Topology set up, now starting peers!\n");
3223       GNUNET_TESTING_daemons_continue_startup (pg);
3224     }
3225   else
3226     {
3227       GNUNET_SCHEDULER_cancel (die_task);
3228       die_task = GNUNET_SCHEDULER_add_now (&end_badly,
3229                                            "from create topology (bad return)");
3230     }
3231   GNUNET_free_non_null(blacklist_transports);
3232   GNUNET_SCHEDULER_cancel (die_task);
3233   die_task
3234       = GNUNET_SCHEDULER_add_delayed (
3235                                       GNUNET_TIME_relative_multiply (
3236                                                                      seconds_per_peer_start,
3237                                                                      num_peers),
3238                                       &end_badly,
3239                                       "from continue startup (timeout)");
3240 }
3241
3242 /**
3243  * Callback indicating that the hostkey was created for a peer.
3244  *
3245  * @param cls NULL
3246  * @param id the peer identity
3247  * @param d the daemon handle (pretty useless at this point, remove?)
3248  * @param emsg non-null on failure
3249  */
3250 static void
3251 hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3252     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3253 {
3254   if (emsg != NULL)
3255     {
3256       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3257                   "Hostkey callback received error: %s\n", emsg);
3258     }
3259
3260 #if VERBOSE > 1
3261   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3262       "Hostkey (%d/%d) created for peer `%s'\n",
3263       num_peers - peers_left, num_peers, GNUNET_i2s(id));
3264 #endif
3265
3266   peers_left--;
3267   if (GNUNET_YES == update_meter (hostkey_meter))
3268     {
3269       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3270                   "All %d hostkeys created, now creating topology!\n",
3271                   num_peers);
3272       GNUNET_SCHEDULER_cancel (die_task);
3273       /* Set up task in case topology creation doesn't finish
3274        * within a reasonable amount of time */
3275       die_task = GNUNET_SCHEDULER_add_delayed (DEFAULT_TOPOLOGY_TIMEOUT,
3276                                                &end_badly,
3277                                                "from create_topology");
3278       GNUNET_SCHEDULER_add_now (&create_topology, NULL);
3279       ok = 0;
3280     }
3281 }
3282
3283 static void
3284 run(void *cls, char * const *args, const char *cfgfile,
3285     const struct GNUNET_CONFIGURATION_Handle *cfg)
3286 {
3287   struct stat frstat;
3288   struct GNUNET_DHTLOG_TrialInfo trial_info;
3289   struct GNUNET_TESTING_Host *hosts;
3290   struct GNUNET_TESTING_Host *temphost;
3291   struct GNUNET_TESTING_Host *tempnext;
3292   char *topology_str;
3293   char *connect_topology_str;
3294   char *blacklist_topology_str;
3295   char *connect_topology_option_str;
3296   char *connect_topology_option_modifier_string;
3297   char *trialmessage;
3298   char *topology_percentage_str;
3299   float topology_percentage;
3300   char *topology_probability_str;
3301   char *hostfile;
3302   float topology_probability;
3303   unsigned long long temp_config_number;
3304   int stop_closest;
3305   int stop_found;
3306   int strict_kademlia;
3307   char *buf;
3308   char *data;
3309   char *churn_data;
3310   char *churn_filename;
3311   int count;
3312   int ret;
3313   int line_number;
3314   int k;
3315
3316   config = cfg;
3317   rounds_finished = 0;
3318   memset (&trial_info, 0, sizeof(struct GNUNET_DHTLOG_TrialInfo));
3319   /* Get path from configuration file */
3320   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "paths",
3321                                                            "servicehome",
3322                                                            &test_directory))
3323     {
3324       ok = 404;
3325       return;
3326     }
3327
3328   /* Get number of peers to start from configuration */
3329   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3330                                                               "num_peers",
3331                                                               &num_peers))
3332     {
3333       GNUNET_log (
3334                   GNUNET_ERROR_TYPE_WARNING,
3335                   "Number of peers must be specified in section %s option %s\n",
3336                   "TESTING", "NUM_PEERS");
3337     }
3338   GNUNET_assert(num_peers > 0 && num_peers < ULONG_MAX);
3339
3340   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3341                                                           "connect_timeout",
3342                                                           &temp_config_number))
3343     connect_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3344                                                      temp_config_number);
3345   else
3346     {
3347       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3348                   "testing", "connect_timeout");
3349       return;
3350     }
3351
3352   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3353                                                           "connect_attempts",
3354                                                           &connect_attempts))
3355     {
3356       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3357                   "testing", "connect_attempts");
3358       return;
3359     }
3360
3361   if (GNUNET_OK
3362       != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3363                                                 "max_outstanding_connections",
3364                                                 &max_outstanding_connections))
3365     {
3366       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3367                   "testing", "max_outstanding_connections");
3368       return;
3369     }
3370
3371   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3372                                                           "max_concurrent_ssh",
3373                                                           &max_concurrent_ssh))
3374     {
3375       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3376                   "testing", "max_concurrent_ssh");
3377       return;
3378     }
3379
3380   /**
3381    * Get DHT specific testing options.
3382    */
3383   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3384                                                            "mysql_logging"))
3385       || (GNUNET_YES
3386           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3387                                                    "mysql_logging_extended"))
3388       || (GNUNET_YES
3389           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3390                                                    "mysql_logging_minimal")))
3391     {
3392       if (GNUNET_YES
3393           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3394                                                    "mysql_logging_minimal"))
3395         dhtlog_minimal = GNUNET_YES;
3396
3397       dhtlog_handle = GNUNET_DHTLOG_connect (cfg);
3398       if (dhtlog_handle == NULL)
3399         {
3400           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3401                       "Could not connect to mysql server for logging, will NOT log dht operations!");
3402           ok = 3306;
3403           return;
3404         }
3405     }
3406
3407   stop_closest = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3408                                                        "stop_on_closest");
3409   if (stop_closest == GNUNET_SYSERR)
3410     stop_closest = GNUNET_NO;
3411
3412   stop_found = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht", "stop_found");
3413   if (stop_found == GNUNET_SYSERR)
3414     stop_found = GNUNET_NO;
3415
3416   strict_kademlia = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3417                                                           "strict_kademlia");
3418   if (strict_kademlia == GNUNET_SYSERR)
3419     strict_kademlia = GNUNET_NO;
3420
3421   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3422                                                           "comment",
3423                                                           &trialmessage))
3424     trialmessage = NULL;
3425
3426   churn_data = NULL;
3427   /** Check for a churn file to do churny simulation */
3428   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3429                                                           "churn_file",
3430                                                           &churn_filename))
3431     {
3432       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Reading churn data from %s\n",
3433                   churn_filename);
3434       if (GNUNET_OK != GNUNET_DISK_file_test (churn_filename))
3435         {
3436           GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Error reading churn file!\n");
3437           GNUNET_free_non_null(trialmessage);
3438           GNUNET_free(churn_filename);
3439           return;
3440         }
3441       if ((0 != STAT (churn_filename, &frstat)) || (frstat.st_size == 0))
3442         {
3443           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3444                       "Could not open file specified for churn data, ending test!");
3445           ok = 1119;
3446           GNUNET_free_non_null(trialmessage);
3447           GNUNET_free(churn_filename);
3448           return;
3449         }
3450
3451       churn_data = GNUNET_malloc_large (frstat.st_size);
3452       GNUNET_assert(churn_data != NULL);
3453       if (frstat.st_size != GNUNET_DISK_fn_read (churn_filename, churn_data,
3454                                                  frstat.st_size))
3455         {
3456           GNUNET_log (
3457                       GNUNET_ERROR_TYPE_ERROR,
3458                       "Could not read file %s specified for churn, ending test!",
3459                       churn_filename);
3460           GNUNET_free (churn_filename);
3461           GNUNET_free (churn_data);
3462           GNUNET_free_non_null(trialmessage);
3463           return;
3464         }
3465
3466       GNUNET_free_non_null(churn_filename);
3467
3468       buf = churn_data;
3469       count = 0;
3470       /* Read the first line */
3471       while (count < frstat.st_size)
3472         {
3473           count++;
3474           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3475             {
3476               churn_data[count] = '\0';
3477               if (1 != sscanf (buf, "%u", &churn_rounds))
3478                 {
3479                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3480                               "Failed to read number of rounds from churn file, ending test!\n");
3481                   ret = 4200;
3482                   GNUNET_free_non_null(trialmessage);
3483                   GNUNET_free_non_null(churn_data);
3484                   return;
3485                 }
3486               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3487                           "Read %u rounds from churn file\n", churn_rounds);
3488               buf = &churn_data[count + 1];
3489               churn_array = GNUNET_malloc(sizeof(unsigned int) * churn_rounds);
3490               break; /* Done with this part */
3491             }
3492         }
3493
3494       if (GNUNET_OK
3495           != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3496                                                     "churns_per_round",
3497                                                     &churns_per_round))
3498         {
3499           churns_per_round = (unsigned long long) churn_rounds;
3500         }
3501
3502       line_number = 0;
3503       while ((count < frstat.st_size) && (line_number < churn_rounds))
3504         {
3505           count++;
3506           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3507             {
3508               churn_data[count] = '\0';
3509
3510               ret = sscanf (buf, "%u", &churn_array[line_number]);
3511               if (1 == ret)
3512                 {
3513                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3514                               "Read %u peers in round %u\n",
3515                               churn_array[line_number], line_number);
3516                   line_number++;
3517                 }
3518               else
3519                 {
3520                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3521                               "Error reading line `%s' in hostfile\n", buf);
3522                   buf = &churn_data[count + 1];
3523                   continue;
3524                 }
3525               buf = &churn_data[count + 1];
3526             }
3527           else if (churn_data[count] == '\n') /* Blank line */
3528             buf = &churn_data[count + 1];
3529         }
3530     }
3531   GNUNET_free_non_null(churn_data);
3532
3533   /* Check for a hostfile containing user@host:port triples */
3534   if (GNUNET_OK
3535       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "hostfile",
3536                                                 &hostfile))
3537     hostfile = NULL;
3538
3539   hosts = NULL;
3540   temphost = NULL;
3541   data = NULL;
3542   if (hostfile != NULL)
3543     {
3544       if (GNUNET_OK != GNUNET_DISK_file_test (hostfile))
3545         GNUNET_DISK_fn_write (hostfile, NULL, 0, GNUNET_DISK_PERM_USER_READ
3546             | GNUNET_DISK_PERM_USER_WRITE);
3547       if ((0 != STAT (hostfile, &frstat)) || (frstat.st_size == 0))
3548         {
3549           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3550                       "Could not open file specified for host list, ending test!");
3551           ok = 1119;
3552           GNUNET_free_non_null(trialmessage);
3553           GNUNET_free(hostfile);
3554           return;
3555         }
3556
3557       data = GNUNET_malloc_large (frstat.st_size);
3558       GNUNET_assert(data != NULL);
3559       if (frstat.st_size
3560           != GNUNET_DISK_fn_read (hostfile, data, frstat.st_size))
3561         {
3562           GNUNET_log (
3563                       GNUNET_ERROR_TYPE_ERROR,
3564                       "Could not read file %s specified for host list, ending test!",
3565                       hostfile);
3566           GNUNET_free (hostfile);
3567           GNUNET_free (data);
3568           GNUNET_free_non_null(trialmessage);
3569           return;
3570         }
3571
3572       GNUNET_free_non_null(hostfile);
3573
3574       buf = data;
3575       count = 0;
3576       while (count < frstat.st_size - 1)
3577         {
3578           count++;
3579           if (((data[count] == '\n')) && (buf != &data[count]))
3580             {
3581               data[count] = '\0';
3582               temphost = GNUNET_malloc(sizeof(struct GNUNET_TESTING_Host));
3583               ret = sscanf (buf, "%a[a-zA-Z0-9_]@%a[a-zA-Z0-9.]:%hd",
3584                             &temphost->username, &temphost->hostname,
3585                             &temphost->port);
3586               if (3 == ret)
3587                 {
3588                   GNUNET_log (
3589                               GNUNET_ERROR_TYPE_DEBUG,
3590                               "Successfully read host %s, port %d and user %s from file\n",
3591                               temphost->hostname, temphost->port,
3592                               temphost->username);
3593                 }
3594               else
3595                 {
3596                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3597                               "Error reading line `%s' in hostfile\n", buf);
3598                   GNUNET_free(temphost);
3599                   buf = &data[count + 1];
3600                   continue;
3601                 }
3602               temphost->next = hosts;
3603               hosts = temphost;
3604               buf = &data[count + 1];
3605             }
3606           else if ((data[count] == '\n') || (data[count] == '\0'))
3607             buf = &data[count + 1];
3608         }
3609     }
3610   GNUNET_free_non_null(data);
3611   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3612                                                           "malicious_getters",
3613                                                           &malicious_getters))
3614     malicious_getters = 0;
3615
3616   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3617                                                           "malicious_putters",
3618                                                           &malicious_putters))
3619     malicious_putters = 0;
3620
3621   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3622                                                           "malicious_droppers",
3623                                                           &malicious_droppers))
3624     malicious_droppers = 0;
3625
3626   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3627                                                           "settle_time",
3628                                                           &settle_time))
3629     settle_time = 0;
3630
3631   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3632                                                               "dht_testing",
3633                                                               "num_puts",
3634                                                               &num_puts))
3635     num_puts = num_peers;
3636
3637   if (GNUNET_SYSERR
3638       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3639                                                 "put_replication",
3640                                                 &put_replication))
3641     put_replication = DEFAULT_PUT_REPLICATION;
3642
3643   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3644                                                               "dht_testing",
3645                                                               "num_gets",
3646                                                               &num_gets))
3647     num_gets = num_peers;
3648
3649   if (GNUNET_SYSERR
3650       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3651                                                 "get_replication",
3652                                                 &get_replication))
3653     get_replication = DEFAULT_GET_REPLICATION;
3654
3655   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3656                                                           "find_peer_delay",
3657                                                           &temp_config_number))
3658     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3659                                                      temp_config_number);
3660   else
3661     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3662
3663   if (GNUNET_OK
3664       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3665                                                 "concurrent_find_peers",
3666                                                 &temp_config_number))
3667     max_outstanding_find_peers = temp_config_number;
3668   else
3669     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3670
3671   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3672                                                           "get_timeout",
3673                                                           &temp_config_number))
3674     get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3675                                                  temp_config_number);
3676   else
3677     get_timeout = DEFAULT_GET_TIMEOUT;
3678
3679   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3680                                                           "concurrent_puts",
3681                                                           &temp_config_number))
3682     max_outstanding_puts = temp_config_number;
3683   else
3684     max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
3685
3686   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3687                                                           "concurrent_gets",
3688                                                           &temp_config_number))
3689     max_outstanding_gets = temp_config_number;
3690   else
3691     max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
3692
3693   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3694                                                           "timeout",
3695                                                           &temp_config_number))
3696     all_get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3697                                                      temp_config_number);
3698   else
3699     all_get_timeout.rel_value = get_timeout.rel_value * num_gets;
3700
3701   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3702                                                           "get_delay",
3703                                                           &temp_config_number))
3704     get_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3705                                                temp_config_number);
3706   else
3707     get_delay = DEFAULT_GET_DELAY;
3708
3709   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3710                                                           "put_delay",
3711                                                           &temp_config_number))
3712     put_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3713                                                temp_config_number);
3714   else
3715     put_delay = DEFAULT_PUT_DELAY;
3716
3717   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3718                                                           "peer_start_timeout",
3719                                                           &temp_config_number))
3720     seconds_per_peer_start
3721         = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3722                                          temp_config_number);
3723   else
3724     seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START;
3725
3726   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3727                                                           "data_size",
3728                                                           &temp_config_number))
3729     test_data_size = temp_config_number;
3730   else
3731     test_data_size = DEFAULT_TEST_DATA_SIZE;
3732
3733   /**
3734    * Get testing related options.
3735    */
3736   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3737                                                           "REPLICATE_SAME"))
3738     replicate_same = GNUNET_YES;
3739
3740   /**
3741    * Get testing related options.
3742    */
3743   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3744                                                           "GET_FROM_SAME"))
3745     get_from_same = GNUNET_YES;
3746
3747   if (GNUNET_NO
3748       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3749                                               "MALICIOUS_GET_FREQUENCY",
3750                                               &malicious_get_frequency))
3751     malicious_get_frequency = DEFAULT_MALICIOUS_GET_FREQUENCY;
3752
3753   if (GNUNET_NO
3754       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3755                                               "MALICIOUS_PUT_FREQUENCY",
3756                                               &malicious_put_frequency))
3757     malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY;
3758
3759   if (GNUNET_YES
3760       == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3761                                                "MALICIOUS_AFTER_SETTLE"))
3762     malicious_after_settle = GNUNET_YES;
3763
3764   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3765                                                           "MALICIOUS_SYBIL"))
3766     {
3767       /* Set up the malicious target at random for this round */
3768       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
3769                                         &sybil_target);
3770       malicious_sybil = GNUNET_YES;
3771     }
3772
3773   /* Create the bloomfilter for choosing which peers to set malicious */
3774
3775   /* Bloomfilter size must be 2^k for some integer k */
3776   k = 1;
3777   while (1 << k < malicious_droppers)
3778     k++;
3779   if (malicious_droppers > 0)
3780     malicious_bloom = GNUNET_CONTAINER_bloomfilter_init (NULL, 1 << k,
3781                                                          DHT_BLOOM_K);
3782
3783   /* The normal behavior of the DHT is to do find peer requests
3784    * on its own.  Only if this is explicitly turned off should
3785    * the testing driver issue find peer requests (even though
3786    * this is likely the default when testing).
3787    */
3788   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3789                                                          "do_find_peer"))
3790     do_find_peer = GNUNET_YES;
3791
3792   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3793                                                           "republish"))
3794     in_dht_replication = GNUNET_YES;
3795
3796   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3797                                                            "TRIAL_TO_RUN",
3798                                                            &trial_to_run))
3799     trial_to_run = 0;
3800
3801   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3802                                                            "REVISION",
3803                                                            &revision))
3804     revision = 0;
3805
3806   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3807                                                            "FIND_PEER_DELAY",
3808                                                            &temp_config_number))
3809     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3810                                                      temp_config_number);
3811   else
3812     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3813
3814   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3815                                                            "ROUND_DELAY",
3816                                                            &round_delay))
3817     round_delay = 0;
3818
3819   if (GNUNET_NO
3820       == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3821                                                 "OUTSTANDING_FIND_PEERS",
3822                                                 &max_outstanding_find_peers))
3823     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3824
3825   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3826                                                           "strict_kademlia"))
3827     max_outstanding_find_peers = max_outstanding_find_peers * 1;
3828
3829   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
3830                                                   max_outstanding_find_peers);
3831
3832   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3833                                                               "dht_testing",
3834                                                               "num_rounds",
3835                                                               &total_rounds))
3836     total_rounds = 1;
3837
3838   if ((GNUNET_SYSERR
3839       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3840                                                 "target_total_connections",
3841                                                 &target_total_connections))
3842       || (target_total_connections == 0))
3843     target_total_connections = connection_estimate (num_peers,
3844                                                     DEFAULT_BUCKET_SIZE);
3845
3846   topology_str = NULL;
3847   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3848                                                             "topology",
3849                                                             &topology_str))
3850       && (GNUNET_NO == GNUNET_TESTING_topology_get (&topology, topology_str)))
3851     {
3852       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3853                   "Invalid topology `%s' given for section %s option %s\n",
3854                   topology_str, "TESTING", "TOPOLOGY");
3855       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
3856     }
3857
3858   if (GNUNET_OK
3859       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage",
3860                                                 &topology_percentage_str))
3861     topology_percentage = 0.5;
3862   else
3863     {
3864       topology_percentage = atof (topology_percentage_str);
3865       GNUNET_free(topology_percentage_str);
3866     }
3867
3868   if (GNUNET_OK
3869       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability",
3870                                                 &topology_probability_str))
3871     topology_probability = 0.5;
3872   else
3873     {
3874       topology_probability = atof (topology_probability_str);
3875       GNUNET_free(topology_probability_str);
3876     }
3877
3878   if ((GNUNET_YES
3879       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3880                                                 "connect_topology",
3881                                                 &connect_topology_str))
3882       && (GNUNET_NO == GNUNET_TESTING_topology_get (&connect_topology,
3883                                                     connect_topology_str)))
3884     {
3885       GNUNET_log (
3886                   GNUNET_ERROR_TYPE_WARNING,
3887                   "Invalid connect topology `%s' given for section %s option %s\n",
3888                   connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
3889     }
3890   GNUNET_free_non_null(connect_topology_str);
3891
3892   if ((GNUNET_YES
3893       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3894                                                 "connect_topology_option",
3895                                                 &connect_topology_option_str))
3896       && (GNUNET_NO
3897           == GNUNET_TESTING_topology_option_get (&connect_topology_option,
3898                                                  connect_topology_option_str)))
3899     {
3900       GNUNET_log (
3901                   GNUNET_ERROR_TYPE_WARNING,
3902                   "Invalid connect topology option `%s' given for section %s option %s\n",
3903                   connect_topology_option_str, "TESTING",
3904                   "CONNECT_TOPOLOGY_OPTION");
3905       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
3906     }
3907   GNUNET_free_non_null(connect_topology_option_str);
3908
3909   if (GNUNET_YES
3910       == GNUNET_CONFIGURATION_get_value_string (
3911                                                 cfg,
3912                                                 "testing",
3913                                                 "connect_topology_option_modifier",
3914                                                 &connect_topology_option_modifier_string))
3915     {
3916       if (sscanf (connect_topology_option_modifier_string, "%lf",
3917                   &connect_topology_option_modifier) != 1)
3918         {
3919           GNUNET_log (
3920                       GNUNET_ERROR_TYPE_WARNING,
3921                       _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
3922                       connect_topology_option_modifier_string,
3923                       "connect_topology_option_modifier", "TESTING");
3924         }
3925       GNUNET_free (connect_topology_option_modifier_string);
3926     }
3927
3928   if (GNUNET_YES
3929       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3930                                                 "blacklist_transports",
3931                                                 &blacklist_transports))
3932     blacklist_transports = NULL;
3933
3934   if ((GNUNET_YES
3935       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3936                                                 "blacklist_topology",
3937                                                 &blacklist_topology_str))
3938       && (GNUNET_NO == GNUNET_TESTING_topology_get (&blacklist_topology,
3939                                                     blacklist_topology_str)))
3940     {
3941       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3942                   "Invalid topology `%s' given for section %s option %s\n",
3943                   topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
3944     }
3945   GNUNET_free_non_null(topology_str);
3946   GNUNET_free_non_null(blacklist_topology_str);
3947
3948   /* Set peers_left so we know when all peers started */
3949   peers_left = num_peers;
3950
3951   /* Set up a task to end testing if peer start fails */
3952   die_task
3953       = GNUNET_SCHEDULER_add_delayed (
3954                                       GNUNET_TIME_relative_multiply (
3955                                                                      seconds_per_peer_start,
3956                                                                      num_peers),
3957                                       &end_badly,
3958                                       "didn't generate all hostkeys within allowed startup time!");
3959
3960   if (dhtlog_handle == NULL)
3961     GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "dhtlog_handle is NULL!");
3962
3963   trial_info.other_identifier = (unsigned int) trial_to_run;
3964   trial_info.num_nodes = peers_left;
3965   trial_info.topology = topology;
3966   trial_info.blacklist_topology = blacklist_topology;
3967   trial_info.connect_topology = connect_topology;
3968   trial_info.connect_topology_option = connect_topology_option;
3969   trial_info.connect_topology_option_modifier
3970       = connect_topology_option_modifier;
3971   trial_info.topology_percentage = topology_percentage;
3972   trial_info.topology_probability = topology_probability;
3973   trial_info.puts = num_puts;
3974   trial_info.gets = num_gets;
3975   trial_info.concurrent = max_outstanding_gets;
3976   trial_info.settle_time = settle_time;
3977   trial_info.num_rounds = total_rounds;
3978   trial_info.malicious_getters = malicious_getters;
3979   trial_info.malicious_putters = malicious_putters;
3980   trial_info.malicious_droppers = malicious_droppers;
3981   trial_info.malicious_get_frequency = malicious_get_frequency.rel_value;
3982   trial_info.malicious_put_frequency = malicious_put_frequency.rel_value;
3983   trial_info.stop_closest = stop_closest;
3984   trial_info.stop_found = stop_found;
3985   trial_info.strict_kademlia = strict_kademlia;
3986
3987   if (trialmessage != NULL)
3988     trial_info.message = trialmessage;
3989   else
3990     trial_info.message = "";
3991
3992   if (dhtlog_handle != NULL)
3993     dhtlog_handle->insert_trial (&trial_info);
3994
3995   GNUNET_free_non_null(trialmessage);
3996
3997   hostkey_meter = create_meter (peers_left, "Hostkeys created ", GNUNET_YES);
3998   peer_start_meter = create_meter (peers_left, "Peers started ", GNUNET_YES);
3999
4000   put_meter = create_meter (num_puts, "Puts completed ", GNUNET_YES);
4001   get_meter = create_meter (num_gets, "Gets completed ", GNUNET_YES);
4002   pg
4003       = GNUNET_TESTING_daemons_start (
4004                                       cfg,
4005                                       peers_left,
4006                                       max_outstanding_connections,
4007                                       max_concurrent_ssh,
4008                                       GNUNET_TIME_relative_multiply (
4009                                                                      seconds_per_peer_start,
4010                                                                      num_peers),
4011                                       &hostkey_callback, NULL,
4012                                       &peers_started_callback, NULL,
4013                                       &topology_callback, NULL, hosts);
4014   temphost = hosts;
4015   while (temphost != NULL)
4016     {
4017       tempnext = temphost->next;
4018       GNUNET_free (temphost->username);
4019       GNUNET_free (temphost->hostname);
4020       GNUNET_free (temphost);
4021       temphost = tempnext;
4022     }
4023 }
4024
4025 int
4026 main(int argc, char *argv[])
4027 {
4028   int ret;
4029   struct GNUNET_GETOPT_CommandLineOption options[] =
4030     {
4031     GNUNET_GETOPT_OPTION_END };
4032
4033   ret = GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-driver", "nohelp", options,
4034                             &run, &ok);
4035
4036   if (malicious_bloom != NULL)
4037     GNUNET_CONTAINER_bloomfilter_free (malicious_bloom);
4038
4039   if (ret != GNUNET_OK)
4040     {
4041       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4042                   "`gnunet-dht-driver': Failed with error code %d\n", ret);
4043     }
4044
4045   /**
4046    * Need to remove base directory, subdirectories taken care
4047    * of by the testing framework.
4048    */
4049   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
4050     {
4051       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4052                   "Failed to remove testing directory %s\n", test_directory);
4053     }
4054   return ret;
4055 }
4056
4057 /* end of gnunet-dht-driver.c */