src/dht/gnunet-dht-driver.c

   1 /*
   2  This file is part of GNUnet.
   3  (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5  GNUnet is free software; you can redistribute it and/or modify
   6  it under the terms of the GNU General Public License as published
   7  by the Free Software Foundation; either version 3, or (at your
   8  option) any later version.
   9
  10  GNUnet is distributed in the hope that it will be useful, but
  11  WITHOUT ANY WARRANTY; without even the implied warranty of
  12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  General Public License for more details.
  14
  15  You should have received a copy of the GNU General Public License
  16  along with GNUnet; see the file COPYING.  If not, write to the
  17  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18  Boston, MA 02111-1307, USA.
  19  */
  20 /**
  21  * @file dht/gnunet-dht-driver.c
  22  * @brief Driver for setting up a group of gnunet peers and
  23  *        then issuing GETS and PUTS on the DHT.  Coarse results
  24  *        are reported, fine grained results (if requested) are
  25  *        logged to a (mysql) database, or to file.
  26  * @author Nathan Evans (who to blame)
  27  */
  28 #include "platform.h"
  29 #ifndef HAVE_MALICIOUS
  30 #error foo
  31 #endif
  32 #include "gnunet_testing_lib.h"
  33 #include "gnunet_core_service.h"
  34 #include "gnunet_dht_service.h"
  35 #include "dhtlog.h"
  36 #include "dht.h"
  37 #include "gauger.h"
  38
  39 /* Specific DEBUG hack, do not use normally (may leak memory, segfault, or eat children.) */
  40 #define ONLY_TESTING GNUNET_NO
  41
  42 /* DEFINES */
  43 #define VERBOSE GNUNET_NO
  44
  45 /* Timeout for entire driver to run */
  46 #define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  47
  48 /* Timeout for waiting for (individual) replies to get requests */
  49 #define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  50
  51 #define DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  52
  53 /* Timeout for waiting for gets to be sent to the service */
  54 #define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  55
  56 /* Timeout for waiting for puts to be sent to the service */
  57 #define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  58
  59 /* Time to allow a find peer request to take */
  60 #define DEFAULT_FIND_PEER_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40)
  61
  62 /* Time to wait for all peers disconnected due to to churn to actually be removed from system */
  63 #define DEFAULT_PEER_DISCONNECT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  64
  65 #define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45)
  66
  67 #define DEFAULT_TEST_DATA_SIZE 8
  68
  69 #define DEFAULT_BUCKET_SIZE 4
  70
  71 /* If more than this many peers are added, slow down sending */
  72 #define MAX_FIND_PEER_CUTOFF 2000
  73
  74 /* If less than this many peers are added, speed up sending */
  75 #define MIN_FIND_PEER_CUTOFF 500
  76
  77 /* How often (in seconds) to print out connection information */
  78 #define CONN_UPDATE_DURATION 10
  79
  80 #define DEFAULT_MAX_OUTSTANDING_PUTS 10
  81
  82 #define DEFAULT_MAX_OUTSTANDING_FIND_PEERS 64
  83
  84 #define DEFAULT_FIND_PEER_OFFSET GNUNET_TIME_relative_divide (DEFAULT_FIND_PEER_DELAY, DEFAULT_MAX_OUTSTANDING_FIND_PEERS)
  85
  86 #define DEFAULT_MAX_OUTSTANDING_GETS 10
  87
  88 #define DEFAULT_CONNECT_TIMEOUT 60
  89
  90 #define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8)
  91
  92 #define DEFAULT_RECONNECT_ATTEMPTS 8
  93
  94 /*
  95  * Default frequency for sending malicious get messages
  96  */
  97 #define DEFAULT_MALICIOUS_GET_FREQUENCY GNUNET_TIME_UNIT_SECONDS
  98
  99 /*
 100  * Default frequency for sending malicious put messages
 101  */
 102 #define DEFAULT_MALICIOUS_PUT_FREQUENCY GNUNET_TIME_UNIT_SECONDS
 103
 104 /* Structs */
 105
 106 struct MaliciousContext
 107 {
 108   /**
 109    * Handle to DHT service (via the API)
 110    */
 111   struct GNUNET_DHT_Handle *dht_handle;
 112
 113   /**
 114    *  Handle to the peer daemon
 115    */
 116   struct GNUNET_TESTING_Daemon *daemon;
 117
 118   /**
 119    * Task for disconnecting DHT handles
 120    */
 121   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 122
 123   /**
 124    * What type of malicious to set this peer to.
 125    */
 126   int malicious_type;
 127 };
 128
 129 struct TestFindPeer
 130 {
 131   /* This is a linked list */
 132   struct TestFindPeer *next;
 133
 134   /* Handle to the bigger context */
 135   struct FindPeerContext *find_peer_context;
 136
 137   /**
 138    * Handle to the peer's DHT service (via the API)
 139    */
 140   struct GNUNET_DHT_Handle *dht_handle;
 141
 142   /**
 143    *  Handle to the peer daemon
 144    */
 145   struct GNUNET_TESTING_Daemon *daemon;
 146
 147   /**
 148    * Task for disconnecting DHT handles
 149    */
 150   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 151 };
 152
 153 struct TestPutContext
 154 {
 155   /* This is a linked list */
 156   struct TestPutContext *next;
 157
 158   /**
 159    * Handle to the first peers DHT service (via the API)
 160    */
 161   struct GNUNET_DHT_Handle *dht_handle;
 162
 163   /**
 164    *  Handle to the PUT peer daemon
 165    */
 166   struct GNUNET_TESTING_Daemon *daemon;
 167
 168   /**
 169    *  Identifier for this PUT
 170    */
 171   uint32_t uid;
 172
 173   /**
 174    * Task for disconnecting DHT handles
 175    */
 176   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 177 };
 178
 179 struct TestGetContext
 180 {
 181   /* This is a linked list */
 182   struct TestGetContext *next;
 183
 184   /**
 185    * Handle to the first peers DHT service (via the API)
 186    */
 187   struct GNUNET_DHT_Handle *dht_handle;
 188
 189   /**
 190    * Handle for the DHT get request
 191    */
 192   struct GNUNET_DHT_GetHandle *get_handle;
 193
 194   /**
 195    *  Handle to the GET peer daemon
 196    */
 197   struct GNUNET_TESTING_Daemon *daemon;
 198
 199   /**
 200    *  Identifier for this GET
 201    */
 202   uint32_t uid;
 203
 204   /**
 205    * Task for disconnecting DHT handles (and stopping GET)
 206    */
 207   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 208
 209   /**
 210    * Whether or not this request has been fulfilled already.
 211    */
 212   int succeeded;
 213 };
 214
 215 /**
 216  * Simple struct to keep track of progress, and print a
 217  * nice little percentage meter for long running tasks.
 218  */
 219 struct ProgressMeter
 220 {
 221   unsigned int total;
 222
 223   unsigned int modnum;
 224
 225   unsigned int dotnum;
 226
 227   unsigned int completed;
 228
 229   int print;
 230
 231   char *startup_string;
 232 };
 233
 234 /**
 235  * Linked list of information for populating statistics
 236  * before ending trial.
 237  */
 238 struct StatisticsIteratorContext
 239 {
 240   const struct GNUNET_PeerIdentity *peer;
 241   unsigned int stat_routes;
 242   unsigned int stat_route_forwards;
 243   unsigned int stat_results;
 244   unsigned int stat_results_to_client;
 245   unsigned int stat_result_forwards;
 246   unsigned int stat_gets;
 247   unsigned int stat_puts;
 248   unsigned int stat_puts_inserted;
 249   unsigned int stat_find_peer;
 250   unsigned int stat_find_peer_start;
 251   unsigned int stat_get_start;
 252   unsigned int stat_put_start;
 253   unsigned int stat_find_peer_reply;
 254   unsigned int stat_get_reply;
 255   unsigned int stat_find_peer_answer;
 256   unsigned int stat_get_response_start;
 257 };
 258
 259 /**
 260  * Context for getting a topology, logging it, and continuing
 261  * on with some next operation.
 262  */
 263 struct TopologyIteratorContext
 264 {
 265   unsigned int total_iterations;
 266   unsigned int current_iteration;
 267   unsigned int total_connections;
 268   unsigned int total_peers;
 269   struct GNUNET_CONTAINER_MultiHashMap *peers_seen;
 270   struct GNUNET_PeerIdentity *peer;
 271   GNUNET_SCHEDULER_Task cont;
 272   void *cls;
 273   struct GNUNET_TIME_Relative timeout;
 274 };
 275
 276 struct PeerCount
 277 {
 278   /** Node in the heap */
 279   struct GNUNET_CONTAINER_HeapNode *heap_node;
 280
 281   /** Peer the count refers to */
 282   struct GNUNET_PeerIdentity peer_id;
 283
 284   /** Count of connections this peer has */
 285   unsigned int count;
 286 };
 287
 288 /**
 289  * Context for sending out find peer requests.
 290  */
 291 struct FindPeerContext
 292 {
 293   /**
 294    * How long to send find peer requests, once the settle time
 295    * is over don't send any more out!
 296    */
 297   struct GNUNET_TIME_Absolute endtime;
 298
 299   /**
 300    * Number of connections in the current topology
 301    * (after this round of find peer requests has ended).
 302    */
 303   unsigned int current_peers;
 304
 305   /**
 306    * Number of connections in the current topology
 307    * (before this round of find peer requests started).
 308    */
 309   unsigned int previous_peers;
 310
 311   /**
 312    * Number of find peer requests we have currently
 313    * outstanding.
 314    */
 315   unsigned int outstanding;
 316
 317   /**
 318    * Number of find peer requests to send in this round.
 319    */
 320   unsigned int total;
 321
 322   /**
 323    * Number of find peer requests sent last time around.
 324    */
 325   unsigned int last_sent;
 326
 327   /**
 328    * Hashmap of peers in the current topology, value
 329    * is a PeerCount, with the number of connections
 330    * this peer has.
 331    */
 332   struct GNUNET_CONTAINER_MultiHashMap *peer_hash;
 333
 334   /**
 335    * Min heap which orders values in the peer_hash for
 336    * easy lookup.
 337    */
 338   struct GNUNET_CONTAINER_Heap *peer_min_heap;
 339
 340   /**
 341    * Callback for counting the peers in the current topology.
 342    */
 343   GNUNET_TESTING_NotifyTopology count_peers_cb;
 344 };
 345
 346 enum DHT_ROUND_TYPES
 347 {
 348   /**
 349    * Next full round (puts + gets).
 350    */
 351   DHT_ROUND_NORMAL,
 352
 353   /**
 354    * Next round of gets.
 355    */
 356   DHT_ROUND_GET,
 357
 358   /**
 359    * Next round of puts.
 360    */
 361   DHT_ROUND_PUT,
 362
 363   /**
 364    * Next round of churn.
 365    */
 366   DHT_ROUND_CHURN
 367 };
 368
 369 /* Globals */
 370
 371 /**
 372  * How long to try to connect two peers.
 373  */
 374 struct GNUNET_TIME_Relative connect_timeout;
 375
 376 /**
 377  * How many times to re-attempt connecting two peers.
 378  */
 379 static unsigned long long connect_attempts;
 380
 381 /**
 382  * Timeout to let all GET requests happen.
 383  */
 384 static struct GNUNET_TIME_Relative all_get_timeout;
 385
 386 /**
 387  * Per get timeout
 388  */
 389 static struct GNUNET_TIME_Relative get_timeout;
 390
 391 /**
 392  * Time to allow for GET requests to be sent to service.
 393  */
 394 static struct GNUNET_TIME_Relative get_delay;
 395
 396 /**
 397  * Time to allow for PUT requests to be sent to service.
 398  */
 399 static struct GNUNET_TIME_Relative put_delay;
 400
 401 /**
 402  * Delay between sending find peer requests (if
 403  * handled by the driver, no effect if sent by service).
 404  */
 405 static struct GNUNET_TIME_Relative find_peer_delay;
 406
 407 /**
 408  * Time between find peer requests
 409  * (find_peer_delay / max_outstanding_find_peer)
 410  */
 411 static struct GNUNET_TIME_Relative find_peer_offset;
 412
 413 /**
 414  * How many seconds to allow each peer to start.
 415  */
 416 static struct GNUNET_TIME_Relative seconds_per_peer_start;
 417
 418 /**
 419  * At what time did we start the connection process.
 420  */
 421 static struct GNUNET_TIME_Absolute connect_start_time;
 422
 423 /**
 424  * What was the last time we updated connection/second information.
 425  */
 426 static struct GNUNET_TIME_Absolute connect_last_time;
 427
 428 /**
 429  * Boolean value, should the driver issue find peer requests
 430  * (GNUNET_YES) or should it be left to the service (GNUNET_NO)
 431  */
 432 static unsigned int do_find_peer;
 433
 434 #if ONLY_TESTING
 435 /**
 436  * Are we currently trying to connect two peers repeatedly?
 437  */
 438 static unsigned int repeat_connect_mode;
 439
 440 /**
 441  * Task for repeating connects.
 442  */
 443 GNUNET_SCHEDULER_TaskIdentifier repeat_connect_task;
 444
 445 struct GNUNET_TESTING_Daemon *repeat_connect_peer1;
 446 struct GNUNET_TESTING_Daemon *repeat_connect_peer2;
 447 #endif
 448
 449 /**
 450  * Boolean value, should replication be done by the dht
 451  * service (GNUNET_YES) or by the driver (GNUNET_NO)
 452  */
 453 static unsigned int in_dht_replication;
 454
 455 /**
 456  * Size of test data to insert/retrieve during testing.
 457  */
 458 static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE;
 459
 460 /**
 461  * Maximum number of concurrent connections to peers.
 462  */
 463 static unsigned long long max_outstanding_connections;
 464
 465 /**
 466  * Maximum number of concurrent ssh instances to peers.
 467  */
 468 static unsigned long long max_concurrent_ssh;
 469
 470 /**
 471  * Maximum number of concurrent PUT requests.
 472  */
 473 static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
 474
 475 /**
 476  * Maximum number of concurrent GET requests.
 477  */
 478 static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
 479
 480 /**
 481  * Number of nodes issuing malicious GET messages.
 482  */
 483 static unsigned long long malicious_getters;
 484
 485 /**
 486  * Maximum number of concurrent find peer messages being sent.
 487  */
 488 static unsigned long long max_outstanding_find_peers;
 489
 490 /**
 491  * Number of nodes issuing malicious PUT messages.
 492  */
 493 static unsigned long long malicious_putters;
 494
 495 /**
 496  * Time (in seconds) to delay between rounds.
 497  */
 498 static unsigned long long round_delay;
 499
 500 /**
 501  * The identifier for this trial (if we have one)
 502  * for external data collection.
 503  */
 504 static unsigned long long trial_to_run;
 505
 506 /**
 507  * How many malicious droppers to seed in the network.
 508  */
 509 static unsigned long long malicious_droppers;
 510
 511 /**
 512  * Bloom filter to restrict malicious nodes chosen.
 513  */
 514 struct GNUNET_CONTAINER_BloomFilter *malicious_bloom;
 515
 516 /**
 517  * Whether malicious droppers should be chosen based on proximity to a key.
 518  */
 519 static int malicious_sybil;
 520
 521 /**
 522  * Target for the malicious sybil nodes (choose the closest to this key).
 523  */
 524 static GNUNET_HashCode sybil_target;
 525
 526 /**
 527  * How often to send malicious GET messages.
 528  */
 529 static struct GNUNET_TIME_Relative malicious_get_frequency;
 530
 531 /**
 532  * How often to send malicious PUT messages.
 533  */
 534 static struct GNUNET_TIME_Relative malicious_put_frequency;
 535
 536 /**
 537  * How long to send find peer requests.
 538  */
 539 static unsigned long long settle_time;
 540
 541 /**
 542  * Handle to the dhtlog service.
 543  */
 544 static struct GNUNET_DHTLOG_Handle *dhtlog_handle;
 545
 546 /**
 547  * Replication value for GET requests.
 548  */
 549 static unsigned long long get_replication;
 550
 551 /**
 552  * Replication value for PUT requests.
 553  */
 554 static unsigned long long put_replication;
 555
 556 /**
 557  * If GNUNET_YES, insert data at the same peers every time.
 558  * Otherwise, choose a new random peer to insert at each time.
 559  */
 560 static unsigned int replicate_same;
 561
 562 /**
 563  * If GNUNET_YES, issue GET requests at the same peers every time.
 564  * Otherwise, choose a new random peer/data combination to search
 565  * each time.
 566  */
 567 static unsigned int get_from_same;
 568
 569 /**
 570  * Should malicious peers be set after allowing for settle time?
 571  * Default is to set them malicious after initial connection setup.
 572  */
 573 static unsigned int malicious_after_settle;
 574
 575 /**
 576  * Number of rounds for testing (PUTS + GETS)
 577  */
 578 static unsigned long long total_rounds;
 579
 580 /**
 581  * Target number of connections (will stop sending find peer
 582  * messages when this number is exceeded)
 583  */
 584 static unsigned long long target_total_connections;
 585
 586 /**
 587  * Number of rounds already run
 588  */
 589 static unsigned int rounds_finished;
 590
 591 /**
 592  * Number of rounds of churn to read from the file (first line, should be a single number).
 593  */
 594 static unsigned int churn_rounds;
 595
 596 /**
 597  * Current round we are in for churn, tells us how many peers to connect/disconnect.
 598  */
 599 static unsigned int current_churn_round;
 600
 601 /**
 602  * Number of times to churn per round
 603  */
 604 static unsigned long long churns_per_round;
 605
 606 /**
 607  * Array of churn values.
 608  */
 609 static unsigned int *churn_array;
 610
 611 /**
 612  * Hash map of stats contexts.
 613  */
 614 static struct GNUNET_CONTAINER_MultiHashMap *stats_map;
 615
 616 /**
 617  * LL of malicious settings.
 618  */
 619 struct MaliciousContext *all_malicious;
 620
 621 /**
 622  * List of GETS to perform
 623  */
 624 struct TestGetContext *all_gets;
 625
 626 /**
 627  * List of PUTS to perform
 628  */
 629 struct TestPutContext *all_puts;
 630
 631 /**
 632  * Directory to store temporary data in, defined in config file
 633  */
 634 static char *test_directory;
 635
 636 /**
 637  * Variable used to store the number of connections we should wait for.
 638  */
 639 static unsigned int expected_connections;
 640
 641 /**
 642  * Variable used to keep track of how many peers aren't yet started.
 643  */
 644 static unsigned long long peers_left;
 645
 646 /**
 647  * Handle to the set of all peers run for this test.
 648  */
 649 static struct GNUNET_TESTING_PeerGroup *pg;
 650
 651 /**
 652  * Global config handle.
 653  */
 654 static const struct GNUNET_CONFIGURATION_Handle *config;
 655
 656 /**
 657  * Total number of peers to run, set based on config file.
 658  */
 659 static unsigned long long num_peers;
 660
 661 /**
 662  * Total number of items to insert.
 663  */
 664 static unsigned long long num_puts;
 665
 666 /**
 667  * How many puts do we currently have in flight?
 668  */
 669 static unsigned long long outstanding_puts;
 670
 671 /**
 672  * How many puts are done?
 673  */
 674 static unsigned long long puts_completed;
 675
 676 /**
 677  * Total number of items to attempt to get.
 678  */
 679 static unsigned long long num_gets;
 680
 681 /**
 682  * How many puts do we currently have in flight?
 683  */
 684 static unsigned long long outstanding_gets;
 685
 686 /**
 687  * How many gets are done?
 688  */
 689 static unsigned long long gets_completed;
 690
 691 /**
 692  * Total number of items to attempt to get.
 693  */
 694 static unsigned long long cumulative_num_gets;
 695
 696 /**
 697  * How many gets are done?
 698  */
 699 static unsigned long long cumulative_successful_gets;
 700
 701 /**
 702  * How many gets failed?
 703  */
 704 static unsigned long long gets_failed;
 705
 706 /**
 707  * How many malicious control messages do
 708  * we currently have in flight?
 709  */
 710 static unsigned long long outstanding_malicious;
 711
 712 /**
 713  * How many set malicious peers are done?
 714  */
 715 static unsigned int malicious_completed;
 716
 717 /**
 718  * For gauger logging, what specific identifier (svn revision)
 719  * should be used?
 720  */
 721 static unsigned long long revision;
 722
 723 /**
 724  * Global used to count how many connections we have currently
 725  * been notified about (how many times has topology_callback been called
 726  * with success?)
 727  */
 728 static unsigned int total_connections;
 729
 730 /**
 731  * Previous connections, for counting new connections during some duration.
 732  */
 733 static unsigned int previous_connections;
 734
 735 /**
 736  * For counting failed connections during some duration.
 737  */
 738 static unsigned int previous_failed_connections;
 739
 740 /**
 741  * Global used to count how many failed connections we have
 742  * been notified about (how many times has topology_callback
 743  * been called with failure?)
 744  */
 745 static unsigned int failed_connections;
 746
 747 /**
 748  * If GNUNET_YES, only log PUT/GET round data to mysql, otherwise
 749  * log everything (including each dht service logging).
 750  */
 751 static unsigned int dhtlog_minimal;
 752
 753 /* Task handle to use to schedule shutdown if something goes wrong */
 754 GNUNET_SCHEDULER_TaskIdentifier die_task;
 755
 756 static char *blacklist_transports;
 757
 758 static enum GNUNET_TESTING_Topology topology;
 759
 760 static enum GNUNET_TESTING_Topology blacklist_topology =
 761     GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 762
 763 static enum GNUNET_TESTING_Topology connect_topology =
 764     GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 765
 766 static enum GNUNET_TESTING_TopologyOption connect_topology_option =
 767     GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 768
 769 static double connect_topology_option_modifier = 0.0;
 770
 771 static struct ProgressMeter *hostkey_meter;
 772
 773 static struct ProgressMeter *peer_start_meter;
 774
 775 static struct ProgressMeter *peer_connect_meter;
 776
 777 static struct ProgressMeter *put_meter;
 778
 779 static struct ProgressMeter *get_meter;
 780
 781 static GNUNET_HashCode *known_keys;
 782
 783 /* Global return value (0 for success, anything else for failure) */
 784 static int ok;
 785
 786 /**
 787  * Create a meter to keep track of the progress of some task.
 788  *
 789  * @param total the total number of items to complete
 790  * @param start_string a string to prefix the meter with (if printing)
 791  * @param print GNUNET_YES to print the meter, GNUNET_NO to count
 792  *              internally only
 793  *
 794  * @return the progress meter
 795  */
 796 static struct ProgressMeter *
 797 create_meter(unsigned int total, char * start_string, int print)
 798 {
 799   struct ProgressMeter *ret;
 800   ret = GNUNET_malloc(sizeof(struct ProgressMeter));
 801   ret->print = print;
 802   ret->total = total;
 803   ret->modnum = total / 4;
 804   ret->dotnum = (total / 50) + 1;
 805   if (start_string != NULL)
 806     ret->startup_string = GNUNET_strdup(start_string);
 807   else
 808     ret->startup_string = GNUNET_strdup("");
 809
 810   return ret;
 811 }
 812
 813 /**
 814  * Update progress meter (increment by one).
 815  *
 816  * @param meter the meter to update and print info for
 817  *
 818  * @return GNUNET_YES if called the total requested,
 819  *         GNUNET_NO if more items expected
 820  */
 821 static int
 822 update_meter(struct ProgressMeter *meter)
 823 {
 824   if (meter->print == GNUNET_YES)
 825     {
 826       if (meter->completed % meter->modnum == 0)
 827         {
 828           if (meter->completed == 0)
 829             {
 830               fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
 831             }
 832           else
 833             fprintf (stdout, "%d%%", (int) (((float) meter->completed
 834                 / meter->total) * 100));
 835         }
 836       else if (meter->completed % meter->dotnum == 0)
 837         fprintf (stdout, ".");
 838
 839       if (meter->completed + 1 == meter->total)
 840         fprintf (stdout, "%d%%]\n", 100);
 841       fflush (stdout);
 842     }
 843   meter->completed++;
 844
 845   if (meter->completed == meter->total)
 846     return GNUNET_YES;
 847   return GNUNET_NO;
 848 }
 849
 850 /**
 851  * Reset progress meter.
 852  *
 853  * @param meter the meter to reset
 854  *
 855  * @return GNUNET_YES if meter reset,
 856  *         GNUNET_SYSERR on error
 857  */
 858 static int
 859 reset_meter(struct ProgressMeter *meter)
 860 {
 861   if (meter == NULL)
 862     return GNUNET_SYSERR;
 863
 864   meter->completed = 0;
 865   return GNUNET_YES;
 866 }
 867
 868 /**
 869  * Release resources for meter
 870  *
 871  * @param meter the meter to free
 872  */
 873 static void
 874 free_meter(struct ProgressMeter *meter)
 875 {
 876   GNUNET_free_non_null (meter->startup_string);
 877   GNUNET_free (meter);
 878 }
 879
 880 /**
 881  * Check whether peers successfully shut down.
 882  */
 883 static void
 884 shutdown_callback(void *cls, const char *emsg)
 885 {
 886   if (emsg != NULL)
 887     {
 888       if (ok == 0)
 889         ok = 2;
 890     }
 891 }
 892
 893 /**
 894  * Task to release DHT handles for PUT
 895  */
 896 static void
 897 put_disconnect_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 898 {
 899   struct TestPutContext *test_put = cls;
 900   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 901   GNUNET_DHT_disconnect (test_put->dht_handle);
 902   test_put->dht_handle = NULL;
 903   if (replicate_same == GNUNET_NO)
 904     test_put->daemon
 905         = GNUNET_TESTING_daemon_get (
 906                                      pg,
 907                                      GNUNET_CRYPTO_random_u32 (
 908                                                                GNUNET_CRYPTO_QUALITY_WEAK,
 909                                                                num_peers));
 910 }
 911
 912 /**
 913  * Function scheduled to be run on the successful completion of this
 914  * testcase.
 915  */
 916 static void
 917 finish_testing(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 918 {
 919   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ending test normally!\n",
 920               (char *) cls);
 921   GNUNET_assert (pg != NULL);
 922   struct TestPutContext *test_put = all_puts;
 923   struct TestGetContext *test_get = all_gets;
 924   char *temp_get_string;
 925   char *revision_str;
 926
 927   while (test_put != NULL)
 928     {
 929       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 930         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
 931       if (test_put->dht_handle != NULL)
 932         GNUNET_DHT_disconnect (test_put->dht_handle);
 933       test_put = test_put->next;
 934     }
 935
 936   while (test_get != NULL)
 937     {
 938       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 939         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
 940       if (test_get->get_handle != NULL)
 941         GNUNET_DHT_get_stop (test_get->get_handle);
 942       if (test_get->dht_handle != NULL)
 943         GNUNET_DHT_disconnect (test_get->dht_handle);
 944       test_get = test_get->next;
 945     }
 946
 947   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
 948
 949   if (dhtlog_handle != NULL)
 950     {
 951       fprintf (stderr, "Update trial endtime\n");
 952       dhtlog_handle->update_trial (cumulative_successful_gets);
 953       GNUNET_DHTLOG_disconnect (dhtlog_handle);
 954       dhtlog_handle = NULL;
 955     }
 956
 957   if (hostkey_meter != NULL)
 958     free_meter (hostkey_meter);
 959   if (peer_start_meter != NULL)
 960     free_meter (peer_start_meter);
 961   if (peer_connect_meter != NULL)
 962     free_meter (peer_connect_meter);
 963   if (put_meter != NULL)
 964     free_meter (put_meter);
 965   if (get_meter != NULL)
 966     free_meter (get_meter);
 967
 968   GNUNET_asprintf (&temp_get_string,
 969                    "DHT Successful GETs (trial %d)", trial_to_run);
 970   GNUNET_asprintf (&revision_str, "%llu", revision);
 971   GAUGER_ID("DHT_TESTING", temp_get_string, cumulative_successful_gets / (double)cumulative_num_gets, "percent successful", revision_str);
 972   fprintf (
 973            stderr,
 974            "Finished trial, had %llu successful gets out of %llu total, %.2f percent succeeded\n",
 975            cumulative_successful_gets, cumulative_num_gets,
 976            cumulative_successful_gets / (double) cumulative_num_gets);
 977   GNUNET_free(temp_get_string);
 978
 979   ok = 0;
 980 }
 981
 982 /**
 983  * Callback for iterating over all the peer connections of a peer group.
 984  */
 985 static void
 986 log_topology_cb(void *cls, const struct GNUNET_PeerIdentity *first,
 987     const struct GNUNET_PeerIdentity *second, const char *emsg)
 988 {
 989   struct TopologyIteratorContext *topo_ctx = cls;
 990   if ((first != NULL) && (second != NULL))
 991     {
 992       if ((topo_ctx->peers_seen != NULL) && (GNUNET_NO
 993           == GNUNET_CONTAINER_multihashmap_contains (topo_ctx->peers_seen,
 994                                                      &first->hashPubKey)))
 995         {
 996           GNUNET_CONTAINER_multihashmap_put (topo_ctx->peers_seen,
 997                                              &first->hashPubKey, NULL,
 998                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
 999           topo_ctx->total_peers++;
1000         }
1001       topo_ctx->total_connections++;
1002       if ((GNUNET_NO == dhtlog_minimal) && (dhtlog_handle != NULL))
1003         dhtlog_handle->insert_extended_topology (first, second);
1004     }
1005   else
1006     {
1007       GNUNET_assert(dhtlog_handle != NULL);
1008       GNUNET_log (
1009                   GNUNET_ERROR_TYPE_WARNING,
1010                   "Topology iteration (%u/%u) finished (%u connections, %u peers)\n",
1011                   topo_ctx->current_iteration, topo_ctx->total_iterations,
1012                   topo_ctx->total_connections, topo_ctx->total_peers);
1013       dhtlog_handle->update_topology (topo_ctx->total_connections);
1014       if (topo_ctx->cont != NULL)
1015         GNUNET_SCHEDULER_add_now (topo_ctx->cont, topo_ctx->cls);
1016       if (topo_ctx->peers_seen != NULL)
1017         GNUNET_CONTAINER_multihashmap_destroy (topo_ctx->peers_seen);
1018       GNUNET_free(topo_ctx);
1019     }
1020 }
1021
1022 /**
1023  * Iterator over hash map entries.
1024  *
1025  * @param cls closure - always NULL
1026  * @param key current key code
1027  * @param value value in the hash map, a stats context
1028  * @return GNUNET_YES if we should continue to
1029  *         iterate,
1030  *         GNUNET_NO if not.
1031  */
1032 static int
1033 stats_iterate(void *cls, const GNUNET_HashCode * key, void *value)
1034 {
1035   struct StatisticsIteratorContext *stats_ctx;
1036   if (value == NULL)
1037     return GNUNET_NO;
1038   stats_ctx = value;
1039   dhtlog_handle->insert_stat (stats_ctx->peer, stats_ctx->stat_routes,
1040                               stats_ctx->stat_route_forwards,
1041                               stats_ctx->stat_results,
1042                               stats_ctx->stat_results_to_client,
1043                               stats_ctx->stat_result_forwards,
1044                               stats_ctx->stat_gets, stats_ctx->stat_puts,
1045                               stats_ctx->stat_puts_inserted,
1046                               stats_ctx->stat_find_peer,
1047                               stats_ctx->stat_find_peer_start,
1048                               stats_ctx->stat_get_start,
1049                               stats_ctx->stat_put_start,
1050                               stats_ctx->stat_find_peer_reply,
1051                               stats_ctx->stat_get_reply,
1052                               stats_ctx->stat_find_peer_answer,
1053                               stats_ctx->stat_get_response_start);
1054   GNUNET_free(stats_ctx);
1055   return GNUNET_YES;
1056 }
1057
1058 static void
1059 stats_finished(void *cls, int result)
1060 {
1061   fprintf (stderr, "Finished getting all peers statistics, iterating!\n");
1062   GNUNET_CONTAINER_multihashmap_iterate (stats_map, &stats_iterate, NULL);
1063   GNUNET_CONTAINER_multihashmap_destroy (stats_map);
1064   GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1065 }
1066
1067 /**
1068  * Callback function to process statistic values.
1069  *
1070  * @param cls closure
1071  * @param peer the peer the statistics belong to
1072  * @param subsystem name of subsystem that created the statistic
1073  * @param name the name of the datum
1074  * @param value the current value
1075  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1076  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1077  */
1078 static int
1079 stats_handle(void *cls, const struct GNUNET_PeerIdentity *peer,
1080     const char *subsystem, const char *name, uint64_t value, int is_persistent)
1081 {
1082   struct StatisticsIteratorContext *stats_ctx;
1083
1084   if (dhtlog_handle != NULL)
1085     dhtlog_handle->add_generic_stat (peer, name, subsystem, value);
1086   if (GNUNET_CONTAINER_multihashmap_contains (stats_map, &peer->hashPubKey))
1087     {
1088       stats_ctx = GNUNET_CONTAINER_multihashmap_get (stats_map,
1089                                                      &peer->hashPubKey);
1090     }
1091   else
1092     {
1093       stats_ctx = GNUNET_malloc(sizeof(struct StatisticsIteratorContext));
1094       stats_ctx->peer = peer;
1095       GNUNET_CONTAINER_multihashmap_put (stats_map, &peer->hashPubKey,
1096                                          stats_ctx,
1097                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1098     }
1099   GNUNET_assert(stats_ctx != NULL);
1100
1101   if (strcmp (name, STAT_ROUTES) == 0)
1102     stats_ctx->stat_routes = value;
1103   else if (strcmp (name, STAT_ROUTE_FORWARDS) == 0)
1104     stats_ctx->stat_route_forwards = value;
1105   else if (strcmp (name, STAT_RESULTS) == 0)
1106     stats_ctx->stat_results = value;
1107   else if (strcmp (name, STAT_RESULTS_TO_CLIENT) == 0)
1108     stats_ctx->stat_results_to_client = value;
1109   else if (strcmp (name, STAT_RESULT_FORWARDS) == 0)
1110     stats_ctx->stat_result_forwards = value;
1111   else if (strcmp (name, STAT_GETS) == 0)
1112     stats_ctx->stat_gets = value;
1113   else if (strcmp (name, STAT_PUTS) == 0)
1114     stats_ctx->stat_puts = value;
1115   else if (strcmp (name, STAT_PUTS_INSERTED) == 0)
1116     stats_ctx->stat_puts_inserted = value;
1117   else if (strcmp (name, STAT_FIND_PEER) == 0)
1118     stats_ctx->stat_find_peer = value;
1119   else if (strcmp (name, STAT_FIND_PEER_START) == 0)
1120     stats_ctx->stat_find_peer_start = value;
1121   else if (strcmp (name, STAT_GET_START) == 0)
1122     stats_ctx->stat_get_start = value;
1123   else if (strcmp (name, STAT_PUT_START) == 0)
1124     stats_ctx->stat_put_start = value;
1125   else if (strcmp (name, STAT_FIND_PEER_REPLY) == 0)
1126     stats_ctx->stat_find_peer_reply = value;
1127   else if (strcmp (name, STAT_GET_REPLY) == 0)
1128     stats_ctx->stat_get_reply = value;
1129   else if (strcmp (name, STAT_FIND_PEER_ANSWER) == 0)
1130     stats_ctx->stat_find_peer_answer = value;
1131   else if (strcmp (name, STAT_GET_RESPONSE_START) == 0)
1132     stats_ctx->stat_get_response_start = value;
1133
1134   return GNUNET_OK;
1135 }
1136
1137 /**
1138  * Connect to statistics service for each peer and get the appropriate
1139  * dht statistics for safe keeping.
1140  */
1141 static void
1142 log_dht_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1143 {
1144   stats_map = GNUNET_CONTAINER_multihashmap_create (num_peers);
1145   fprintf (stderr, "Starting statistics logging\n");
1146   GNUNET_TESTING_get_statistics (pg, &stats_finished, &stats_handle, NULL);
1147 }
1148
1149 /**
1150  * Connect to all peers in the peer group and iterate over their
1151  * connections.
1152  */
1153 static void
1154 capture_current_topology(void *cls,
1155     const struct GNUNET_SCHEDULER_TaskContext * tc)
1156 {
1157   struct TopologyIteratorContext *topo_ctx = cls;
1158   dhtlog_handle->insert_topology (0);
1159   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Called capture_current_topology\n");
1160   GNUNET_TESTING_get_topology (pg, &log_topology_cb, topo_ctx);
1161 }
1162
1163 /**
1164  * Check if the get_handle is being used, if so stop the request.  Either
1165  * way, schedule the end_badly_cont function which actually shuts down the
1166  * test.
1167  */
1168 static void
1169 end_badly(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1170 {
1171   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failing test with error: `%s'!\n",
1172               (char *) cls);
1173
1174   struct TestPutContext *test_put = all_puts;
1175   struct TestGetContext *test_get = all_gets;
1176
1177   while (test_put != NULL)
1178     {
1179       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1180         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
1181       if (test_put->dht_handle != NULL)
1182         GNUNET_DHT_disconnect (test_put->dht_handle);
1183       test_put = test_put->next;
1184     }
1185
1186   while (test_get != NULL)
1187     {
1188       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1189         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
1190       if (test_get->get_handle != NULL)
1191         GNUNET_DHT_get_stop (test_get->get_handle);
1192       if (test_get->dht_handle != NULL)
1193         GNUNET_DHT_disconnect (test_get->dht_handle);
1194       test_get = test_get->next;
1195     }
1196
1197   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
1198
1199   if (dhtlog_handle != NULL)
1200     {
1201       fprintf (stderr, "Update trial endtime\n");
1202       dhtlog_handle->update_trial (gets_completed);
1203       GNUNET_DHTLOG_disconnect (dhtlog_handle);
1204       dhtlog_handle = NULL;
1205     }
1206
1207   if (hostkey_meter != NULL)
1208     free_meter (hostkey_meter);
1209   if (peer_start_meter != NULL)
1210     free_meter (peer_start_meter);
1211   if (peer_connect_meter != NULL)
1212     free_meter (peer_connect_meter);
1213   if (put_meter != NULL)
1214     free_meter (put_meter);
1215   if (get_meter != NULL)
1216     free_meter (get_meter);
1217
1218   ok = 1;
1219 }
1220
1221 /**
1222  * Forward declaration.
1223  */
1224 static void
1225 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1226
1227 /**
1228  * Forward declaration.
1229  */
1230 static void
1231 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1232
1233 /**
1234  * Iterator over hash map entries.
1235  *
1236  * @param cls closure
1237  * @param key current key code
1238  * @param value value in the hash map
1239  * @return GNUNET_YES if we should continue to
1240  *         iterate,
1241  *         GNUNET_NO if not.
1242  */
1243 static int
1244 remove_peer_count(void *cls, const GNUNET_HashCode * key, void *value)
1245 {
1246   struct FindPeerContext *find_peer_ctx = cls;
1247   struct PeerCount *peer_count = value;
1248   GNUNET_CONTAINER_heap_remove_node (find_peer_ctx->peer_min_heap,
1249                                      peer_count->heap_node);
1250   GNUNET_free(peer_count);
1251
1252   return GNUNET_YES;
1253 }
1254
1255 /**
1256  * Connect to all peers in the peer group and iterate over their
1257  * connections.
1258  */
1259 static void
1260 count_new_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1261 {
1262   struct FindPeerContext *find_peer_context = cls;
1263   find_peer_context->previous_peers = find_peer_context->current_peers;
1264   find_peer_context->current_peers = 0;
1265   GNUNET_TESTING_get_topology (pg, find_peer_context->count_peers_cb,
1266                                find_peer_context);
1267 }
1268
1269 static void
1270 decrement_find_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1271 {
1272   struct TestFindPeer *test_find_peer = cls;
1273   GNUNET_assert(test_find_peer->find_peer_context->outstanding > 0);
1274   test_find_peer->find_peer_context->outstanding--;
1275   test_find_peer->find_peer_context->total--;
1276   if (0 == test_find_peer->find_peer_context->total)
1277     {
1278       GNUNET_SCHEDULER_add_now (&count_new_peers,
1279                                 test_find_peer->find_peer_context);
1280     }
1281   GNUNET_free(test_find_peer);
1282 }
1283
1284 /**
1285  * A find peer request has been sent to the server, now we will schedule a task
1286  * to wait the appropriate time to allow the request to go out and back.
1287  *
1288  * @param cls closure - a TestFindPeer struct
1289  * @param tc context the task is being called with
1290  */
1291 static void
1292 handle_find_peer_sent(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1293 {
1294   struct TestFindPeer *test_find_peer = cls;
1295
1296   GNUNET_DHT_disconnect (test_find_peer->dht_handle);
1297   GNUNET_SCHEDULER_add_delayed (
1298                                 GNUNET_TIME_relative_divide (find_peer_delay, 2),
1299                                 &decrement_find_peers, test_find_peer);
1300 }
1301
1302 static void
1303 send_find_peer_request(void *cls,
1304     const struct GNUNET_SCHEDULER_TaskContext * tc)
1305 {
1306   struct TestFindPeer *test_find_peer = cls;
1307
1308   if (test_find_peer->find_peer_context->outstanding
1309       > max_outstanding_find_peers)
1310     {
1311       GNUNET_SCHEDULER_add_delayed (find_peer_offset, &send_find_peer_request,
1312                                     test_find_peer);
1313       return;
1314     }
1315
1316   test_find_peer->find_peer_context->outstanding++;
1317   if (GNUNET_TIME_absolute_get_remaining (
1318                                           test_find_peer->find_peer_context->endtime).rel_value
1319       == 0)
1320     {
1321       GNUNET_SCHEDULER_add_now (&decrement_find_peers, test_find_peer);
1322       return;
1323     }
1324
1325   test_find_peer->dht_handle = GNUNET_DHT_connect (test_find_peer->daemon->cfg,
1326                                                    1);
1327   GNUNET_assert(test_find_peer->dht_handle != NULL);
1328   GNUNET_DHT_find_peers (test_find_peer->dht_handle, &handle_find_peer_sent,
1329                          test_find_peer);
1330 }
1331
1332 /**
1333  * Add a connection to the find_peer_context given.  This may
1334  * be complete overkill, but allows us to choose the peers with
1335  * the least connections to initiate find peer requests from.
1336  */
1337 static void
1338 add_new_connection(struct FindPeerContext *find_peer_context,
1339     const struct GNUNET_PeerIdentity *first,
1340     const struct GNUNET_PeerIdentity *second)
1341 {
1342   struct PeerCount *first_count;
1343   struct PeerCount *second_count;
1344
1345   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1346                                               &first->hashPubKey))
1347     {
1348       first_count
1349           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1350                                                &first->hashPubKey);
1351       GNUNET_assert(first_count != NULL);
1352       first_count->count++;
1353       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1354                                          first_count->heap_node,
1355                                          first_count->count);
1356     }
1357   else
1358     {
1359       first_count = GNUNET_malloc(sizeof(struct PeerCount));
1360       first_count->count = 1;
1361       memcpy (&first_count->peer_id, first, sizeof(struct GNUNET_PeerIdentity));
1362       first_count->heap_node
1363           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1364                                           first_count, first_count->count);
1365       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1366                                          &first->hashPubKey, first_count,
1367                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1368     }
1369
1370   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1371                                               &second->hashPubKey))
1372     {
1373       second_count
1374           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1375                                                &second->hashPubKey);
1376       GNUNET_assert(second_count != NULL);
1377       second_count->count++;
1378       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1379                                          second_count->heap_node,
1380                                          second_count->count);
1381     }
1382   else
1383     {
1384       second_count = GNUNET_malloc(sizeof(struct PeerCount));
1385       second_count->count = 1;
1386       memcpy (&second_count->peer_id, second,
1387               sizeof(struct GNUNET_PeerIdentity));
1388       second_count->heap_node
1389           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1390                                           second_count, second_count->count);
1391       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1392                                          &second->hashPubKey, second_count,
1393                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1394     }
1395 }
1396
1397 /**
1398  * Iterate over min heap of connections per peer.  For any
1399  * peer that has 0 connections, attempt to connect them to
1400  * some random peer.
1401  *
1402  * @param cls closure a struct FindPeerContext
1403  * @param node internal node of the heap
1404  * @param element value stored, a struct PeerCount
1405  * @param cost cost associated with the node
1406  * @return GNUNET_YES if we should continue to iterate,
1407  *         GNUNET_NO if not.
1408  */
1409 static int
1410 iterate_min_heap_peers(void *cls, struct GNUNET_CONTAINER_HeapNode *node,
1411     void *element, GNUNET_CONTAINER_HeapCostType cost)
1412 {
1413   struct FindPeerContext *find_peer_context = cls;
1414   struct PeerCount *peer_count = element;
1415   struct GNUNET_TESTING_Daemon *d1;
1416   struct GNUNET_TESTING_Daemon *d2;
1417   struct GNUNET_TIME_Relative timeout;
1418   if (cost == 0)
1419     {
1420       d1 = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1421       GNUNET_assert(d1 != NULL);
1422       d2 = d1;
1423       while ((d2 == d1) || (GNUNET_YES != GNUNET_TESTING_daemon_running (d2)))
1424         {
1425           d2
1426               = GNUNET_TESTING_daemon_get (
1427                                            pg,
1428                                            GNUNET_CRYPTO_random_u32 (
1429                                                                      GNUNET_CRYPTO_QUALITY_WEAK,
1430                                                                      num_peers));
1431           GNUNET_assert(d2 != NULL);
1432         }
1433
1434       /** Just try to connect the peers, don't worry about callbacks, etc. **/
1435       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1436                   "Peer %s has 0 connections.  Trying to connect to %s...\n",
1437                   GNUNET_i2s (&peer_count->peer_id), d2->shortname);
1438       timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
1439                                                DEFAULT_CONNECT_TIMEOUT);
1440       if (GNUNET_TIME_relative_to_absolute (timeout).abs_value
1441           > find_peer_context->endtime.abs_value)
1442         {
1443           timeout
1444               = GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime);
1445         }
1446       GNUNET_TESTING_daemons_connect (d1, d2, timeout,
1447                                       DEFAULT_RECONNECT_ATTEMPTS, GNUNET_YES,
1448                                       NULL, NULL);
1449     }
1450   if (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1451       > 0)
1452     return GNUNET_YES;
1453   else
1454     return GNUNET_NO;
1455 }
1456
1457 /**
1458  * Forward declaration.
1459  */
1460 static void
1461 schedule_churn_find_peer_requests(void *cls,
1462     const struct GNUNET_SCHEDULER_TaskContext * tc);
1463
1464 /**
1465  * Callback for iterating over all the peer connections of a peer group.
1466  * Used after we have churned on some peers to find which ones have zero
1467  * connections so we can make them issue find peer requests.
1468  */
1469 static void
1470 count_peers_churn_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1471     const struct GNUNET_PeerIdentity *second, const char *emsg)
1472 {
1473   struct FindPeerContext *find_peer_context = cls;
1474   struct TopologyIteratorContext *topo_ctx;
1475   struct PeerCount *peer_count;
1476
1477   if ((first != NULL) && (second != NULL))
1478     {
1479       add_new_connection (find_peer_context, first, second);
1480       find_peer_context->current_peers++;
1481     }
1482   else
1483     {
1484       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1485                   "Peer count finished (%u connections)\n",
1486                   find_peer_context->current_peers);
1487       peer_count
1488           = GNUNET_CONTAINER_heap_peek (find_peer_context->peer_min_heap);
1489       GNUNET_assert(peer_count != NULL);
1490       /* WAIT. When peers are churned they will come back with their peers (at least in peerinfo), because the HOSTS file doesn't likely get removed. CRAP. */
1491       /* NO they won't, because we have disabled peerinfo writing to disk (remember?) so we WILL have to give them new connections */
1492       /* Best course of action: have DHT automatically try to add peers from peerinfo on startup. This way IF peerinfo writes to file
1493        * then some peers will end up connected.
1494        *
1495        * Also, find any peers that have zero connections here and set up a task to choose at random another peer in the network to
1496        * connect to.  Of course, if they are blacklisted from that peer they won't be able to connect, so we will have to keep trying
1497        * until they get a peer.
1498        */
1499       /* However, they won't automatically be connected to any of their previous peers... How can we handle that? */
1500       /* So now we have choices: do we want them to come back with all their connections?  Probably not, but it solves this mess. */
1501
1502       /* Second problem, which is still a problem, is that a FIND_PEER request won't work when a peer has no connections */
1503
1504       /**
1505        * Okay, so here's how this *should* work now.
1506        *
1507        * 1. We check the min heap for any peers that have 0 connections.
1508        *    a. If any are found, we iterate over the heap and just randomly
1509        *       choose another peer and ask testing to please connect the two.
1510        *       This takes care of the case that a peer just randomly joins the
1511        *       network.  However, if there are strict topology restrictions
1512        *       (imagine a ring) choosing randomly most likely won't help.
1513        *       We make sure the connection attempt doesn't take longer than
1514        *       the total timeout, but don't care too much about the result.
1515        *    b. After that, we still schedule the find peer requests (concurrently
1516        *       with the connect attempts most likely).  This handles the case
1517        *       that the DHT iterates over peerinfo and just needs to try to send
1518        *       a message to get connected.  This should handle the case that the
1519        *       topology is very strict.
1520        *
1521        * 2. If all peers have > 0 connections, we still send find peer requests
1522        *    as long as possible (until timeout is reached) to help out those
1523        *    peers that were newly churned and need more connections.  This is because
1524        *    once all new peers have established a single connection, they won't be
1525        *    well connected.
1526        *
1527        * 3. Once we reach the timeout, we can do no more.  We must schedule the
1528        *    next iteration of get requests regardless of connections that peers
1529        *    may or may not have.
1530        *
1531        * Caveat: it would be nice to get peers to take data offline with them and
1532        *         come back with it (or not) based on the testing framework.  The
1533        *         same goes for remembering previous connections, but putting either
1534        *         into the general testing churn options seems like overkill because
1535        *         these are very specialized cases.
1536        */
1537       GNUNET_log (
1538                   GNUNET_ERROR_TYPE_WARNING,
1539                   "Out of %u peers, fewest connections is %d\n",
1540                   GNUNET_CONTAINER_heap_get_size (
1541                                                   find_peer_context->peer_min_heap),
1542                   peer_count->count);
1543       if ((peer_count->count == 0)
1544           && (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1545               > 0))
1546         {
1547           GNUNET_log (
1548                       GNUNET_ERROR_TYPE_WARNING,
1549                       "Found peer with no connections, will choose some peer(s) at random to connect to!\n");
1550           GNUNET_CONTAINER_heap_iterate (find_peer_context->peer_min_heap,
1551                                          &iterate_min_heap_peers,
1552                                          find_peer_context);
1553           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1554                                     find_peer_context);
1555         }
1556       else if ((GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1557           > 0) && (find_peer_context->last_sent != 0))
1558         {
1559           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1560                                     find_peer_context);
1561         }
1562       else
1563         {
1564           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
1565                                                  &remove_peer_count,
1566                                                  find_peer_context);
1567           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
1568           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
1569           GNUNET_free(find_peer_context);
1570           GNUNET_log (
1571                       GNUNET_ERROR_TYPE_WARNING,
1572                       "Churn round %u of %llu finished, scheduling next GET round.\n",
1573                       current_churn_round, churn_rounds);
1574           if (dhtlog_handle != NULL)
1575             {
1576               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1577               topo_ctx->cont = &do_get;
1578               topo_ctx->cls = all_gets;
1579               topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1580               topo_ctx->peers_seen
1581                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1582               die_task
1583                   = GNUNET_SCHEDULER_add_delayed (
1584                                                   GNUNET_TIME_relative_add (
1585                                                                             GNUNET_TIME_relative_add (
1586                                                                                                       DEFAULT_GET_TIMEOUT,
1587                                                                                                       all_get_timeout),
1588                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1589                                                   &end_badly,
1590                                                   "from do gets (count_peers_churn_cb)");
1591               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1592             }
1593           else
1594             {
1595               die_task
1596                   = GNUNET_SCHEDULER_add_delayed (
1597                                                   GNUNET_TIME_relative_add (
1598                                                                             GNUNET_TIME_relative_add (
1599                                                                                                       DEFAULT_GET_TIMEOUT,
1600                                                                                                       all_get_timeout),
1601                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1602                                                   &end_badly,
1603                                                   "from do gets (count_peers_churn_cb)");
1604               GNUNET_SCHEDULER_add_now (&do_get, all_gets);
1605             }
1606         }
1607     }
1608 }
1609
1610 /**
1611  * Set up a single find peer request for each peer in the topology.  Do this
1612  * until the settle time is over, limited by the number of outstanding requests
1613  * and the time allowed for each one!
1614  */
1615 static void
1616 schedule_churn_find_peer_requests(void *cls,
1617     const struct GNUNET_SCHEDULER_TaskContext * tc)
1618 {
1619   struct FindPeerContext *find_peer_ctx = cls;
1620   struct TestFindPeer *test_find_peer;
1621   struct PeerCount *peer_count;
1622   uint32_t i;
1623
1624   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
1625     find_peer_ctx->total = 1;
1626   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1627       < MIN_FIND_PEER_CUTOFF)
1628     find_peer_ctx->total = find_peer_ctx->total / 2;
1629   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1630       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
1631     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
1632         / 4);
1633   else
1634     find_peer_ctx->total = find_peer_ctx->last_sent * 4;
1635
1636   if (find_peer_ctx->total > max_outstanding_find_peers)
1637     find_peer_ctx->total = max_outstanding_find_peers;
1638
1639   find_peer_ctx->last_sent = find_peer_ctx->total;
1640   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1641               "Sending %u find peer messages (after churn)\n",
1642               find_peer_ctx->total);
1643
1644   if (find_peer_ctx->total > 0)
1645     find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
1646                                                     find_peer_ctx->total);
1647   else
1648     {
1649       find_peer_ctx->previous_peers = find_peer_ctx->current_peers;
1650       find_peer_ctx->current_peers = 0;
1651       GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_ctx);
1652     }
1653
1654   for (i = 0; i < find_peer_ctx->total; i++)
1655     {
1656       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
1657       /* If we have sent requests, choose peers with a low number of connections to send requests from */
1658       peer_count
1659           = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
1660       GNUNET_assert(peer_count != NULL);
1661       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1662                   "Sending find peer request from peer with %u connections\n",
1663                   peer_count->count);
1664       GNUNET_CONTAINER_multihashmap_remove (find_peer_ctx->peer_hash,
1665                                             &peer_count->peer_id.hashPubKey,
1666                                             peer_count);
1667       test_find_peer->daemon
1668           = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1669       GNUNET_assert(test_find_peer->daemon != NULL);
1670       test_find_peer->find_peer_context = find_peer_ctx;
1671       GNUNET_SCHEDULER_add_delayed (
1672                                     GNUNET_TIME_relative_multiply (
1673                                                                    find_peer_offset,
1674                                                                    i),
1675                                     &send_find_peer_request, test_find_peer);
1676     }
1677
1678   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
1679       == NULL))
1680     {
1681       find_peer_ctx->peer_hash
1682           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1683       find_peer_ctx->peer_min_heap
1684           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1685     }
1686   else
1687     {
1688       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
1689                                              &remove_peer_count, find_peer_ctx);
1690       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
1691       find_peer_ctx->peer_hash
1692           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1693     }
1694
1695   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
1696   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
1697 }
1698
1699 static void
1700 schedule_churn_get_topology(void *cls,
1701     const struct GNUNET_SCHEDULER_TaskContext * tc)
1702 {
1703   struct FindPeerContext *find_peer_context = cls;
1704   GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_context);
1705 }
1706
1707 /**
1708  * Called when churning of the topology has finished.
1709  *
1710  * @param cls closure unused
1711  * @param emsg NULL on success, or a printable error on failure
1712  */
1713 static void
1714 churn_complete(void *cls, const char *emsg)
1715 {
1716   struct FindPeerContext *find_peer_context = cls;
1717   struct PeerCount *peer_count;
1718   unsigned int i;
1719   struct GNUNET_TESTING_Daemon *temp_daemon;
1720   struct TopologyIteratorContext *topo_ctx;
1721   struct GNUNET_TIME_Relative calc_timeout;
1722   int count_added;
1723
1724   if (emsg != NULL)
1725     {
1726       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1727                   "Ending test, churning of peers failed with error `%s'", emsg);
1728       GNUNET_SCHEDULER_add_now (&end_badly, (void *) emsg);
1729       return;
1730     }
1731
1732   /**
1733    * If we switched any peers on, we have to somehow force connect the new peer to
1734    * SOME bootstrap peer in the network.  First schedule a task to find all peers
1735    * with no connections, then choose a random peer for each and connect them.
1736    */
1737   if (find_peer_context != NULL)
1738     {
1739       GNUNET_log (
1740                   GNUNET_ERROR_TYPE_WARNING,
1741                   "We have churned on some peers, so we must schedule find peer requests for them!\n");
1742       count_added = 0;
1743       for (i = 0; i < num_peers; i++)
1744         {
1745           temp_daemon = GNUNET_TESTING_daemon_get (pg, i);
1746           if (GNUNET_YES == GNUNET_TESTING_daemon_running (temp_daemon))
1747             {
1748               peer_count = GNUNET_malloc (sizeof(struct PeerCount));
1749               memcpy (&peer_count->peer_id, &temp_daemon->id,
1750                       sizeof(struct GNUNET_PeerIdentity));
1751               GNUNET_assert(peer_count->count == 0);
1752               peer_count->heap_node
1753                   = GNUNET_CONTAINER_heap_insert (
1754                                                   find_peer_context->peer_min_heap,
1755                                                   peer_count, peer_count->count);
1756               GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1757                                                  &temp_daemon->id.hashPubKey,
1758                                                  peer_count,
1759                                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1760               count_added++;
1761             }
1762         }
1763       GNUNET_log (
1764                   GNUNET_ERROR_TYPE_WARNING,
1765                   "Added %d peers to heap, total size %d\n",
1766                   count_added,
1767                   GNUNET_CONTAINER_heap_get_size (
1768                                                   find_peer_context->peer_min_heap));
1769       GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1770                                     &schedule_churn_get_topology,
1771                                     find_peer_context);
1772     }
1773   else
1774     {
1775       GNUNET_log (
1776                   GNUNET_ERROR_TYPE_WARNING,
1777                   "Only churned off peers, no find peer requests, scheduling more gets (after allowing time for peers to disconnect properly!)...\n");
1778       if (dhtlog_handle != NULL)
1779         {
1780           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1781           topo_ctx->cont = &do_get;
1782           topo_ctx->cls = all_gets;
1783           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1784           topo_ctx->peers_seen
1785               = GNUNET_CONTAINER_multihashmap_create (num_peers);
1786           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1787                                                    all_get_timeout);
1788           calc_timeout
1789               = GNUNET_TIME_relative_add (calc_timeout,
1790                                           DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT);
1791           calc_timeout
1792               = GNUNET_TIME_relative_add (calc_timeout,
1793                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1794           die_task
1795               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1796                                               "from do gets (churn_complete)");
1797           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1798                                         &capture_current_topology, topo_ctx);
1799           dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1800         }
1801       else
1802         {
1803           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1804                                                    all_get_timeout);
1805           calc_timeout
1806               = GNUNET_TIME_relative_add (calc_timeout,
1807                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1808           die_task
1809               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1810                                               "from do gets (churn_complete)");
1811           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1812                                         &do_get, all_gets);
1813         }
1814     }
1815 }
1816
1817 /**
1818  * Decide how many peers to turn on or off in this round, make sure the
1819  * numbers actually make sense, then do so.  This function sets in motion
1820  * churn, find peer requests for newly joined peers, and issuing get
1821  * requests once the new peers have done so.
1822  *
1823  * @param cls closure (unused)
1824  * @param tc task context (unused)
1825  */
1826 static void
1827 churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1828 {
1829   unsigned int count_running;
1830   unsigned int churn_up;
1831   unsigned int churn_down;
1832   struct GNUNET_TIME_Relative timeout;
1833   struct FindPeerContext *find_peer_context;
1834
1835   churn_up = churn_down = 0;
1836   count_running = GNUNET_TESTING_daemons_running (pg);
1837   if (count_running > churn_array[current_churn_round])
1838     churn_down = count_running - churn_array[current_churn_round];
1839   else if (count_running < churn_array[current_churn_round])
1840     churn_up = churn_array[current_churn_round] - count_running;
1841   else
1842     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1843                 "Not churning any peers, topology unchanged.\n");
1844
1845   if (churn_up > num_peers - count_running)
1846     {
1847       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1848                   "Churn file specified %u peers (up); only have %u!",
1849                   churn_array[current_churn_round], num_peers);
1850       churn_up = num_peers - count_running;
1851     }
1852   else if (churn_down > count_running)
1853     {
1854       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1855                   "Churn file specified %u peers (down); only have %u!",
1856                   churn_array[current_churn_round], count_running);
1857       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1858                   "This will leave NO peers running (mistake in churn configuration?)!");
1859       churn_down = count_running;
1860     }
1861   //timeout = GNUNET_TIME_relative_multiply(seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1862   //timeout = GNUNET_TIME_relative_multiply (seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1863   timeout = GNUNET_TIME_relative_multiply (DEFAULT_TIMEOUT, 2); /* FIXME: Lack of intelligent choice here */
1864   find_peer_context = NULL;
1865   if (churn_up > 0) /* Only need to do find peer requests if we turned new peers on */
1866     {
1867       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
1868       find_peer_context->count_peers_cb = &count_peers_churn_cb;
1869       find_peer_context->previous_peers = 0;
1870       find_peer_context->current_peers = 0;
1871       find_peer_context->endtime = GNUNET_TIME_relative_to_absolute (timeout);
1872       find_peer_context->peer_hash
1873           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1874       find_peer_context->peer_min_heap
1875           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1876     }
1877   GNUNET_log (
1878               GNUNET_ERROR_TYPE_WARNING,
1879               "churn_peers: want %u total, %u running, starting %u, stopping %u\n",
1880               churn_array[current_churn_round], count_running, churn_up,
1881               churn_down);
1882   GNUNET_TESTING_daemons_churn (pg, churn_down, churn_up, timeout,
1883                                 &churn_complete, find_peer_context);
1884   current_churn_round++;
1885 }
1886
1887 /**
1888  * Task to release DHT handle associated with GET request.
1889  */
1890 static void
1891 get_stop_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1892 {
1893   struct TestGetContext *test_get = cls;
1894   struct TopologyIteratorContext *topo_ctx;
1895
1896   /* The dht_handle may be null if this get was scheduled from a down peer */
1897   if (test_get->dht_handle != NULL)
1898     {
1899       GNUNET_DHT_disconnect (test_get->dht_handle);
1900       outstanding_gets--; /* GET is really finished */
1901       test_get->dht_handle = NULL;
1902     }
1903
1904   /* Reset the uid (which item to search for) and the daemon (which peer to search from) for later get request iterations */
1905   if (get_from_same == GNUNET_NO)
1906     {
1907       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1908                                                 num_puts);
1909       test_get->daemon
1910           = GNUNET_TESTING_daemon_get (
1911                                        pg,
1912                                        GNUNET_CRYPTO_random_u32 (
1913                                                                  GNUNET_CRYPTO_QUALITY_WEAK,
1914                                                                  num_peers));
1915     }
1916
1917 #if VERBOSE > 1
1918   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
1919 #endif
1920   update_meter (get_meter);
1921   if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0))
1922     {
1923       fprintf (
1924                stderr,
1925                "Canceling die task (get_stop_finished) %llu gets completed, %llu gets failed\n",
1926                gets_completed, gets_failed);
1927       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
1928         dhtlog_handle->insert_round_details (DHT_ROUND_GET, rounds_finished,
1929                                              num_gets, gets_completed);
1930       GNUNET_SCHEDULER_cancel (die_task);
1931       reset_meter (put_meter);
1932       reset_meter (get_meter);
1933       /**
1934        *  Handle all cases:
1935        *    1) Testing is completely finished, call the topology iteration dealy and die
1936        *    2) Testing is not finished, churn the network and do gets again (current_churn_round < churn_rounds)
1937        *    3) Testing is not finished, reschedule all the PUTS *and* GETS again (num_rounds > 1)
1938        */
1939       if (rounds_finished == total_rounds - 1) /* Everything is finished, end testing */
1940         {
1941           if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
1942             {
1943               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1944               topo_ctx->cont = &log_dht_statistics;
1945               topo_ctx->peers_seen
1946                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1947               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1948             }
1949           else
1950             GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1951         }
1952       else if (current_churn_round < churns_per_round * (rounds_finished + 1)) /* Do next round of churn */
1953         {
1954           GNUNET_log (
1955                       GNUNET_ERROR_TYPE_WARNING,
1956                       "Current churn round %u, real round %u, scheduling next round of churn.\n",
1957                       current_churn_round, rounds_finished + 1);
1958           gets_completed = 0;
1959           gets_failed = 0;
1960
1961           if (dhtlog_handle != NULL)
1962             dhtlog_handle->insert_round (DHT_ROUND_CHURN, rounds_finished);
1963
1964           GNUNET_SCHEDULER_add_now (&churn_peers, NULL);
1965         }
1966       else if (rounds_finished < total_rounds - 1) /* Start a new complete round */
1967         {
1968           rounds_finished++;
1969           gets_completed = 0;
1970           gets_failed = 0;
1971           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1972                       "Round %u of %llu finished, scheduling next round.\n",
1973                       rounds_finished, total_rounds);
1974
1975           /** We reset the peer daemon for puts and gets on each disconnect, so all we need to do is start another round! */
1976           if (GNUNET_YES == in_dht_replication) /* Replication done in DHT, don't redo puts! */
1977             {
1978               if (dhtlog_handle != NULL)
1979                 dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1980
1981               die_task
1982                   = GNUNET_SCHEDULER_add_delayed (
1983                                                   GNUNET_TIME_relative_add (
1984                                                                             GNUNET_TIME_relative_add (
1985                                                                                                       GNUNET_TIME_relative_multiply (
1986                                                                                                                                      GNUNET_TIME_UNIT_SECONDS,
1987                                                                                                                                      round_delay),
1988                                                                                                       all_get_timeout),
1989                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1990                                                   &end_badly,
1991                                                   "from do gets (next round)");
1992               GNUNET_SCHEDULER_add_delayed (
1993                                             GNUNET_TIME_relative_multiply (
1994                                                                            GNUNET_TIME_UNIT_SECONDS,
1995                                                                            round_delay),
1996                                             &do_get, all_gets);
1997             }
1998           else
1999             {
2000               if (dhtlog_handle != NULL)
2001                 dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2002               die_task
2003                   = GNUNET_SCHEDULER_add_delayed (
2004                                                   GNUNET_TIME_relative_add (
2005                                                                             GNUNET_TIME_relative_multiply (
2006                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2007                                                                                                            round_delay),
2008                                                                             GNUNET_TIME_relative_multiply (
2009                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2010                                                                                                            num_puts
2011                                                                                                                * 2)),
2012                                                   &end_badly, "from do puts");
2013               GNUNET_SCHEDULER_add_delayed (
2014                                             GNUNET_TIME_relative_multiply (
2015                                                                            GNUNET_TIME_UNIT_SECONDS,
2016                                                                            round_delay),
2017                                             &do_put, all_puts);
2018             }
2019         }
2020     }
2021 }
2022
2023 /**
2024  * Task to release get handle.
2025  */
2026 static void
2027 get_stop_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2028 {
2029   struct TestGetContext *test_get = cls;
2030
2031   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2032     gets_failed++;
2033   else
2034     cumulative_successful_gets++;
2035
2036   GNUNET_assert(test_get->get_handle != NULL);
2037   GNUNET_DHT_get_stop (test_get->get_handle);
2038   test_get->get_handle = NULL;
2039   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2040   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2041 }
2042
2043 /**
2044  * Iterator called if the GET request initiated returns a response.
2045  *
2046  * @param cls closure
2047  * @param exp when will this value expire
2048  * @param key key of the result
2049  * @param get_path NULL-terminated array of pointers
2050  *                 to the peers on reverse GET path (or NULL if not recorded)
2051  * @param put_path NULL-terminated array of pointers
2052  *                 to the peers on the PUT path (or NULL if not recorded)
2053  * @param type type of the result
2054  * @param size number of bytes in data
2055  * @param data pointer to the result data
2056  */
2057 static void
2058 get_result_iterator(void *cls, struct GNUNET_TIME_Absolute exp,
2059     const GNUNET_HashCode * key,
2060     const struct GNUNET_PeerIdentity * const *get_path,
2061     const struct GNUNET_PeerIdentity * const *put_path,
2062     enum GNUNET_BLOCK_Type type, size_t size, const void *data)
2063 {
2064   struct TestGetContext *test_get = cls;
2065
2066   if (test_get->succeeded == GNUNET_YES)
2067     return; /* Get has already been successful, probably ending now */
2068
2069   if (0 != memcmp (&known_keys[test_get->uid], key, sizeof(GNUNET_HashCode))) /* || (0 != memcmp(original_data, data, sizeof(original_data))))*/
2070     {
2071       gets_completed++;
2072       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2073                   "Key or data is not the same as was inserted!\n");
2074     }
2075   else
2076     {
2077       gets_completed++;
2078       test_get->succeeded = GNUNET_YES;
2079     }
2080 #if VERBOSE > 1
2081   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
2082 #endif
2083   GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
2084   GNUNET_SCHEDULER_add_continuation (&get_stop_task, test_get,
2085                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
2086 }
2087
2088 /**
2089  * Set up some data, and call API PUT function
2090  */
2091 static void
2092 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2093 {
2094   struct TestGetContext *test_get = cls;
2095
2096   if (num_gets == 0)
2097     {
2098       GNUNET_SCHEDULER_cancel (die_task);
2099       GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
2100     }
2101
2102   if (test_get == NULL)
2103     return; /* End of the list */
2104
2105   /* Set this here in case we are re-running gets */
2106   test_get->succeeded = GNUNET_NO;
2107
2108   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_get->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2109     {
2110       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2111                   "Peer we should issue get request from is down, skipping.\n");
2112       gets_failed++;
2113       GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2114       GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2115       return;
2116     }
2117
2118   /* Check if more gets are outstanding than should be */
2119   if (outstanding_gets > max_outstanding_gets)
2120     {
2121       GNUNET_SCHEDULER_add_delayed (
2122                                     GNUNET_TIME_relative_multiply (
2123                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2124                                                                    200),
2125                                     &do_get, test_get);
2126       return;
2127     }
2128
2129   /* Connect to the first peer's DHT */
2130   test_get->dht_handle = GNUNET_DHT_connect (test_get->daemon->cfg, 10);
2131   GNUNET_assert(test_get->dht_handle != NULL);
2132   outstanding_gets++;
2133
2134   cumulative_num_gets++;
2135   /* Insert the data at the first peer */
2136   test_get->get_handle = GNUNET_DHT_get_start (test_get->dht_handle, get_delay,
2137                                                GNUNET_BLOCK_TYPE_TEST,
2138                                                &known_keys[test_get->uid],
2139                                                get_replication,
2140                                                GNUNET_DHT_RO_NONE, NULL, 0,
2141                                                NULL, 0, &get_result_iterator,
2142                                                test_get);
2143
2144 #if VERBOSE > 1
2145   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
2146       test_get->uid,
2147       test_get->daemon->shortname);
2148 #endif
2149   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed (get_timeout,
2150                                                             &get_stop_task,
2151                                                             test_get);
2152
2153   /* Schedule the next request in the linked list of get requests */
2154   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2155 }
2156
2157 /**
2158  * Called when the PUT request has been transmitted to the DHT service.
2159  * Schedule the GET request for some time in the future.
2160  */
2161 static void
2162 put_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2163 {
2164   struct TestPutContext *test_put = cls;
2165   struct TopologyIteratorContext *topo_ctx;
2166   outstanding_puts--;
2167   puts_completed++;
2168
2169   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2170     fprintf (stderr, "PUT Request failed!\n");
2171
2172   /* Reset the daemon (which peer to insert at) for later put request iterations */
2173   if (replicate_same == GNUNET_NO)
2174     test_put->daemon
2175         = GNUNET_TESTING_daemon_get (
2176                                      pg,
2177                                      GNUNET_CRYPTO_random_u32 (
2178                                                                GNUNET_CRYPTO_QUALITY_WEAK,
2179                                                                num_peers));
2180
2181   GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
2182   test_put->disconnect_task = GNUNET_SCHEDULER_add_now (&put_disconnect_task,
2183                                                         test_put);
2184   if (GNUNET_YES == update_meter (put_meter))
2185     {
2186       GNUNET_assert(outstanding_puts == 0);
2187       GNUNET_SCHEDULER_cancel (die_task);
2188       if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2189         {
2190           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2191           topo_ctx->cont = &do_get;
2192           topo_ctx->cls = all_gets;
2193           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
2194           topo_ctx->peers_seen
2195               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2196           die_task
2197               = GNUNET_SCHEDULER_add_delayed (
2198                                               GNUNET_TIME_relative_add (
2199                                                                         GNUNET_TIME_relative_add (
2200                                                                                                   DEFAULT_GET_TIMEOUT,
2201                                                                                                   all_get_timeout),
2202                                                                         DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2203                                               &end_badly,
2204                                               "from do gets (put finished)");
2205           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
2206         }
2207       else
2208         {
2209           fprintf (stderr, "Scheduling die task (put finished)\n");
2210           die_task
2211               = GNUNET_SCHEDULER_add_delayed (
2212                                               GNUNET_TIME_relative_add (
2213                                                                         DEFAULT_GET_TIMEOUT,
2214                                                                         all_get_timeout),
2215                                               &end_badly,
2216                                               "from do gets (put finished)");
2217           GNUNET_SCHEDULER_add_delayed (DEFAULT_GET_TIMEOUT, &do_get, all_gets);
2218         }
2219       return;
2220     }
2221 }
2222
2223 /**
2224  * Set up some data, and call API PUT function
2225  */
2226 static void
2227 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2228 {
2229   struct TestPutContext *test_put = cls;
2230   char data[test_data_size]; /* Made up data to store */
2231   uint32_t rand;
2232   int i;
2233
2234   if (test_put == NULL)
2235     return; /* End of list */
2236
2237   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_put->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2238     {
2239       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2240                   "Peer we should issue put request at is down, skipping.\n");
2241       update_meter (put_meter);
2242       GNUNET_SCHEDULER_add_now (&do_put, test_put->next);
2243       return;
2244     }
2245
2246   for (i = 0; i < sizeof(data); i++)
2247     {
2248       memset (&data[i], GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2249                                                   UINT32_MAX), 1);
2250     }
2251
2252   if (outstanding_puts > max_outstanding_puts)
2253     {
2254       GNUNET_SCHEDULER_add_delayed (
2255                                     GNUNET_TIME_relative_multiply (
2256                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2257                                                                    200),
2258                                     &do_put, test_put);
2259       return;
2260     }
2261
2262 #if VERBOSE > 1
2263   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
2264       test_put->uid,
2265       test_put->daemon->shortname);
2266 #endif
2267   test_put->dht_handle = GNUNET_DHT_connect (test_put->daemon->cfg, 10);
2268
2269   GNUNET_assert(test_put->dht_handle != NULL);
2270   outstanding_puts++;
2271   GNUNET_DHT_put (test_put->dht_handle, &known_keys[test_put->uid],
2272                   put_replication, GNUNET_DHT_RO_NONE, GNUNET_BLOCK_TYPE_TEST,
2273                   sizeof(data), data, GNUNET_TIME_UNIT_FOREVER_ABS, put_delay,
2274                   &put_finished, test_put);
2275   test_put->disconnect_task
2276       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever (),
2277                                       &put_disconnect_task, test_put);
2278   rand = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2);
2279   GNUNET_SCHEDULER_add_delayed (
2280                                 GNUNET_TIME_relative_multiply (
2281                                                                GNUNET_TIME_UNIT_SECONDS,
2282                                                                rand), &do_put,
2283                                 test_put->next);
2284 }
2285
2286 static void
2287 schedule_find_peer_requests(void *cls,
2288     const struct GNUNET_SCHEDULER_TaskContext * tc);
2289
2290 static void
2291     setup_malicious_peers(void *cls,
2292         const struct GNUNET_SCHEDULER_TaskContext * tc);
2293
2294 /**
2295  * Given a number of total peers and a bucket size, estimate the number of
2296  * connections in a perfect kademlia topology.
2297  */
2298 static unsigned int
2299 connection_estimate(unsigned int peer_count, unsigned int bucket_size)
2300 {
2301   unsigned int i;
2302   unsigned int filled;
2303   i = num_peers;
2304
2305   filled = 0;
2306   while (i >= bucket_size)
2307     {
2308       filled++;
2309       i = i / 2;
2310     }
2311   filled++; /* Add one filled bucket to account for one "half full" and some miscellaneous */
2312   return filled * bucket_size * peer_count;
2313
2314 }
2315
2316 /**
2317  * Callback for iterating over all the peer connections of a peer group.
2318  */
2319 static void
2320 count_peers_cb(void *cls, const struct GNUNET_PeerIdentity *first,
2321     const struct GNUNET_PeerIdentity *second, const char *emsg)
2322 {
2323   struct FindPeerContext *find_peer_context = cls;
2324   if ((first != NULL) && (second != NULL))
2325     {
2326       add_new_connection (find_peer_context, first, second);
2327       find_peer_context->current_peers++;
2328     }
2329   else
2330     {
2331       GNUNET_log (
2332                   GNUNET_ERROR_TYPE_WARNING,
2333                   "Peer count finished (%u connections), %u new peers, connection estimate %u (target %u)\n",
2334                   find_peer_context->current_peers,
2335                   find_peer_context->current_peers
2336                       - find_peer_context->previous_peers,
2337                   connection_estimate (num_peers, DEFAULT_BUCKET_SIZE),
2338                   target_total_connections);
2339
2340       if ((find_peer_context->last_sent < 8)
2341           || ((find_peer_context->current_peers < 2
2342               * connection_estimate (num_peers, DEFAULT_BUCKET_SIZE))
2343               && (GNUNET_TIME_absolute_get_remaining (
2344                                                       find_peer_context->endtime).rel_value
2345                   > 0) && (find_peer_context->current_peers
2346               < target_total_connections)))
2347         {
2348           GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests,
2349                                     find_peer_context);
2350         }
2351       else
2352         {
2353           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
2354                                                  &remove_peer_count,
2355                                                  find_peer_context);
2356           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
2357           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
2358           GNUNET_free(find_peer_context);
2359           fprintf (stderr, "Not sending any more find peer requests.\n");
2360
2361 #if HAVE_MALICIOUS
2362           if (GNUNET_YES == malicious_after_settle)
2363             {
2364               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "calling setup_malicious_peers\n");
2365               GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2366             }
2367 #endif
2368         }
2369     }
2370 }
2371
2372 /**
2373  * Set up a single find peer request for each peer in the topology.  Do this
2374  * until the settle time is over, limited by the number of outstanding requests
2375  * and the time allowed for each one!
2376  */
2377 static void
2378 schedule_find_peer_requests(void *cls,
2379     const struct GNUNET_SCHEDULER_TaskContext * tc)
2380 {
2381   struct FindPeerContext *find_peer_ctx = cls;
2382   struct TestFindPeer *test_find_peer;
2383   struct PeerCount *peer_count;
2384   uint32_t i;
2385   uint32_t random;
2386
2387   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
2388     find_peer_ctx->total = 1;
2389   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
2390       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
2391     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
2392         / 8);
2393   else
2394     find_peer_ctx->total = find_peer_ctx->last_sent * 2;
2395
2396   if (find_peer_ctx->total > max_outstanding_find_peers)
2397     find_peer_ctx->total = max_outstanding_find_peers;
2398
2399   if (find_peer_ctx->total > num_peers) /* Don't try to send more messages than we have peers! */
2400     find_peer_ctx->total = num_peers;
2401
2402   find_peer_ctx->last_sent = find_peer_ctx->total;
2403   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2404               "Sending %u find peer messages (goal at least %u connections)\n",
2405               find_peer_ctx->total, target_total_connections);
2406
2407   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
2408                                                   find_peer_ctx->total);
2409   for (i = 0; i < find_peer_ctx->total; i++)
2410     {
2411       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
2412       /* If we haven't sent any requests yet, choose random peers */
2413       /* Also choose random in _half_ of all cases, so we don't
2414        * get stuck choosing topologically restricted peers with
2415        * few connections that will never be able to find any new
2416        * peers! */
2417       if ((find_peer_ctx->previous_peers == 0) || (i % 2 == 0))
2418         {
2419           /**
2420            * Attempt to spread find peer requests across even sections of the peer address
2421            * space.  Choose basically 1 peer in every num_peers / max_outstanding_requests
2422            * each time, then offset it by a randomish value.
2423            *
2424            * For instance, if num_peers is 100 and max_outstanding is 10, first chosen peer
2425            * will be between 0 - 10, second between 10 - 20, etc.
2426            */
2427           random = (num_peers / find_peer_ctx->total) * i;
2428           random = random
2429               + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2430                                           (num_peers / find_peer_ctx->total));
2431           if (random >= num_peers)
2432             {
2433               random = random - num_peers;
2434             }
2435 #if REAL_RANDOM
2436           random = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2437 #endif
2438           test_find_peer->daemon = GNUNET_TESTING_daemon_get (pg, random);
2439         }
2440       else /* If we have sent requests, choose peers with a low number of connections to send requests from */
2441         {
2442           peer_count
2443               = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
2444           GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(find_peer_ctx->peer_hash, &peer_count->peer_id.hashPubKey, peer_count));
2445           test_find_peer->daemon
2446               = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
2447           GNUNET_assert(test_find_peer->daemon != NULL);
2448         }
2449
2450       test_find_peer->find_peer_context = find_peer_ctx;
2451       GNUNET_SCHEDULER_add_delayed (
2452                                     GNUNET_TIME_relative_multiply (
2453                                                                    find_peer_offset,
2454                                                                    i),
2455                                     &send_find_peer_request, test_find_peer);
2456     }
2457
2458   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
2459       == NULL))
2460     {
2461       find_peer_ctx->peer_hash
2462           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2463       find_peer_ctx->peer_min_heap
2464           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2465     }
2466   else
2467     {
2468       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
2469                                              &remove_peer_count, find_peer_ctx);
2470       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
2471       find_peer_ctx->peer_hash
2472           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2473     }
2474
2475   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
2476   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
2477
2478 }
2479
2480 /**
2481  * Convert unique ID to hash code.
2482  *
2483  * @param uid unique ID to convert
2484  * @param hash set to uid (extended with zeros)
2485  */
2486 static void
2487 hash_from_uid(uint32_t uid, GNUNET_HashCode *hash)
2488 {
2489   memset (hash, 0, sizeof(GNUNET_HashCode));
2490   *((uint32_t *) hash) = uid;
2491 }
2492
2493 /**
2494  * Set up all of the put and get operations we want to do
2495  * in the current round.  Allocate data structure for each,
2496  * add to list, then schedule the actual PUT operations.
2497  */
2498 static void
2499 setup_puts_and_gets(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2500 {
2501   int i;
2502   struct TestPutContext *test_put;
2503   struct TestGetContext *test_get;
2504   uint32_t temp_peer;
2505   GNUNET_HashCode uid_hash;
2506   int count;
2507 #if REMEMBER
2508   int remember[num_puts][num_peers];
2509   memset(&remember, 0, sizeof(int) * num_puts * num_peers);
2510 #endif
2511   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "in setup_puts_and_gets\n");
2512   known_keys = GNUNET_malloc(sizeof(GNUNET_HashCode) * num_puts);
2513   for (i = 0; i < num_puts; i++)
2514     {
2515       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
2516       test_put->uid = i;
2517       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
2518                                         &known_keys[i]);
2519       /* Set first X bits to match the chosen sybil location if we want to do the sybil attack! */
2520       if (GNUNET_YES == malicious_sybil)
2521         {
2522           memcpy (&known_keys[i], &sybil_target, sizeof(GNUNET_HashCode) / 2);
2523           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2524                       "Distance between sybil location and key is %d\n",
2525                       GNUNET_CRYPTO_hash_matching_bits (&known_keys[i],
2526                                                         &sybil_target));
2527         }
2528       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2529                                             num_peers);
2530       test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2531       /* Don't start PUTs at malicious peers! */
2532       if (malicious_bloom != NULL)
2533         {
2534           count = 0;
2535           hash_from_uid (temp_peer, &uid_hash);
2536           while ((GNUNET_YES
2537               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2538               && (count < num_peers))
2539             {
2540               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2541                                                     num_peers);
2542               hash_from_uid (temp_peer, &uid_hash);
2543               test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2544               count++;
2545             }
2546           if (count == num_peers)
2547             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2548                         "Couldn't find peer not in malicious bloom to select!\n");
2549         }
2550
2551       test_put->next = all_puts;
2552       all_puts = test_put;
2553     }
2554
2555   for (i = 0; i < num_gets; i++)
2556     {
2557       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
2558       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2559                                                 num_puts);
2560 #if REMEMBER
2561       while (remember[test_get->uid][temp_daemon] == 1)
2562       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2563       remember[test_get->uid][temp_daemon] = 1;
2564 #endif
2565       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2566                                             num_peers);
2567       test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2568       /* Don't start GETs at malicious peers! */
2569       if (malicious_bloom != NULL)
2570         {
2571           hash_from_uid (temp_peer, &uid_hash);
2572           count = 0;
2573           while ((GNUNET_YES
2574               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2575               && (count < num_peers))
2576             {
2577               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2578                                                     num_peers);
2579               hash_from_uid (temp_peer, &uid_hash);
2580               test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2581               count++;
2582             }
2583           if (count == num_peers)
2584             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2585                         "Couldn't find peer not in malicious bloom to select!\n");
2586         }
2587       test_get->next = all_gets;
2588       all_gets = test_get;
2589     }
2590
2591   /*GNUNET_SCHEDULER_cancel (die_task);*/
2592   die_task
2593       = GNUNET_SCHEDULER_add_delayed (
2594                                       GNUNET_TIME_relative_multiply (
2595                                                                      GNUNET_TIME_UNIT_SECONDS,
2596                                                                      num_puts
2597                                                                          * 2),
2598                                       &end_badly, "from do puts");
2599   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
2600
2601 }
2602
2603 /**
2604  * Set up some all of the put and get operations we want
2605  * to do.  Allocate data structure for each, add to list,
2606  * then call actual insert functions.
2607  */
2608 static void
2609 continue_puts_and_gets(void *cls,
2610     const struct GNUNET_SCHEDULER_TaskContext * tc)
2611 {
2612   int i;
2613   int max;
2614   struct TopologyIteratorContext *topo_ctx;
2615   struct FindPeerContext *find_peer_context;
2616   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "In continue_puts_and_gets\n");
2617   if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2618     {
2619       if (settle_time >= 180 * 2)
2620         max = (settle_time / 180) - 2;
2621       else
2622         max = 1;
2623       for (i = 1; i < max; i++)
2624         {
2625           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2626           topo_ctx->current_iteration = i;
2627           topo_ctx->total_iterations = max;
2628           topo_ctx->peers_seen
2629               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2630           //fprintf(stderr, "scheduled topology iteration in %d minutes\n", i);
2631           GNUNET_SCHEDULER_add_delayed (
2632                                         GNUNET_TIME_relative_multiply (
2633                                                                        GNUNET_TIME_UNIT_MINUTES,
2634                                                                        i * 3),
2635                                         &capture_current_topology, topo_ctx);
2636         }
2637       topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2638       topo_ctx->cont = &setup_puts_and_gets;
2639       topo_ctx->peers_seen = GNUNET_CONTAINER_multihashmap_create (num_peers);
2640       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2641                   "setting setup_puts_and_gets for %d seconds in the future\n",
2642                   settle_time + 10);
2643       GNUNET_SCHEDULER_add_delayed (
2644                                     GNUNET_TIME_relative_multiply (
2645                                                                    GNUNET_TIME_UNIT_SECONDS,
2646                                                                    (settle_time
2647                                                                        + 10)),
2648                                     &capture_current_topology, topo_ctx);
2649     }
2650   else
2651     GNUNET_SCHEDULER_add_delayed (
2652                                   GNUNET_TIME_relative_multiply (
2653                                                                  GNUNET_TIME_UNIT_SECONDS,
2654                                                                  (settle_time
2655                                                                      + 10)),
2656                                   &setup_puts_and_gets, NULL);
2657
2658   if (dhtlog_handle != NULL)
2659     dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2660
2661 #if HAVE_MALICIOUS
2662   if ((GNUNET_YES != malicious_after_settle) || (settle_time == 0))
2663     {
2664       GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2665     }
2666 #endif
2667
2668   if ((GNUNET_YES == do_find_peer) && (settle_time > 0))
2669     {
2670       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2671                   "Scheduling find peer requests during \"settle\" time.\n");
2672       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
2673       find_peer_context->count_peers_cb = &count_peers_cb;
2674       find_peer_context->endtime
2675           = GNUNET_TIME_relative_to_absolute (
2676                                               GNUNET_TIME_relative_multiply (
2677                                                                              GNUNET_TIME_UNIT_SECONDS,
2678                                                                              settle_time));
2679       GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests, find_peer_context);
2680     }
2681   else
2682     {
2683       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2684                   "Assuming automatic DHT find peer requests.\n");
2685     }
2686 }
2687
2688 /**
2689  * Task to release DHT handles
2690  */
2691 static void
2692 malicious_disconnect_task(void *cls,
2693                           const struct GNUNET_SCHEDULER_TaskContext * tc)
2694 {
2695   struct MaliciousContext *ctx = cls;
2696   outstanding_malicious--;
2697   malicious_completed++;
2698   ctx->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2699   GNUNET_DHT_disconnect (ctx->dht_handle);
2700   ctx->dht_handle = NULL;
2701   GNUNET_free(ctx);
2702
2703   if (malicious_completed == malicious_getters + malicious_putters
2704       + malicious_droppers)
2705     {
2706       fprintf (stderr, "Finished setting all malicious peers up!\n");
2707     }
2708 }
2709
2710 /**
2711  * Task to release DHT handles
2712  */
2713 static void
2714 malicious_done_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2715 {
2716   struct MaliciousContext *ctx = cls;
2717   GNUNET_SCHEDULER_cancel (ctx->disconnect_task);
2718   GNUNET_SCHEDULER_add_now (&malicious_disconnect_task, ctx);
2719 }
2720
2721 /**
2722  * Set up some data, and call API PUT function
2723  */
2724 static void
2725 set_malicious(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2726 {
2727   struct MaliciousContext *ctx = cls;
2728
2729   if (outstanding_malicious > DEFAULT_MAX_OUTSTANDING_GETS)
2730     {
2731       GNUNET_SCHEDULER_add_delayed (
2732                                     GNUNET_TIME_relative_multiply (
2733                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2734                                                                    100),
2735                                     &set_malicious, ctx);
2736       return;
2737     }
2738
2739   if (ctx->dht_handle == NULL)
2740     {
2741       ctx->dht_handle = GNUNET_DHT_connect (ctx->daemon->cfg, 1);
2742       outstanding_malicious++;
2743     }
2744
2745   GNUNET_assert(ctx->dht_handle != NULL);
2746
2747 #if VERBOSE > 1
2748   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Setting peer %s malicious type %d\n",
2749       ctx->daemon->shortname, ctx->malicious_type);
2750 #endif
2751
2752   switch (ctx->malicious_type)
2753     {
2754   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET:
2755     GNUNET_DHT_set_malicious_getter (ctx->dht_handle, malicious_get_frequency, &malicious_done_task, ctx);
2756     break;
2757   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT:
2758     GNUNET_DHT_set_malicious_putter (ctx->dht_handle, malicious_put_frequency, &malicious_done_task, ctx);
2759     break;
2760   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP:
2761     GNUNET_DHT_set_malicious_dropper (ctx->dht_handle, &malicious_done_task, ctx);
2762     break;
2763   default:
2764     break;
2765     }
2766
2767   ctx->disconnect_task
2768       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
2769                                       &malicious_disconnect_task, ctx);
2770 }
2771
2772 #if HAVE_MALICIOUS
2773 /**
2774  * Choose the next peer from the peer group to set as malicious.
2775  * If we are doing a sybil attack, find the nearest peer to the
2776  * sybil location that has not already been set malicious.  Otherwise
2777  * just choose a random not already chosen peer.
2778  *
2779  * @param pg the peer group
2780  * @param bloom the bloomfilter which contains all peer already
2781  *        chosen to be malicious
2782  */
2783 static uint32_t
2784 choose_next_malicious (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_CONTAINER_BloomFilter *bloom)
2785   {
2786     int i;
2787     int nearest;
2788     int bits_match;
2789     int curr_distance;
2790     int count;
2791     struct GNUNET_TESTING_Daemon *temp_daemon;
2792     GNUNET_HashCode uid_hash;
2793
2794     curr_distance = 0;
2795     nearest = 0;
2796     GNUNET_assert (bloom != NULL);
2797
2798     if (GNUNET_YES == malicious_sybil)
2799       {
2800         for (i = 0; i < num_peers; i++)
2801           {
2802             temp_daemon = GNUNET_TESTING_daemon_get(pg, i);
2803             hash_from_uid(i, &uid_hash);
2804             /* Check if this peer matches the bloomfilter */
2805             if ((GNUNET_NO == GNUNET_TESTING_daemon_running(temp_daemon)) || (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)))
2806             continue;
2807
2808             bits_match = GNUNET_CRYPTO_hash_matching_bits (&temp_daemon->id.hashPubKey, &sybil_target);
2809             if (bits_match >= curr_distance)
2810               {
2811                 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found nearer peer %s to %s, old matching bits %d, new %d\n", GNUNET_i2s(&temp_daemon->id), GNUNET_h2s(&sybil_target), curr_distance, bits_match);
2812                 nearest = i;
2813                 curr_distance = bits_match;
2814               }
2815           }
2816       }
2817     else
2818       {
2819         nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2820         hash_from_uid(nearest, &uid_hash);
2821         count = 0;
2822         while ((GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)) && (count < num_peers))
2823           {
2824             GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Peer %d already in bloom (tried %d times)\n", nearest, count);
2825             nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2826             hash_from_uid(nearest, &uid_hash);
2827             count++;
2828           }
2829         if (count == num_peers)
2830         GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Tried %d times to find a peer, selecting %d at random!!\n", count, nearest);
2831       }
2832
2833     return nearest;
2834   }
2835
2836 /**
2837  * Select randomly from set of known peers,
2838  * set the desired number of peers to the
2839  * proper malicious types.
2840  */
2841 static void
2842 setup_malicious_peers (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2843   {
2844     struct MaliciousContext *ctx;
2845     int i;
2846     uint32_t temp_daemon;
2847     GNUNET_HashCode uid_hash;
2848
2849     for (i = 0; i < malicious_getters; i++)
2850       {
2851         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2852         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2853         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2854         hash_from_uid(temp_daemon, &uid_hash);
2855         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2856         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET;
2857         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2858
2859       }
2860
2861     for (i = 0; i < malicious_putters; i++)
2862       {
2863         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2864         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2865         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2866         hash_from_uid(temp_daemon, &uid_hash);
2867         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2868         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT;
2869         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2870
2871       }
2872
2873     for (i = 0; i < malicious_droppers; i++)
2874       {
2875         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2876         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2877         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2878         hash_from_uid(temp_daemon, &uid_hash);
2879         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2880         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP;
2881         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2882       }
2883   }
2884 #endif
2885
2886 #if ONLY_TESTING
2887 /* Forward declaration */
2888 static void
2889 topology_callback (void *cls,
2890     const struct GNUNET_PeerIdentity *first,
2891     const struct GNUNET_PeerIdentity *second,
2892     uint32_t distance,
2893     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2894     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2895     struct GNUNET_TESTING_Daemon *first_daemon,
2896     struct GNUNET_TESTING_Daemon *second_daemon,
2897     const char *emsg);
2898
2899 /**
2900  * Retry connecting two specific peers until they connect,
2901  * at a specific interval.  These two peers previously failed
2902  * to connect, and we hope they continue to so that we can
2903  * debug the reason they are having issues.
2904  */
2905 static void
2906 repeat_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2907   {
2908
2909     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Repeating connect attempt between %s and %s.\n", repeat_connect_peer1->shortname, repeat_connect_peer2->shortname);
2910     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 1 configuration `%s'\n", repeat_connect_peer1->cfgfile);
2911     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 2 configuration `%s'\n", repeat_connect_peer2->cfgfile);
2912
2913     repeat_connect_task = GNUNET_SCHEDULER_NO_TASK;
2914     GNUNET_TESTING_daemons_connect(repeat_connect_peer1,
2915         repeat_connect_peer2,
2916         GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60),
2917         2, &topology_callback, NULL);
2918   }
2919 #endif
2920
2921 /**
2922  * This function is called whenever a connection attempt is finished between two of
2923  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
2924  * number of times this function is called should equal the number returned
2925  * from the GNUNET_TESTING_connect_topology call.
2926  *
2927  * The emsg variable is NULL on success (peers connected), and non-NULL on
2928  * failure (peers failed to connect).
2929  */
2930 static void
2931 topology_callback(void *cls, const struct GNUNET_PeerIdentity *first,
2932     const struct GNUNET_PeerIdentity *second, uint32_t distance,
2933     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2934     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2935     struct GNUNET_TESTING_Daemon *first_daemon,
2936     struct GNUNET_TESTING_Daemon *second_daemon, const char *emsg)
2937 {
2938   struct TopologyIteratorContext *topo_ctx;
2939   unsigned long long duration;
2940   unsigned long long total_duration;
2941   unsigned int new_connections;
2942   unsigned int new_failed_connections;
2943   double conns_per_sec_recent;
2944   double conns_per_sec_total;
2945   double failed_conns_per_sec_recent;
2946   double failed_conns_per_sec_total;
2947   char *temp_conn_string;
2948   char *temp_conn_failed_string;
2949   char *revision_str;
2950
2951 #if ONLY_TESTING
2952   if (repeat_connect_mode == GNUNET_YES)
2953     {
2954       if ((first_daemon == repeat_connect_peer1) &&
2955           (second_daemon == repeat_connect_peer2))
2956         {
2957           if (emsg != NULL) /* Peers failed to connect again! */
2958             {
2959               GNUNET_assert(repeat_connect_task == GNUNET_SCHEDULER_NO_TASK);
2960               repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
2961               return;
2962             }
2963           else /* Repeat peers actually connected! */
2964             {
2965               if (repeat_connect_task != GNUNET_SCHEDULER_NO_TASK)
2966               GNUNET_SCHEDULER_cancel(repeat_connect_task);
2967               repeat_connect_peer1 = NULL;
2968               repeat_connect_peer2 = NULL;
2969               repeat_connect_mode = GNUNET_NO;
2970               GNUNET_TESTING_resume_connections(pg);
2971               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Resuming normal connection mode, debug connection was successful!\n");
2972             }
2973         }
2974     }
2975 #endif
2976
2977   if (GNUNET_TIME_absolute_get_difference (connect_last_time,
2978                                            GNUNET_TIME_absolute_get ()).rel_value
2979       > GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
2980                                        CONN_UPDATE_DURATION).rel_value)
2981     {
2982       /* Get number of new connections */
2983       new_connections = total_connections - previous_connections;
2984
2985       /* Get number of new FAILED connections */
2986       new_failed_connections = failed_connections - previous_failed_connections;
2987
2988       /* Get duration in seconds */
2989       duration
2990           = GNUNET_TIME_absolute_get_difference (connect_last_time,
2991                                                  GNUNET_TIME_absolute_get ()).rel_value
2992               / 1000;
2993       total_duration
2994           = GNUNET_TIME_absolute_get_difference (connect_start_time,
2995                                                  GNUNET_TIME_absolute_get ()).rel_value
2996               / 1000;
2997
2998       failed_conns_per_sec_recent = (double) new_failed_connections / duration;
2999       failed_conns_per_sec_total = (double) failed_connections / total_duration;
3000       conns_per_sec_recent = (double) new_connections / duration;
3001       conns_per_sec_total = (double) total_connections / total_duration;
3002       GNUNET_log (
3003                   GNUNET_ERROR_TYPE_WARNING,
3004                   "Recent: %.2f/s, Total: %.2f/s, Recent failed: %.2f/s, total failed %.2f/s\n",
3005                   conns_per_sec_recent, CONN_UPDATE_DURATION,
3006                   conns_per_sec_total, failed_conns_per_sec_recent,
3007                   failed_conns_per_sec_total);
3008       connect_last_time = GNUNET_TIME_absolute_get ();
3009       previous_connections = total_connections;
3010       previous_failed_connections = failed_connections;
3011       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3012                   "have %u total_connections, %u failed\n", total_connections,
3013                   failed_connections);
3014 #if ONLY_TESTING
3015       /* These conditions likely mean we've entered the death spiral of doom */
3016       if ((total_connections > 20000) &&
3017           (conns_per_sec_recent < 5.0) &&
3018           (conns_per_sec_total > 10.0) &&
3019           (emsg != NULL) &&
3020           (repeat_connect_mode == GNUNET_NO))
3021         {
3022           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Entering repeat connection attempt mode!\n");
3023           repeat_connect_peer1 = first_daemon;
3024           repeat_connect_peer2 = second_daemon;
3025           repeat_connect_mode = GNUNET_YES;
3026           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping NEW connections from being scheduled!\n");
3027           GNUNET_TESTING_stop_connections(pg);
3028           repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
3029         }
3030 #endif
3031     }
3032
3033   if (emsg == NULL)
3034     {
3035       total_connections++;
3036 #if VERBOSE > 1
3037       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
3038           first_daemon->shortname,
3039           second_daemon->shortname,
3040           distance);
3041 #endif
3042     }
3043   else
3044     {
3045       failed_connections++;
3046 #if VERBOSE
3047       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peer %s to peer %s with error :\n%s\n",
3048           first_daemon->shortname,
3049           second_daemon->shortname, emsg);
3050
3051       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
3052           first_daemon->shortname,
3053           second_daemon->shortname, emsg);
3054 #endif
3055     }
3056
3057 #if ONLY_TESTING
3058   if ((repeat_connect_mode == GNUNET_YES) )
3059   return;
3060 #endif
3061
3062   GNUNET_assert(peer_connect_meter != NULL);
3063   if (GNUNET_YES == update_meter (peer_connect_meter))
3064     {
3065 #if VERBOSE
3066       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3067           "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
3068           total_connections);
3069 #endif
3070       if (failed_connections > 0)
3071         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3072                     "While connecting, had %u failed connections.\n",
3073                     failed_connections);
3074       if (dhtlog_handle != NULL)
3075         {
3076           dhtlog_handle->update_connections (total_connections);
3077           dhtlog_handle->insert_topology (expected_connections);
3078         }
3079
3080       total_duration
3081           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3082                                                  GNUNET_TIME_absolute_get ()).rel_value
3083               / 1000;
3084       failed_conns_per_sec_total = (double) failed_connections / total_duration;
3085       conns_per_sec_total = (double) total_connections / total_duration;
3086       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3087                   "Overall connection info --- Total: %u, Total Failed %u/s\n",
3088                   total_connections, failed_connections);
3089       GNUNET_log (
3090                   GNUNET_ERROR_TYPE_WARNING,
3091                   "Overall connection info --- Total: %.2f/s, Total Failed %.2f/s\n",
3092                   conns_per_sec_total, failed_conns_per_sec_total);
3093
3094       GNUNET_asprintf (&temp_conn_string,
3095                        "DHT Profiler Connection (trial %d)",
3096                        trial_to_run);
3097       GNUNET_asprintf (&temp_conn_failed_string,
3098                        "DHT Profiler Connection failed (trial %d)",
3099                        trial_to_run);
3100       GNUNET_asprintf (&revision_str, "%llu", revision);
3101       GAUGER_ID("DHT_TESTING", temp_conn_string, conns_per_sec_total, "conns/s", revision_str);
3102       GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_conns_per_sec_total, "failed_conns", revision_str);
3103       GNUNET_free(temp_conn_string);
3104       GNUNET_free(temp_conn_failed_string);
3105       GNUNET_asprintf (&temp_conn_string,
3106                        "DHT Profiler Total Connections (trial %d)",
3107                        trial_to_run);
3108       GNUNET_asprintf (
3109                        &temp_conn_failed_string,
3110                        "DHT Profiler Total Connections failed (trial %d)",
3111                        trial_to_run);
3112       GAUGER_ID("DHT_TESTING", temp_conn_string, total_connections, "conns", revision_str);
3113       GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_connections, "failed conns", revision_str);
3114       GNUNET_free(temp_conn_string);
3115       GNUNET_free(temp_conn_failed_string);
3116       GNUNET_free(revision_str);
3117
3118       GNUNET_SCHEDULER_cancel (die_task);
3119
3120       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
3121         {
3122           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
3123           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3124                       "Setting continue gets and puts as topo_cont\n");
3125           topo_ctx->cont = &continue_puts_and_gets;
3126           topo_ctx->peers_seen
3127               = GNUNET_CONTAINER_multihashmap_create (num_peers);
3128           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
3129         }
3130       else
3131         {
3132           GNUNET_log (
3133                       GNUNET_ERROR_TYPE_WARNING,
3134                       "For some reason, NOT scheduling final topology capture (settle_time %d, dhtlog_handle %s)!\n",
3135                       settle_time, dhtlog_handle);
3136           GNUNET_SCHEDULER_add_now (&continue_puts_and_gets, NULL);
3137         }
3138     }
3139   else if (total_connections + failed_connections == expected_connections)
3140     {
3141       GNUNET_SCHEDULER_cancel (die_task);
3142       die_task
3143           = GNUNET_SCHEDULER_add_now (&end_badly,
3144                                       "from topology_callback (too many failed connections)");
3145     }
3146 }
3147
3148 static void
3149 peers_started_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3150     const struct GNUNET_CONFIGURATION_Handle *cfg,
3151     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3152 {
3153   if (emsg != NULL)
3154     {
3155       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3156                   "Failed to start daemon with error: `%s'\n", emsg);
3157       return;
3158     }
3159   GNUNET_assert (id != NULL);
3160
3161 #if VERBOSE > 1
3162   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
3163       (num_peers - peers_left) + 1, num_peers);
3164 #endif
3165
3166   peers_left--;
3167
3168   if (GNUNET_YES == update_meter (peer_start_meter))
3169     {
3170 #if VERBOSE
3171       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3172           "All %d daemons started, now connecting peers!\n",
3173           num_peers);
3174 #endif
3175       GNUNET_SCHEDULER_cancel (die_task);
3176
3177       expected_connections = UINT_MAX;
3178       if ((pg != NULL) && (peers_left == 0))
3179         {
3180           connect_start_time = GNUNET_TIME_absolute_get ();
3181           expected_connections
3182               = GNUNET_TESTING_connect_topology (
3183                                                  pg,
3184                                                  connect_topology,
3185                                                  connect_topology_option,
3186                                                  connect_topology_option_modifier,
3187                                                  connect_timeout,
3188                                                  connect_attempts, NULL, NULL);
3189
3190           peer_connect_meter = create_meter (expected_connections,
3191                                              "Peer connection ", GNUNET_YES);
3192           fprintf (stderr, "Have %d expected connections\n",
3193                    expected_connections);
3194         }
3195
3196       if (expected_connections == GNUNET_SYSERR)
3197         {
3198           die_task
3199               = GNUNET_SCHEDULER_add_now (&end_badly,
3200                                           "from connect topology (bad return)");
3201         }
3202
3203       die_task
3204           = GNUNET_SCHEDULER_add_delayed (
3205                                           GNUNET_TIME_relative_multiply (
3206                                                                          GNUNET_TIME_UNIT_SECONDS,
3207                                                                          DEFAULT_CONNECT_TIMEOUT
3208                                                                              * expected_connections),
3209                                           &end_badly,
3210                                           "from connect topology (timeout)");
3211
3212       ok = 0;
3213     }
3214 }
3215
3216 static void
3217 create_topology()
3218 {
3219   peers_left = num_peers; /* Reset counter */
3220   if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology,
3221                                       blacklist_transports) != GNUNET_SYSERR)
3222     {
3223       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3224                   "Topology set up, now starting peers!\n");
3225       GNUNET_TESTING_daemons_continue_startup (pg);
3226     }
3227   else
3228     {
3229       GNUNET_SCHEDULER_cancel (die_task);
3230       die_task = GNUNET_SCHEDULER_add_now (&end_badly,
3231                                            "from create topology (bad return)");
3232     }
3233   GNUNET_free_non_null(blacklist_transports);
3234   GNUNET_SCHEDULER_cancel (die_task);
3235   die_task
3236       = GNUNET_SCHEDULER_add_delayed (
3237                                       GNUNET_TIME_relative_multiply (
3238                                                                      seconds_per_peer_start,
3239                                                                      num_peers),
3240                                       &end_badly,
3241                                       "from continue startup (timeout)");
3242 }
3243
3244 /**
3245  * Callback indicating that the hostkey was created for a peer.
3246  *
3247  * @param cls NULL
3248  * @param id the peer identity
3249  * @param d the daemon handle (pretty useless at this point, remove?)
3250  * @param emsg non-null on failure
3251  */
3252 static void
3253 hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3254     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3255 {
3256   if (emsg != NULL)
3257     {
3258       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3259                   "Hostkey callback received error: %s\n", emsg);
3260     }
3261
3262 #if VERBOSE > 1
3263   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3264       "Hostkey (%d/%d) created for peer `%s'\n",
3265       num_peers - peers_left, num_peers, GNUNET_i2s(id));
3266 #endif
3267
3268   peers_left--;
3269   if (GNUNET_YES == update_meter (hostkey_meter))
3270     {
3271       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3272                   "All %d hostkeys created, now creating topology!\n",
3273                   num_peers);
3274       GNUNET_SCHEDULER_cancel (die_task);
3275       /* Set up task in case topology creation doesn't finish
3276        * within a reasonable amount of time */
3277       die_task = GNUNET_SCHEDULER_add_delayed (DEFAULT_TOPOLOGY_TIMEOUT,
3278                                                &end_badly,
3279                                                "from create_topology");
3280       GNUNET_SCHEDULER_add_now (&create_topology, NULL);
3281       ok = 0;
3282     }
3283 }
3284
3285 static void
3286 run(void *cls, char * const *args, const char *cfgfile,
3287     const struct GNUNET_CONFIGURATION_Handle *cfg)
3288 {
3289   struct stat frstat;
3290   struct GNUNET_DHTLOG_TrialInfo trial_info;
3291   struct GNUNET_TESTING_Host *hosts;
3292   struct GNUNET_TESTING_Host *temphost;
3293   struct GNUNET_TESTING_Host *tempnext;
3294   char *topology_str;
3295   char *connect_topology_str;
3296   char *blacklist_topology_str;
3297   char *connect_topology_option_str;
3298   char *connect_topology_option_modifier_string;
3299   char *trialmessage;
3300   char *topology_percentage_str;
3301   float topology_percentage;
3302   char *topology_probability_str;
3303   char *hostfile;
3304   float topology_probability;
3305   unsigned long long temp_config_number;
3306   int stop_closest;
3307   int stop_found;
3308   int strict_kademlia;
3309   char *buf;
3310   char *data;
3311   char *churn_data;
3312   char *churn_filename;
3313   int count;
3314   int ret;
3315   int line_number;
3316   int k;
3317
3318   config = cfg;
3319   rounds_finished = 0;
3320   memset (&trial_info, 0, sizeof(struct GNUNET_DHTLOG_TrialInfo));
3321   /* Get path from configuration file */
3322   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "paths",
3323                                                            "servicehome",
3324                                                            &test_directory))
3325     {
3326       ok = 404;
3327       return;
3328     }
3329
3330   /* Get number of peers to start from configuration */
3331   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3332                                                               "num_peers",
3333                                                               &num_peers))
3334     {
3335       GNUNET_log (
3336                   GNUNET_ERROR_TYPE_WARNING,
3337                   "Number of peers must be specified in section %s option %s\n",
3338                   "TESTING", "NUM_PEERS");
3339     }
3340   GNUNET_assert(num_peers > 0 && num_peers < ULONG_MAX);
3341
3342   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3343                                                           "connect_timeout",
3344                                                           &temp_config_number))
3345     connect_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3346                                                      temp_config_number);
3347   else
3348     {
3349       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3350                   "testing", "connect_timeout");
3351       return;
3352     }
3353
3354   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3355                                                           "connect_attempts",
3356                                                           &connect_attempts))
3357     {
3358       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3359                   "testing", "connect_attempts");
3360       return;
3361     }
3362
3363   if (GNUNET_OK
3364       != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3365                                                 "max_outstanding_connections",
3366                                                 &max_outstanding_connections))
3367     {
3368       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3369                   "testing", "max_outstanding_connections");
3370       return;
3371     }
3372
3373   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3374                                                           "max_concurrent_ssh",
3375                                                           &max_concurrent_ssh))
3376     {
3377       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3378                   "testing", "max_concurrent_ssh");
3379       return;
3380     }
3381
3382   /**
3383    * Get DHT specific testing options.
3384    */
3385   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3386                                                            "mysql_logging"))
3387       || (GNUNET_YES
3388           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3389                                                    "mysql_logging_extended"))
3390       || (GNUNET_YES
3391           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3392                                                    "mysql_logging_minimal")))
3393     {
3394       if (GNUNET_YES
3395           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3396                                                    "mysql_logging_minimal"))
3397         dhtlog_minimal = GNUNET_YES;
3398
3399       dhtlog_handle = GNUNET_DHTLOG_connect (cfg);
3400       if (dhtlog_handle == NULL)
3401         {
3402           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3403                       "Could not connect to mysql server for logging, will NOT log dht operations!");
3404           ok = 3306;
3405           return;
3406         }
3407     }
3408
3409   stop_closest = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3410                                                        "stop_on_closest");
3411   if (stop_closest == GNUNET_SYSERR)
3412     stop_closest = GNUNET_NO;
3413
3414   stop_found = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht", "stop_found");
3415   if (stop_found == GNUNET_SYSERR)
3416     stop_found = GNUNET_NO;
3417
3418   strict_kademlia = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3419                                                           "strict_kademlia");
3420   if (strict_kademlia == GNUNET_SYSERR)
3421     strict_kademlia = GNUNET_NO;
3422
3423   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3424                                                           "comment",
3425                                                           &trialmessage))
3426     trialmessage = NULL;
3427
3428   churn_data = NULL;
3429   /** Check for a churn file to do churny simulation */
3430   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3431                                                           "churn_file",
3432                                                           &churn_filename))
3433     {
3434       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Reading churn data from %s\n",
3435                   churn_filename);
3436       if (GNUNET_OK != GNUNET_DISK_file_test (churn_filename))
3437         {
3438           GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Error reading churn file!\n");
3439           GNUNET_free_non_null(trialmessage);
3440           GNUNET_free(churn_filename);
3441           return;
3442         }
3443       if ((0 != STAT (churn_filename, &frstat)) || (frstat.st_size == 0))
3444         {
3445           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3446                       "Could not open file specified for churn data, ending test!");
3447           ok = 1119;
3448           GNUNET_free_non_null(trialmessage);
3449           GNUNET_free(churn_filename);
3450           return;
3451         }
3452
3453       churn_data = GNUNET_malloc_large (frstat.st_size);
3454       GNUNET_assert(churn_data != NULL);
3455       if (frstat.st_size != GNUNET_DISK_fn_read (churn_filename, churn_data,
3456                                                  frstat.st_size))
3457         {
3458           GNUNET_log (
3459                       GNUNET_ERROR_TYPE_ERROR,
3460                       "Could not read file %s specified for churn, ending test!",
3461                       churn_filename);
3462           GNUNET_free (churn_filename);
3463           GNUNET_free (churn_data);
3464           GNUNET_free_non_null(trialmessage);
3465           return;
3466         }
3467
3468       GNUNET_free_non_null(churn_filename);
3469
3470       buf = churn_data;
3471       count = 0;
3472       /* Read the first line */
3473       while (count < frstat.st_size)
3474         {
3475           count++;
3476           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3477             {
3478               churn_data[count] = '\0';
3479               if (1 != sscanf (buf, "%u", &churn_rounds))
3480                 {
3481                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3482                               "Failed to read number of rounds from churn file, ending test!\n");
3483                   ret = 4200;
3484                   GNUNET_free_non_null(trialmessage);
3485                   GNUNET_free_non_null(churn_data);
3486                   return;
3487                 }
3488               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3489                           "Read %u rounds from churn file\n", churn_rounds);
3490               buf = &churn_data[count + 1];
3491               churn_array = GNUNET_malloc(sizeof(unsigned int) * churn_rounds);
3492               break; /* Done with this part */
3493             }
3494         }
3495
3496       if (GNUNET_OK
3497           != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3498                                                     "churns_per_round",
3499                                                     &churns_per_round))
3500         {
3501           churns_per_round = (unsigned long long) churn_rounds;
3502         }
3503
3504       line_number = 0;
3505       while ((count < frstat.st_size) && (line_number < churn_rounds))
3506         {
3507           count++;
3508           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3509             {
3510               churn_data[count] = '\0';
3511
3512               ret = sscanf (buf, "%u", &churn_array[line_number]);
3513               if (1 == ret)
3514                 {
3515                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3516                               "Read %u peers in round %u\n",
3517                               churn_array[line_number], line_number);
3518                   line_number++;
3519                 }
3520               else
3521                 {
3522                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3523                               "Error reading line `%s' in hostfile\n", buf);
3524                   buf = &churn_data[count + 1];
3525                   continue;
3526                 }
3527               buf = &churn_data[count + 1];
3528             }
3529           else if (churn_data[count] == '\n') /* Blank line */
3530             buf = &churn_data[count + 1];
3531         }
3532     }
3533   GNUNET_free_non_null(churn_data);
3534
3535   /* Check for a hostfile containing user@host:port triples */
3536   if (GNUNET_OK
3537       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "hostfile",
3538                                                 &hostfile))
3539     hostfile = NULL;
3540
3541   hosts = NULL;
3542   temphost = NULL;
3543   data = NULL;
3544   if (hostfile != NULL)
3545     {
3546       if (GNUNET_OK != GNUNET_DISK_file_test (hostfile))
3547         GNUNET_DISK_fn_write (hostfile, NULL, 0, GNUNET_DISK_PERM_USER_READ
3548             | GNUNET_DISK_PERM_USER_WRITE);
3549       if ((0 != STAT (hostfile, &frstat)) || (frstat.st_size == 0))
3550         {
3551           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3552                       "Could not open file specified for host list, ending test!");
3553           ok = 1119;
3554           GNUNET_free_non_null(trialmessage);
3555           GNUNET_free(hostfile);
3556           return;
3557         }
3558
3559       data = GNUNET_malloc_large (frstat.st_size);
3560       GNUNET_assert(data != NULL);
3561       if (frstat.st_size
3562           != GNUNET_DISK_fn_read (hostfile, data, frstat.st_size))
3563         {
3564           GNUNET_log (
3565                       GNUNET_ERROR_TYPE_ERROR,
3566                       "Could not read file %s specified for host list, ending test!",
3567                       hostfile);
3568           GNUNET_free (hostfile);
3569           GNUNET_free (data);
3570           GNUNET_free_non_null(trialmessage);
3571           return;
3572         }
3573
3574       GNUNET_free_non_null(hostfile);
3575
3576       buf = data;
3577       count = 0;
3578       while (count < frstat.st_size - 1)
3579         {
3580           count++;
3581           if (((data[count] == '\n')) && (buf != &data[count]))
3582             {
3583               data[count] = '\0';
3584               temphost = GNUNET_malloc(sizeof(struct GNUNET_TESTING_Host));
3585               ret = sscanf (buf, "%a[a-zA-Z0-9_]@%a[a-zA-Z0-9.]:%hd",
3586                             &temphost->username, &temphost->hostname,
3587                             &temphost->port);
3588               if (3 == ret)
3589                 {
3590                   GNUNET_log (
3591                               GNUNET_ERROR_TYPE_DEBUG,
3592                               "Successfully read host %s, port %d and user %s from file\n",
3593                               temphost->hostname, temphost->port,
3594                               temphost->username);
3595                 }
3596               else
3597                 {
3598                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3599                               "Error reading line `%s' in hostfile\n", buf);
3600                   GNUNET_free(temphost);
3601                   buf = &data[count + 1];
3602                   continue;
3603                 }
3604               temphost->next = hosts;
3605               hosts = temphost;
3606               buf = &data[count + 1];
3607             }
3608           else if ((data[count] == '\n') || (data[count] == '\0'))
3609             buf = &data[count + 1];
3610         }
3611     }
3612   GNUNET_free_non_null(data);
3613   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3614                                                           "malicious_getters",
3615                                                           &malicious_getters))
3616     malicious_getters = 0;
3617
3618   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3619                                                           "malicious_putters",
3620                                                           &malicious_putters))
3621     malicious_putters = 0;
3622
3623   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3624                                                           "malicious_droppers",
3625                                                           &malicious_droppers))
3626     malicious_droppers = 0;
3627
3628   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3629                                                           "settle_time",
3630                                                           &settle_time))
3631     settle_time = 0;
3632
3633   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3634                                                               "dht_testing",
3635                                                               "num_puts",
3636                                                               &num_puts))
3637     num_puts = num_peers;
3638
3639   if (GNUNET_SYSERR
3640       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3641                                                 "put_replication",
3642                                                 &put_replication))
3643     put_replication = DEFAULT_PUT_REPLICATION;
3644
3645   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3646                                                               "dht_testing",
3647                                                               "num_gets",
3648                                                               &num_gets))
3649     num_gets = num_peers;
3650
3651   if (GNUNET_SYSERR
3652       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3653                                                 "get_replication",
3654                                                 &get_replication))
3655     get_replication = DEFAULT_GET_REPLICATION;
3656
3657   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3658                                                           "find_peer_delay",
3659                                                           &temp_config_number))
3660     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3661                                                      temp_config_number);
3662   else
3663     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3664
3665   if (GNUNET_OK
3666       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3667                                                 "concurrent_find_peers",
3668                                                 &temp_config_number))
3669     max_outstanding_find_peers = temp_config_number;
3670   else
3671     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3672
3673   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3674                                                           "get_timeout",
3675                                                           &temp_config_number))
3676     get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3677                                                  temp_config_number);
3678   else
3679     get_timeout = DEFAULT_GET_TIMEOUT;
3680
3681   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3682                                                           "concurrent_puts",
3683                                                           &temp_config_number))
3684     max_outstanding_puts = temp_config_number;
3685   else
3686     max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
3687
3688   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3689                                                           "concurrent_gets",
3690                                                           &temp_config_number))
3691     max_outstanding_gets = temp_config_number;
3692   else
3693     max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
3694
3695   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3696                                                           "timeout",
3697                                                           &temp_config_number))
3698     all_get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3699                                                      temp_config_number);
3700   else
3701     all_get_timeout.rel_value = get_timeout.rel_value * num_gets;
3702
3703   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3704                                                           "get_delay",
3705                                                           &temp_config_number))
3706     get_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3707                                                temp_config_number);
3708   else
3709     get_delay = DEFAULT_GET_DELAY;
3710
3711   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3712                                                           "put_delay",
3713                                                           &temp_config_number))
3714     put_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3715                                                temp_config_number);
3716   else
3717     put_delay = DEFAULT_PUT_DELAY;
3718
3719   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3720                                                           "peer_start_timeout",
3721                                                           &temp_config_number))
3722     seconds_per_peer_start
3723         = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3724                                          temp_config_number);
3725   else
3726     seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START;
3727
3728   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3729                                                           "data_size",
3730                                                           &temp_config_number))
3731     test_data_size = temp_config_number;
3732   else
3733     test_data_size = DEFAULT_TEST_DATA_SIZE;
3734
3735   /**
3736    * Get testing related options.
3737    */
3738   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3739                                                           "REPLICATE_SAME"))
3740     replicate_same = GNUNET_YES;
3741
3742   /**
3743    * Get testing related options.
3744    */
3745   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3746                                                           "GET_FROM_SAME"))
3747     get_from_same = GNUNET_YES;
3748
3749   if (GNUNET_NO
3750       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3751                                               "MALICIOUS_GET_FREQUENCY",
3752                                               &malicious_get_frequency))
3753     malicious_get_frequency = DEFAULT_MALICIOUS_GET_FREQUENCY;
3754
3755   if (GNUNET_NO
3756       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3757                                               "MALICIOUS_PUT_FREQUENCY",
3758                                               &malicious_put_frequency))
3759     malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY;
3760
3761   if (GNUNET_YES
3762       == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3763                                                "MALICIOUS_AFTER_SETTLE"))
3764     malicious_after_settle = GNUNET_YES;
3765
3766   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3767                                                           "MALICIOUS_SYBIL"))
3768     {
3769       /* Set up the malicious target at random for this round */
3770       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
3771                                         &sybil_target);
3772       malicious_sybil = GNUNET_YES;
3773     }
3774
3775   /* Create the bloomfilter for choosing which peers to set malicious */
3776
3777   /* Bloomfilter size must be 2^k for some integer k */
3778   k = 1;
3779   while (1 << k < malicious_droppers)
3780     k++;
3781   if (malicious_droppers > 0)
3782     malicious_bloom = GNUNET_CONTAINER_bloomfilter_init (NULL, 1 << k,
3783                                                          DHT_BLOOM_K);
3784
3785   /* The normal behavior of the DHT is to do find peer requests
3786    * on its own.  Only if this is explicitly turned off should
3787    * the testing driver issue find peer requests (even though
3788    * this is likely the default when testing).
3789    */
3790   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3791                                                          "do_find_peer"))
3792     do_find_peer = GNUNET_YES;
3793
3794   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3795                                                           "republish"))
3796     in_dht_replication = GNUNET_YES;
3797
3798   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3799                                                            "TRIAL_TO_RUN",
3800                                                            &trial_to_run))
3801     trial_to_run = 0;
3802
3803   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3804                                                            "REVISION",
3805                                                            &revision))
3806     revision = 0;
3807
3808   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3809                                                            "FIND_PEER_DELAY",
3810                                                            &temp_config_number))
3811     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3812                                                      temp_config_number);
3813   else
3814     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3815
3816   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3817                                                            "ROUND_DELAY",
3818                                                            &round_delay))
3819     round_delay = 0;
3820
3821   if (GNUNET_NO
3822       == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3823                                                 "OUTSTANDING_FIND_PEERS",
3824                                                 &max_outstanding_find_peers))
3825     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3826
3827   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3828                                                           "strict_kademlia"))
3829     max_outstanding_find_peers = max_outstanding_find_peers * 1;
3830
3831   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
3832                                                   max_outstanding_find_peers);
3833
3834   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3835                                                               "dht_testing",
3836                                                               "num_rounds",
3837                                                               &total_rounds))
3838     total_rounds = 1;
3839
3840   if ((GNUNET_SYSERR
3841       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3842                                                 "target_total_connections",
3843                                                 &target_total_connections))
3844       || (target_total_connections == 0))
3845     target_total_connections = connection_estimate (num_peers,
3846                                                     DEFAULT_BUCKET_SIZE);
3847
3848   topology_str = NULL;
3849   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3850                                                             "topology",
3851                                                             &topology_str))
3852       && (GNUNET_NO == GNUNET_TESTING_topology_get (&topology, topology_str)))
3853     {
3854       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3855                   "Invalid topology `%s' given for section %s option %s\n",
3856                   topology_str, "TESTING", "TOPOLOGY");
3857       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
3858     }
3859
3860   if (GNUNET_OK
3861       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage",
3862                                                 &topology_percentage_str))
3863     topology_percentage = 0.5;
3864   else
3865     {
3866       topology_percentage = atof (topology_percentage_str);
3867       GNUNET_free(topology_percentage_str);
3868     }
3869
3870   if (GNUNET_OK
3871       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability",
3872                                                 &topology_probability_str))
3873     topology_probability = 0.5;
3874   else
3875     {
3876       topology_probability = atof (topology_probability_str);
3877       GNUNET_free(topology_probability_str);
3878     }
3879
3880   if ((GNUNET_YES
3881       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3882                                                 "connect_topology",
3883                                                 &connect_topology_str))
3884       && (GNUNET_NO == GNUNET_TESTING_topology_get (&connect_topology,
3885                                                     connect_topology_str)))
3886     {
3887       GNUNET_log (
3888                   GNUNET_ERROR_TYPE_WARNING,
3889                   "Invalid connect topology `%s' given for section %s option %s\n",
3890                   connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
3891     }
3892   GNUNET_free_non_null(connect_topology_str);
3893
3894   if ((GNUNET_YES
3895       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3896                                                 "connect_topology_option",
3897                                                 &connect_topology_option_str))
3898       && (GNUNET_NO
3899           == GNUNET_TESTING_topology_option_get (&connect_topology_option,
3900                                                  connect_topology_option_str)))
3901     {
3902       GNUNET_log (
3903                   GNUNET_ERROR_TYPE_WARNING,
3904                   "Invalid connect topology option `%s' given for section %s option %s\n",
3905                   connect_topology_option_str, "TESTING",
3906                   "CONNECT_TOPOLOGY_OPTION");
3907       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
3908     }
3909   GNUNET_free_non_null(connect_topology_option_str);
3910
3911   if (GNUNET_YES
3912       == GNUNET_CONFIGURATION_get_value_string (
3913                                                 cfg,
3914                                                 "testing",
3915                                                 "connect_topology_option_modifier",
3916                                                 &connect_topology_option_modifier_string))
3917     {
3918       if (sscanf (connect_topology_option_modifier_string, "%lf",
3919                   &connect_topology_option_modifier) != 1)
3920         {
3921           GNUNET_log (
3922                       GNUNET_ERROR_TYPE_WARNING,
3923                       _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
3924                       connect_topology_option_modifier_string,
3925                       "connect_topology_option_modifier", "TESTING");
3926         }
3927       GNUNET_free (connect_topology_option_modifier_string);
3928     }
3929
3930   if (GNUNET_YES
3931       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3932                                                 "blacklist_transports",
3933                                                 &blacklist_transports))
3934     blacklist_transports = NULL;
3935
3936   if ((GNUNET_YES
3937       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3938                                                 "blacklist_topology",
3939                                                 &blacklist_topology_str))
3940       && (GNUNET_NO == GNUNET_TESTING_topology_get (&blacklist_topology,
3941                                                     blacklist_topology_str)))
3942     {
3943       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3944                   "Invalid topology `%s' given for section %s option %s\n",
3945                   topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
3946     }
3947   GNUNET_free_non_null(topology_str);
3948   GNUNET_free_non_null(blacklist_topology_str);
3949
3950   /* Set peers_left so we know when all peers started */
3951   peers_left = num_peers;
3952
3953   /* Set up a task to end testing if peer start fails */
3954   die_task
3955       = GNUNET_SCHEDULER_add_delayed (
3956                                       GNUNET_TIME_relative_multiply (
3957                                                                      seconds_per_peer_start,
3958                                                                      num_peers),
3959                                       &end_badly,
3960                                       "didn't generate all hostkeys within allowed startup time!");
3961
3962   if (dhtlog_handle == NULL)
3963     GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "dhtlog_handle is NULL!");
3964
3965   trial_info.other_identifier = (unsigned int) trial_to_run;
3966   trial_info.num_nodes = peers_left;
3967   trial_info.topology = topology;
3968   trial_info.blacklist_topology = blacklist_topology;
3969   trial_info.connect_topology = connect_topology;
3970   trial_info.connect_topology_option = connect_topology_option;
3971   trial_info.connect_topology_option_modifier
3972       = connect_topology_option_modifier;
3973   trial_info.topology_percentage = topology_percentage;
3974   trial_info.topology_probability = topology_probability;
3975   trial_info.puts = num_puts;
3976   trial_info.gets = num_gets;
3977   trial_info.concurrent = max_outstanding_gets;
3978   trial_info.settle_time = settle_time;
3979   trial_info.num_rounds = total_rounds;
3980   trial_info.malicious_getters = malicious_getters;
3981   trial_info.malicious_putters = malicious_putters;
3982   trial_info.malicious_droppers = malicious_droppers;
3983   trial_info.malicious_get_frequency = malicious_get_frequency.rel_value;
3984   trial_info.malicious_put_frequency = malicious_put_frequency.rel_value;
3985   trial_info.stop_closest = stop_closest;
3986   trial_info.stop_found = stop_found;
3987   trial_info.strict_kademlia = strict_kademlia;
3988
3989   if (trialmessage != NULL)
3990     trial_info.message = trialmessage;
3991   else
3992     trial_info.message = "";
3993
3994   if (dhtlog_handle != NULL)
3995     dhtlog_handle->insert_trial (&trial_info);
3996
3997   GNUNET_free_non_null(trialmessage);
3998
3999   hostkey_meter = create_meter (peers_left, "Hostkeys created ", GNUNET_YES);
4000   peer_start_meter = create_meter (peers_left, "Peers started ", GNUNET_YES);
4001
4002   put_meter = create_meter (num_puts, "Puts completed ", GNUNET_YES);
4003   get_meter = create_meter (num_gets, "Gets completed ", GNUNET_YES);
4004   pg
4005       = GNUNET_TESTING_daemons_start (
4006                                       cfg,
4007                                       peers_left,
4008                                       max_outstanding_connections,
4009                                       max_concurrent_ssh,
4010                                       GNUNET_TIME_relative_multiply (
4011                                                                      seconds_per_peer_start,
4012                                                                      num_peers),
4013                                       &hostkey_callback, NULL,
4014                                       &peers_started_callback, NULL,
4015                                       &topology_callback, NULL, hosts);
4016   temphost = hosts;
4017   while (temphost != NULL)
4018     {
4019       tempnext = temphost->next;
4020       GNUNET_free (temphost->username);
4021       GNUNET_free (temphost->hostname);
4022       GNUNET_free (temphost);
4023       temphost = tempnext;
4024     }
4025 }
4026
4027 int
4028 main(int argc, char *argv[])
4029 {
4030   int ret;
4031   struct GNUNET_GETOPT_CommandLineOption options[] =
4032     {
4033     GNUNET_GETOPT_OPTION_END };
4034
4035   ret = GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-driver", "nohelp", options,
4036                             &run, &ok);
4037
4038   if (malicious_bloom != NULL)
4039     GNUNET_CONTAINER_bloomfilter_free (malicious_bloom);
4040
4041   if (ret != GNUNET_OK)
4042     {
4043       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4044                   "`gnunet-dht-driver': Failed with error code %d\n", ret);
4045     }
4046
4047   /**
4048    * Need to remove base directory, subdirectories taken care
4049    * of by the testing framework.
4050    */
4051   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
4052     {
4053       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4054                   "Failed to remove testing directory %s\n", test_directory);
4055     }
4056   return ret;
4057 }
4058
4059 /* end of gnunet-dht-driver.c */