src/dht/gnunet-dht-driver.c

   1 /*
   2  This file is part of GNUnet.
   3  (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5  GNUnet is free software; you can redistribute it and/or modify
   6  it under the terms of the GNU General Public License as published
   7  by the Free Software Foundation; either version 3, or (at your
   8  option) any later version.
   9
  10  GNUnet is distributed in the hope that it will be useful, but
  11  WITHOUT ANY WARRANTY; without even the implied warranty of
  12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  General Public License for more details.
  14
  15  You should have received a copy of the GNU General Public License
  16  along with GNUnet; see the file COPYING.  If not, write to the
  17  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18  Boston, MA 02111-1307, USA.
  19  */
  20 /**
  21  * @file dht/gnunet-dht-driver.c
  22  * @brief Driver for setting up a group of gnunet peers and
  23  *        then issuing GETS and PUTS on the DHT.  Coarse results
  24  *        are reported, fine grained results (if requested) are
  25  *        logged to a (mysql) database, or to file.
  26  * @author Nathan Evans (who to blame)
  27  */
  28 #include "platform.h"
  29 #ifndef HAVE_MALICIOUS
  30 #error foo
  31 #endif
  32 #include "gnunet_testing_lib.h"
  33 #include "gnunet_core_service.h"
  34 #include "gnunet_dht_service.h"
  35 #include "dhtlog.h"
  36 #include "dht.h"
  37 #include "gauger.h"
  38
  39 /* Specific DEBUG hack, do not use normally (may leak memory, segfault, or eat children.) */
  40 #define ONLY_TESTING GNUNET_NO
  41
  42 /* DEFINES */
  43 #define VERBOSE GNUNET_NO
  44
  45 /* Timeout for entire driver to run */
  46 #define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  47
  48 /* Timeout for waiting for (individual) replies to get requests */
  49 #define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  50
  51 #define DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  52
  53 /* Timeout for waiting for gets to be sent to the service */
  54 #define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  55
  56 /* Timeout for waiting for puts to be sent to the service */
  57 #define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  58
  59 /* Time to allow a find peer request to take */
  60 #define DEFAULT_FIND_PEER_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40)
  61
  62 /* Time to wait for all peers disconnected due to to churn to actually be removed from system */
  63 #define DEFAULT_PEER_DISCONNECT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  64
  65 #define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45)
  66
  67 #define DEFAULT_TEST_DATA_SIZE 8
  68
  69 #define DEFAULT_BUCKET_SIZE 4
  70
  71 /* If more than this many peers are added, slow down sending */
  72 #define MAX_FIND_PEER_CUTOFF 2000
  73
  74 /* If less than this many peers are added, speed up sending */
  75 #define MIN_FIND_PEER_CUTOFF 500
  76
  77 /* How often (in seconds) to print out connection information */
  78 #define CONN_UPDATE_DURATION 10
  79
  80 #define DEFAULT_MAX_OUTSTANDING_PUTS 10
  81
  82 #define DEFAULT_MAX_OUTSTANDING_FIND_PEERS 64
  83
  84 #define DEFAULT_FIND_PEER_OFFSET GNUNET_TIME_relative_divide (DEFAULT_FIND_PEER_DELAY, DEFAULT_MAX_OUTSTANDING_FIND_PEERS)
  85
  86 #define DEFAULT_MAX_OUTSTANDING_GETS 10
  87
  88 #define DEFAULT_CONNECT_TIMEOUT 60
  89
  90 #define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8)
  91
  92 #define DEFAULT_RECONNECT_ATTEMPTS 8
  93
  94 /*
  95  * Default frequency for sending malicious get messages
  96  */
  97 #define DEFAULT_MALICIOUS_GET_FREQUENCY GNUNET_TIME_UNIT_SECONDS
  98
  99 /*
 100  * Default frequency for sending malicious put messages
 101  */
 102 #define DEFAULT_MALICIOUS_PUT_FREQUENCY GNUNET_TIME_UNIT_SECONDS
 103
 104 /* Structs */
 105
 106 struct MaliciousContext
 107 {
 108   /**
 109    * Handle to DHT service (via the API)
 110    */
 111   struct GNUNET_DHT_Handle *dht_handle;
 112
 113   /**
 114    *  Handle to the peer daemon
 115    */
 116   struct GNUNET_TESTING_Daemon *daemon;
 117
 118   /**
 119    * Task for disconnecting DHT handles
 120    */
 121   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 122
 123   /**
 124    * What type of malicious to set this peer to.
 125    */
 126   int malicious_type;
 127 };
 128
 129 struct TestFindPeer
 130 {
 131   /* This is a linked list */
 132   struct TestFindPeer *next;
 133
 134   /* Handle to the bigger context */
 135   struct FindPeerContext *find_peer_context;
 136
 137   /**
 138    * Handle to the peer's DHT service (via the API)
 139    */
 140   struct GNUNET_DHT_Handle *dht_handle;
 141
 142   /**
 143    *  Handle to the peer daemon
 144    */
 145   struct GNUNET_TESTING_Daemon *daemon;
 146
 147   /**
 148    * Task for disconnecting DHT handles
 149    */
 150   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 151 };
 152
 153 struct TestPutContext
 154 {
 155   /* This is a linked list */
 156   struct TestPutContext *next;
 157
 158   /**
 159    * Handle to the first peers DHT service (via the API)
 160    */
 161   struct GNUNET_DHT_Handle *dht_handle;
 162
 163   /**
 164    *  Handle to the PUT peer daemon
 165    */
 166   struct GNUNET_TESTING_Daemon *daemon;
 167
 168   /**
 169    *  Identifier for this PUT
 170    */
 171   uint32_t uid;
 172
 173   /**
 174    * Task for disconnecting DHT handles
 175    */
 176   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 177 };
 178
 179 struct TestGetContext
 180 {
 181   /* This is a linked list */
 182   struct TestGetContext *next;
 183
 184   /**
 185    * Handle to the first peers DHT service (via the API)
 186    */
 187   struct GNUNET_DHT_Handle *dht_handle;
 188
 189   /**
 190    * Handle for the DHT get request
 191    */
 192   struct GNUNET_DHT_GetHandle *get_handle;
 193
 194   /**
 195    *  Handle to the GET peer daemon
 196    */
 197   struct GNUNET_TESTING_Daemon *daemon;
 198
 199   /**
 200    *  Identifier for this GET
 201    */
 202   uint32_t uid;
 203
 204   /**
 205    * Task for disconnecting DHT handles (and stopping GET)
 206    */
 207   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 208
 209   /**
 210    * Whether or not this request has been fulfilled already.
 211    */
 212   int succeeded;
 213 };
 214
 215 /**
 216  * Simple struct to keep track of progress, and print a
 217  * nice little percentage meter for long running tasks.
 218  */
 219 struct ProgressMeter
 220 {
 221   unsigned int total;
 222
 223   unsigned int modnum;
 224
 225   unsigned int dotnum;
 226
 227   unsigned int completed;
 228
 229   int print;
 230
 231   char *startup_string;
 232 };
 233
 234 /**
 235  * Linked list of information for populating statistics
 236  * before ending trial.
 237  */
 238 struct StatisticsIteratorContext
 239 {
 240   const struct GNUNET_PeerIdentity *peer;
 241   unsigned int stat_routes;
 242   unsigned int stat_route_forwards;
 243   unsigned int stat_results;
 244   unsigned int stat_results_to_client;
 245   unsigned int stat_result_forwards;
 246   unsigned int stat_gets;
 247   unsigned int stat_puts;
 248   unsigned int stat_puts_inserted;
 249   unsigned int stat_find_peer;
 250   unsigned int stat_find_peer_start;
 251   unsigned int stat_get_start;
 252   unsigned int stat_put_start;
 253   unsigned int stat_find_peer_reply;
 254   unsigned int stat_get_reply;
 255   unsigned int stat_find_peer_answer;
 256   unsigned int stat_get_response_start;
 257 };
 258
 259 /**
 260  * Context for getting a topology, logging it, and continuing
 261  * on with some next operation.
 262  */
 263 struct TopologyIteratorContext
 264 {
 265   unsigned int total_iterations;
 266   unsigned int current_iteration;
 267   unsigned int total_connections;
 268   unsigned int total_peers;
 269   struct GNUNET_CONTAINER_MultiHashMap *peers_seen;
 270   struct GNUNET_PeerIdentity *peer;
 271   GNUNET_SCHEDULER_Task cont;
 272   void *cls;
 273   struct GNUNET_TIME_Relative timeout;
 274 };
 275
 276 struct PeerCount
 277 {
 278   /** Node in the heap */
 279   struct GNUNET_CONTAINER_HeapNode *heap_node;
 280
 281   /** Peer the count refers to */
 282   struct GNUNET_PeerIdentity peer_id;
 283
 284   /** Count of connections this peer has */
 285   unsigned int count;
 286 };
 287
 288 /**
 289  * Context for sending out find peer requests.
 290  */
 291 struct FindPeerContext
 292 {
 293   /**
 294    * How long to send find peer requests, once the settle time
 295    * is over don't send any more out!
 296    */
 297   struct GNUNET_TIME_Absolute endtime;
 298
 299   /**
 300    * Number of connections in the current topology
 301    * (after this round of find peer requests has ended).
 302    */
 303   unsigned int current_peers;
 304
 305   /**
 306    * Number of connections in the current topology
 307    * (before this round of find peer requests started).
 308    */
 309   unsigned int previous_peers;
 310
 311   /**
 312    * Number of find peer requests we have currently
 313    * outstanding.
 314    */
 315   unsigned int outstanding;
 316
 317   /**
 318    * Number of find peer requests to send in this round.
 319    */
 320   unsigned int total;
 321
 322   /**
 323    * Number of find peer requests sent last time around.
 324    */
 325   unsigned int last_sent;
 326
 327   /**
 328    * Hashmap of peers in the current topology, value
 329    * is a PeerCount, with the number of connections
 330    * this peer has.
 331    */
 332   struct GNUNET_CONTAINER_MultiHashMap *peer_hash;
 333
 334   /**
 335    * Min heap which orders values in the peer_hash for
 336    * easy lookup.
 337    */
 338   struct GNUNET_CONTAINER_Heap *peer_min_heap;
 339
 340   /**
 341    * Callback for counting the peers in the current topology.
 342    */
 343   GNUNET_TESTING_NotifyTopology count_peers_cb;
 344 };
 345
 346 enum DHT_ROUND_TYPES
 347 {
 348   /**
 349    * Next full round (puts + gets).
 350    */
 351   DHT_ROUND_NORMAL,
 352
 353   /**
 354    * Next round of gets.
 355    */
 356   DHT_ROUND_GET,
 357
 358   /**
 359    * Next round of puts.
 360    */
 361   DHT_ROUND_PUT,
 362
 363   /**
 364    * Next round of churn.
 365    */
 366   DHT_ROUND_CHURN
 367 };
 368
 369 /* Globals */
 370
 371 /**
 372  * How long to try to connect two peers.
 373  */
 374 struct GNUNET_TIME_Relative connect_timeout;
 375
 376 /**
 377  * How many times to re-attempt connecting two peers.
 378  */
 379 static unsigned long long connect_attempts;
 380
 381 /**
 382  * Timeout to let all GET requests happen.
 383  */
 384 static struct GNUNET_TIME_Relative all_get_timeout;
 385
 386 /**
 387  * Per get timeout
 388  */
 389 static struct GNUNET_TIME_Relative get_timeout;
 390
 391 /**
 392  * Time to allow for GET requests to be sent to service.
 393  */
 394 static struct GNUNET_TIME_Relative get_delay;
 395
 396 /**
 397  * Time to allow for PUT requests to be sent to service.
 398  */
 399 static struct GNUNET_TIME_Relative put_delay;
 400
 401 /**
 402  * Delay between sending find peer requests (if
 403  * handled by the driver, no effect if sent by service).
 404  */
 405 static struct GNUNET_TIME_Relative find_peer_delay;
 406
 407 /**
 408  * Time between find peer requests
 409  * (find_peer_delay / max_outstanding_find_peer)
 410  */
 411 static struct GNUNET_TIME_Relative find_peer_offset;
 412
 413 /**
 414  * How many seconds to allow each peer to start.
 415  */
 416 static struct GNUNET_TIME_Relative seconds_per_peer_start;
 417
 418 /**
 419  * At what time did we start the connection process.
 420  */
 421 static struct GNUNET_TIME_Absolute connect_start_time;
 422
 423 /**
 424  * What was the last time we updated connection/second information.
 425  */
 426 static struct GNUNET_TIME_Absolute connect_last_time;
 427
 428 /**
 429  * At what time did we start the hostkey creation process.
 430  */
 431 static struct GNUNET_TIME_Absolute hostkey_start_time;
 432
 433 /**
 434  * At what time did we start the peer startup process.
 435  */
 436 static struct GNUNET_TIME_Absolute peer_start_time;
 437
 438 /**
 439  * Boolean value, should the driver issue find peer requests
 440  * (GNUNET_YES) or should it be left to the service (GNUNET_NO)
 441  */
 442 static unsigned int do_find_peer;
 443
 444 /**
 445  * Whether or not to insert gauger data.
 446  */
 447 static unsigned int insert_gauger_data;
 448
 449 #if ONLY_TESTING
 450 /**
 451  * Are we currently trying to connect two peers repeatedly?
 452  */
 453 static unsigned int repeat_connect_mode;
 454
 455 /**
 456  * Task for repeating connects.
 457  */
 458 GNUNET_SCHEDULER_TaskIdentifier repeat_connect_task;
 459
 460 struct GNUNET_TESTING_Daemon *repeat_connect_peer1;
 461 struct GNUNET_TESTING_Daemon *repeat_connect_peer2;
 462 #endif
 463
 464 /**
 465  * Boolean value, should replication be done by the dht
 466  * service (GNUNET_YES) or by the driver (GNUNET_NO)
 467  */
 468 static unsigned int in_dht_replication;
 469
 470 /**
 471  * Size of test data to insert/retrieve during testing.
 472  */
 473 static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE;
 474
 475 /**
 476  * Maximum number of concurrent connections to peers.
 477  */
 478 static unsigned long long max_outstanding_connections;
 479
 480 /**
 481  * Maximum number of concurrent ssh instances to peers.
 482  */
 483 static unsigned long long max_concurrent_ssh;
 484
 485 /**
 486  * Maximum number of concurrent PUT requests.
 487  */
 488 static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
 489
 490 /**
 491  * Maximum number of concurrent GET requests.
 492  */
 493 static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
 494
 495 /**
 496  * Number of nodes issuing malicious GET messages.
 497  */
 498 static unsigned long long malicious_getters;
 499
 500 /**
 501  * Maximum number of concurrent find peer messages being sent.
 502  */
 503 static unsigned long long max_outstanding_find_peers;
 504
 505 /**
 506  * Number of nodes issuing malicious PUT messages.
 507  */
 508 static unsigned long long malicious_putters;
 509
 510 /**
 511  * Time (in seconds) to delay between rounds.
 512  */
 513 static unsigned long long round_delay;
 514
 515 /**
 516  * The identifier for this trial (if we have one)
 517  * for external data collection.
 518  */
 519 static unsigned long long trial_to_run;
 520
 521 /**
 522  * How many malicious droppers to seed in the network.
 523  */
 524 static unsigned long long malicious_droppers;
 525
 526 /**
 527  * Bloom filter to restrict malicious nodes chosen.
 528  */
 529 struct GNUNET_CONTAINER_BloomFilter *malicious_bloom;
 530
 531 /**
 532  * Whether malicious droppers should be chosen based on proximity to a key.
 533  */
 534 static int malicious_sybil;
 535
 536 /**
 537  * Target for the malicious sybil nodes (choose the closest to this key).
 538  */
 539 static GNUNET_HashCode sybil_target;
 540
 541 /**
 542  * How often to send malicious GET messages.
 543  */
 544 static struct GNUNET_TIME_Relative malicious_get_frequency;
 545
 546 /**
 547  * How often to send malicious PUT messages.
 548  */
 549 static struct GNUNET_TIME_Relative malicious_put_frequency;
 550
 551 /**
 552  * How long to send find peer requests.
 553  */
 554 static unsigned long long settle_time;
 555
 556 /**
 557  * Handle to the dhtlog service.
 558  */
 559 static struct GNUNET_DHTLOG_Handle *dhtlog_handle;
 560
 561 /**
 562  * Replication value for GET requests.
 563  */
 564 static unsigned long long get_replication;
 565
 566 /**
 567  * Replication value for PUT requests.
 568  */
 569 static unsigned long long put_replication;
 570
 571 /**
 572  * If GNUNET_YES, insert data at the same peers every time.
 573  * Otherwise, choose a new random peer to insert at each time.
 574  */
 575 static unsigned int replicate_same;
 576
 577 /**
 578  * If GNUNET_YES, issue GET requests at the same peers every time.
 579  * Otherwise, choose a new random peer/data combination to search
 580  * each time.
 581  */
 582 static unsigned int get_from_same;
 583
 584 /**
 585  * Should malicious peers be set after allowing for settle time?
 586  * Default is to set them malicious after initial connection setup.
 587  */
 588 static unsigned int malicious_after_settle;
 589
 590 /**
 591  * Number of rounds for testing (PUTS + GETS)
 592  */
 593 static unsigned long long total_rounds;
 594
 595 /**
 596  * Target number of connections (will stop sending find peer
 597  * messages when this number is exceeded)
 598  */
 599 static unsigned long long target_total_connections;
 600
 601 /**
 602  * Number of rounds already run
 603  */
 604 static unsigned int rounds_finished;
 605
 606 /**
 607  * Number of rounds of churn to read from the file (first line, should be a single number).
 608  */
 609 static unsigned int churn_rounds;
 610
 611 /**
 612  * Current round we are in for churn, tells us how many peers to connect/disconnect.
 613  */
 614 static unsigned int current_churn_round;
 615
 616 /**
 617  * Number of times to churn per round
 618  */
 619 static unsigned long long churns_per_round;
 620
 621 /**
 622  * Array of churn values.
 623  */
 624 static unsigned int *churn_array;
 625
 626 /**
 627  * Hash map of stats contexts.
 628  */
 629 static struct GNUNET_CONTAINER_MultiHashMap *stats_map;
 630
 631 /**
 632  * LL of malicious settings.
 633  */
 634 struct MaliciousContext *all_malicious;
 635
 636 /**
 637  * List of GETS to perform
 638  */
 639 struct TestGetContext *all_gets;
 640
 641 /**
 642  * List of PUTS to perform
 643  */
 644 struct TestPutContext *all_puts;
 645
 646 /**
 647  * Directory to store temporary data in, defined in config file
 648  */
 649 static char *test_directory;
 650
 651 /**
 652  * Variable used to store the number of connections we should wait for.
 653  */
 654 static unsigned int expected_connections;
 655
 656 /**
 657  * Variable used to keep track of how many peers aren't yet started.
 658  */
 659 static unsigned long long peers_left;
 660
 661 /**
 662  * Handle to the set of all peers run for this test.
 663  */
 664 static struct GNUNET_TESTING_PeerGroup *pg;
 665
 666 /**
 667  * Global config handle.
 668  */
 669 static const struct GNUNET_CONFIGURATION_Handle *config;
 670
 671 /**
 672  * Total number of peers to run, set based on config file.
 673  */
 674 static unsigned long long num_peers;
 675
 676 /**
 677  * Total number of items to insert.
 678  */
 679 static unsigned long long num_puts;
 680
 681 /**
 682  * How many puts do we currently have in flight?
 683  */
 684 static unsigned long long outstanding_puts;
 685
 686 /**
 687  * How many puts are done?
 688  */
 689 static unsigned long long puts_completed;
 690
 691 /**
 692  * Total number of items to attempt to get.
 693  */
 694 static unsigned long long num_gets;
 695
 696 /**
 697  * How many puts do we currently have in flight?
 698  */
 699 static unsigned long long outstanding_gets;
 700
 701 /**
 702  * How many gets are done?
 703  */
 704 static unsigned long long gets_completed;
 705
 706 /**
 707  * Total number of items to attempt to get.
 708  */
 709 static unsigned long long cumulative_num_gets;
 710
 711 /**
 712  * How many gets are done?
 713  */
 714 static unsigned long long cumulative_successful_gets;
 715
 716 /**
 717  * How many gets failed?
 718  */
 719 static unsigned long long gets_failed;
 720
 721 /**
 722  * How many malicious control messages do
 723  * we currently have in flight?
 724  */
 725 static unsigned long long outstanding_malicious;
 726
 727 /**
 728  * How many set malicious peers are done?
 729  */
 730 static unsigned int malicious_completed;
 731
 732 /**
 733  * For gauger logging, what specific identifier (svn revision)
 734  * should be used?
 735  */
 736 static unsigned long long revision;
 737
 738 /**
 739  * Global used to count how many connections we have currently
 740  * been notified about (how many times has topology_callback been called
 741  * with success?)
 742  */
 743 static unsigned int total_connections;
 744
 745 /**
 746  * Previous connections, for counting new connections during some duration.
 747  */
 748 static unsigned int previous_connections;
 749
 750 /**
 751  * For counting failed connections during some duration.
 752  */
 753 static unsigned int previous_failed_connections;
 754
 755 /**
 756  * Global used to count how many failed connections we have
 757  * been notified about (how many times has topology_callback
 758  * been called with failure?)
 759  */
 760 static unsigned int failed_connections;
 761
 762 /**
 763  * If GNUNET_YES, only log PUT/GET round data to mysql, otherwise
 764  * log everything (including each dht service logging).
 765  */
 766 static unsigned int dhtlog_minimal;
 767
 768 /* Task handle to use to schedule shutdown if something goes wrong */
 769 GNUNET_SCHEDULER_TaskIdentifier die_task;
 770
 771 static char *blacklist_transports;
 772
 773 static enum GNUNET_TESTING_Topology topology;
 774
 775 static enum GNUNET_TESTING_Topology blacklist_topology =
 776     GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 777
 778 static enum GNUNET_TESTING_Topology connect_topology =
 779     GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 780
 781 static enum GNUNET_TESTING_TopologyOption connect_topology_option =
 782     GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 783
 784 static double connect_topology_option_modifier = 0.0;
 785
 786 static struct ProgressMeter *hostkey_meter;
 787
 788 static struct ProgressMeter *peer_start_meter;
 789
 790 static struct ProgressMeter *peer_connect_meter;
 791
 792 static struct ProgressMeter *put_meter;
 793
 794 static struct ProgressMeter *get_meter;
 795
 796 static GNUNET_HashCode *known_keys;
 797
 798 /* Global return value (0 for success, anything else for failure) */
 799 static int ok;
 800
 801 /**
 802  * Create a meter to keep track of the progress of some task.
 803  *
 804  * @param total the total number of items to complete
 805  * @param start_string a string to prefix the meter with (if printing)
 806  * @param print GNUNET_YES to print the meter, GNUNET_NO to count
 807  *              internally only
 808  *
 809  * @return the progress meter
 810  */
 811 static struct ProgressMeter *
 812 create_meter(unsigned int total, char * start_string, int print)
 813 {
 814   struct ProgressMeter *ret;
 815   ret = GNUNET_malloc(sizeof(struct ProgressMeter));
 816   ret->print = print;
 817   ret->total = total;
 818   ret->modnum = total / 4;
 819   ret->dotnum = (total / 50) + 1;
 820   if (start_string != NULL)
 821     ret->startup_string = GNUNET_strdup(start_string);
 822   else
 823     ret->startup_string = GNUNET_strdup("");
 824
 825   return ret;
 826 }
 827
 828 /**
 829  * Update progress meter (increment by one).
 830  *
 831  * @param meter the meter to update and print info for
 832  *
 833  * @return GNUNET_YES if called the total requested,
 834  *         GNUNET_NO if more items expected
 835  */
 836 static int
 837 update_meter(struct ProgressMeter *meter)
 838 {
 839   if (meter->print == GNUNET_YES)
 840     {
 841       if (meter->completed % meter->modnum == 0)
 842         {
 843           if (meter->completed == 0)
 844             {
 845               fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
 846             }
 847           else
 848             fprintf (stdout, "%d%%", (int) (((float) meter->completed
 849                 / meter->total) * 100));
 850         }
 851       else if (meter->completed % meter->dotnum == 0)
 852         fprintf (stdout, ".");
 853
 854       if (meter->completed + 1 == meter->total)
 855         fprintf (stdout, "%d%%]\n", 100);
 856       fflush (stdout);
 857     }
 858   meter->completed++;
 859
 860   if (meter->completed == meter->total)
 861     return GNUNET_YES;
 862   return GNUNET_NO;
 863 }
 864
 865 /**
 866  * Reset progress meter.
 867  *
 868  * @param meter the meter to reset
 869  *
 870  * @return GNUNET_YES if meter reset,
 871  *         GNUNET_SYSERR on error
 872  */
 873 static int
 874 reset_meter(struct ProgressMeter *meter)
 875 {
 876   if (meter == NULL)
 877     return GNUNET_SYSERR;
 878
 879   meter->completed = 0;
 880   return GNUNET_YES;
 881 }
 882
 883 /**
 884  * Release resources for meter
 885  *
 886  * @param meter the meter to free
 887  */
 888 static void
 889 free_meter(struct ProgressMeter *meter)
 890 {
 891   GNUNET_free_non_null (meter->startup_string);
 892   GNUNET_free (meter);
 893 }
 894
 895 /**
 896  * Check whether peers successfully shut down.
 897  */
 898 static void
 899 shutdown_callback(void *cls, const char *emsg)
 900 {
 901   if (emsg != NULL)
 902     {
 903       if (ok == 0)
 904         ok = 2;
 905     }
 906 }
 907
 908 /**
 909  * Task to release DHT handles for PUT
 910  */
 911 static void
 912 put_disconnect_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 913 {
 914   struct TestPutContext *test_put = cls;
 915   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 916   GNUNET_DHT_disconnect (test_put->dht_handle);
 917   test_put->dht_handle = NULL;
 918   if (replicate_same == GNUNET_NO)
 919     test_put->daemon
 920         = GNUNET_TESTING_daemon_get (
 921                                      pg,
 922                                      GNUNET_CRYPTO_random_u32 (
 923                                                                GNUNET_CRYPTO_QUALITY_WEAK,
 924                                                                num_peers));
 925 }
 926
 927 /**
 928  * Function scheduled to be run on the successful completion of this
 929  * testcase.
 930  */
 931 static void
 932 finish_testing(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 933 {
 934   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ending test normally!\n",
 935               (char *) cls);
 936   GNUNET_assert (pg != NULL);
 937   struct TestPutContext *test_put = all_puts;
 938   struct TestGetContext *test_get = all_gets;
 939   char *temp_get_string;
 940   char *revision_str;
 941
 942   while (test_put != NULL)
 943     {
 944       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 945         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
 946       if (test_put->dht_handle != NULL)
 947         GNUNET_DHT_disconnect (test_put->dht_handle);
 948       test_put = test_put->next;
 949     }
 950
 951   while (test_get != NULL)
 952     {
 953       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 954         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
 955       if (test_get->get_handle != NULL)
 956         GNUNET_DHT_get_stop (test_get->get_handle);
 957       if (test_get->dht_handle != NULL)
 958         GNUNET_DHT_disconnect (test_get->dht_handle);
 959       test_get = test_get->next;
 960     }
 961
 962   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
 963
 964   if (dhtlog_handle != NULL)
 965     {
 966       fprintf (stderr, "Update trial endtime\n");
 967       dhtlog_handle->update_trial (cumulative_successful_gets);
 968       GNUNET_DHTLOG_disconnect (dhtlog_handle);
 969       dhtlog_handle = NULL;
 970     }
 971
 972   if (hostkey_meter != NULL)
 973     free_meter (hostkey_meter);
 974   if (peer_start_meter != NULL)
 975     free_meter (peer_start_meter);
 976   if (peer_connect_meter != NULL)
 977     free_meter (peer_connect_meter);
 978   if (put_meter != NULL)
 979     free_meter (put_meter);
 980   if (get_meter != NULL)
 981     free_meter (get_meter);
 982
 983   GNUNET_asprintf (&temp_get_string,
 984                    "DHT Successful GETs", trial_to_run);
 985   GNUNET_asprintf (&revision_str, "%llu", revision);
 986   if (GNUNET_YES == insert_gauger_data)
 987     GAUGER_ID("DHT_TESTING", temp_get_string, cumulative_successful_gets / (double)cumulative_num_gets, "percent successful", revision_str);
 988   fprintf (
 989            stderr,
 990            "Finished trial, had %llu successful gets out of %llu total, %.2f percent succeeded\n",
 991            cumulative_successful_gets, cumulative_num_gets,
 992            cumulative_successful_gets / (double) cumulative_num_gets);
 993   GNUNET_free(temp_get_string);
 994
 995   ok = 0;
 996 }
 997
 998 /**
 999  * Callback for iterating over all the peer connections of a peer group.
1000  */
1001 static void
1002 log_topology_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1003     const struct GNUNET_PeerIdentity *second, const char *emsg)
1004 {
1005   struct TopologyIteratorContext *topo_ctx = cls;
1006   if ((first != NULL) && (second != NULL))
1007     {
1008       if ((topo_ctx->peers_seen != NULL) && (GNUNET_NO
1009           == GNUNET_CONTAINER_multihashmap_contains (topo_ctx->peers_seen,
1010                                                      &first->hashPubKey)))
1011         {
1012           GNUNET_CONTAINER_multihashmap_put (topo_ctx->peers_seen,
1013                                              &first->hashPubKey, NULL,
1014                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1015           topo_ctx->total_peers++;
1016         }
1017       topo_ctx->total_connections++;
1018       if ((GNUNET_NO == dhtlog_minimal) && (dhtlog_handle != NULL))
1019         dhtlog_handle->insert_extended_topology (first, second);
1020     }
1021   else
1022     {
1023       GNUNET_assert(dhtlog_handle != NULL);
1024       GNUNET_log (
1025                   GNUNET_ERROR_TYPE_WARNING,
1026                   "Topology iteration (%u/%u) finished (%u connections, %u peers)\n",
1027                   topo_ctx->current_iteration, topo_ctx->total_iterations,
1028                   topo_ctx->total_connections, topo_ctx->total_peers);
1029       dhtlog_handle->update_topology (topo_ctx->total_connections);
1030       if (topo_ctx->cont != NULL)
1031         GNUNET_SCHEDULER_add_now (topo_ctx->cont, topo_ctx->cls);
1032       if (topo_ctx->peers_seen != NULL)
1033         GNUNET_CONTAINER_multihashmap_destroy (topo_ctx->peers_seen);
1034       GNUNET_free(topo_ctx);
1035     }
1036 }
1037
1038 /**
1039  * Iterator over hash map entries.
1040  *
1041  * @param cls closure - always NULL
1042  * @param key current key code
1043  * @param value value in the hash map, a stats context
1044  * @return GNUNET_YES if we should continue to
1045  *         iterate,
1046  *         GNUNET_NO if not.
1047  */
1048 static int
1049 stats_iterate(void *cls, const GNUNET_HashCode * key, void *value)
1050 {
1051   struct StatisticsIteratorContext *stats_ctx;
1052   if (value == NULL)
1053     return GNUNET_NO;
1054   stats_ctx = value;
1055   dhtlog_handle->insert_stat (stats_ctx->peer, stats_ctx->stat_routes,
1056                               stats_ctx->stat_route_forwards,
1057                               stats_ctx->stat_results,
1058                               stats_ctx->stat_results_to_client,
1059                               stats_ctx->stat_result_forwards,
1060                               stats_ctx->stat_gets, stats_ctx->stat_puts,
1061                               stats_ctx->stat_puts_inserted,
1062                               stats_ctx->stat_find_peer,
1063                               stats_ctx->stat_find_peer_start,
1064                               stats_ctx->stat_get_start,
1065                               stats_ctx->stat_put_start,
1066                               stats_ctx->stat_find_peer_reply,
1067                               stats_ctx->stat_get_reply,
1068                               stats_ctx->stat_find_peer_answer,
1069                               stats_ctx->stat_get_response_start);
1070   GNUNET_free(stats_ctx);
1071   return GNUNET_YES;
1072 }
1073
1074 static void
1075 stats_finished(void *cls, int result)
1076 {
1077   fprintf (stderr, "Finished getting all peers statistics, iterating!\n");
1078   GNUNET_CONTAINER_multihashmap_iterate (stats_map, &stats_iterate, NULL);
1079   GNUNET_CONTAINER_multihashmap_destroy (stats_map);
1080   GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1081 }
1082
1083 /**
1084  * Callback function to process statistic values.
1085  *
1086  * @param cls closure
1087  * @param peer the peer the statistics belong to
1088  * @param subsystem name of subsystem that created the statistic
1089  * @param name the name of the datum
1090  * @param value the current value
1091  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1092  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1093  */
1094 static int
1095 stats_handle(void *cls, const struct GNUNET_PeerIdentity *peer,
1096     const char *subsystem, const char *name, uint64_t value, int is_persistent)
1097 {
1098   struct StatisticsIteratorContext *stats_ctx;
1099
1100   if (dhtlog_handle != NULL)
1101     dhtlog_handle->add_generic_stat (peer, name, subsystem, value);
1102   if (GNUNET_CONTAINER_multihashmap_contains (stats_map, &peer->hashPubKey))
1103     {
1104       stats_ctx = GNUNET_CONTAINER_multihashmap_get (stats_map,
1105                                                      &peer->hashPubKey);
1106     }
1107   else
1108     {
1109       stats_ctx = GNUNET_malloc(sizeof(struct StatisticsIteratorContext));
1110       stats_ctx->peer = peer;
1111       GNUNET_CONTAINER_multihashmap_put (stats_map, &peer->hashPubKey,
1112                                          stats_ctx,
1113                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1114     }
1115   GNUNET_assert(stats_ctx != NULL);
1116
1117   if (strcmp (name, STAT_ROUTES) == 0)
1118     stats_ctx->stat_routes = value;
1119   else if (strcmp (name, STAT_ROUTE_FORWARDS) == 0)
1120     stats_ctx->stat_route_forwards = value;
1121   else if (strcmp (name, STAT_RESULTS) == 0)
1122     stats_ctx->stat_results = value;
1123   else if (strcmp (name, STAT_RESULTS_TO_CLIENT) == 0)
1124     stats_ctx->stat_results_to_client = value;
1125   else if (strcmp (name, STAT_RESULT_FORWARDS) == 0)
1126     stats_ctx->stat_result_forwards = value;
1127   else if (strcmp (name, STAT_GETS) == 0)
1128     stats_ctx->stat_gets = value;
1129   else if (strcmp (name, STAT_PUTS) == 0)
1130     stats_ctx->stat_puts = value;
1131   else if (strcmp (name, STAT_PUTS_INSERTED) == 0)
1132     stats_ctx->stat_puts_inserted = value;
1133   else if (strcmp (name, STAT_FIND_PEER) == 0)
1134     stats_ctx->stat_find_peer = value;
1135   else if (strcmp (name, STAT_FIND_PEER_START) == 0)
1136     stats_ctx->stat_find_peer_start = value;
1137   else if (strcmp (name, STAT_GET_START) == 0)
1138     stats_ctx->stat_get_start = value;
1139   else if (strcmp (name, STAT_PUT_START) == 0)
1140     stats_ctx->stat_put_start = value;
1141   else if (strcmp (name, STAT_FIND_PEER_REPLY) == 0)
1142     stats_ctx->stat_find_peer_reply = value;
1143   else if (strcmp (name, STAT_GET_REPLY) == 0)
1144     stats_ctx->stat_get_reply = value;
1145   else if (strcmp (name, STAT_FIND_PEER_ANSWER) == 0)
1146     stats_ctx->stat_find_peer_answer = value;
1147   else if (strcmp (name, STAT_GET_RESPONSE_START) == 0)
1148     stats_ctx->stat_get_response_start = value;
1149
1150   return GNUNET_OK;
1151 }
1152
1153 /**
1154  * Connect to statistics service for each peer and get the appropriate
1155  * dht statistics for safe keeping.
1156  */
1157 static void
1158 log_dht_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1159 {
1160   stats_map = GNUNET_CONTAINER_multihashmap_create (num_peers);
1161   fprintf (stderr, "Starting statistics logging\n");
1162   GNUNET_TESTING_get_statistics (pg, &stats_finished, &stats_handle, NULL);
1163 }
1164
1165 /**
1166  * Connect to all peers in the peer group and iterate over their
1167  * connections.
1168  */
1169 static void
1170 capture_current_topology(void *cls,
1171     const struct GNUNET_SCHEDULER_TaskContext * tc)
1172 {
1173   struct TopologyIteratorContext *topo_ctx = cls;
1174   dhtlog_handle->insert_topology (0);
1175   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Called capture_current_topology\n");
1176   GNUNET_TESTING_get_topology (pg, &log_topology_cb, topo_ctx);
1177 }
1178
1179 /**
1180  * Check if the get_handle is being used, if so stop the request.  Either
1181  * way, schedule the end_badly_cont function which actually shuts down the
1182  * test.
1183  */
1184 static void
1185 end_badly(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1186 {
1187   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failing test with error: `%s'!\n",
1188               (char *) cls);
1189
1190   struct TestPutContext *test_put = all_puts;
1191   struct TestGetContext *test_get = all_gets;
1192
1193   while (test_put != NULL)
1194     {
1195       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1196         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
1197       if (test_put->dht_handle != NULL)
1198         GNUNET_DHT_disconnect (test_put->dht_handle);
1199       test_put = test_put->next;
1200     }
1201
1202   while (test_get != NULL)
1203     {
1204       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1205         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
1206       if (test_get->get_handle != NULL)
1207         GNUNET_DHT_get_stop (test_get->get_handle);
1208       if (test_get->dht_handle != NULL)
1209         GNUNET_DHT_disconnect (test_get->dht_handle);
1210       test_get = test_get->next;
1211     }
1212
1213   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
1214
1215   if (dhtlog_handle != NULL)
1216     {
1217       fprintf (stderr, "Update trial endtime\n");
1218       dhtlog_handle->update_trial (gets_completed);
1219       GNUNET_DHTLOG_disconnect (dhtlog_handle);
1220       dhtlog_handle = NULL;
1221     }
1222
1223   if (hostkey_meter != NULL)
1224     free_meter (hostkey_meter);
1225   if (peer_start_meter != NULL)
1226     free_meter (peer_start_meter);
1227   if (peer_connect_meter != NULL)
1228     free_meter (peer_connect_meter);
1229   if (put_meter != NULL)
1230     free_meter (put_meter);
1231   if (get_meter != NULL)
1232     free_meter (get_meter);
1233
1234   ok = 1;
1235 }
1236
1237 /**
1238  * Forward declaration.
1239  */
1240 static void
1241 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1242
1243 /**
1244  * Forward declaration.
1245  */
1246 static void
1247 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1248
1249 /**
1250  * Iterator over hash map entries.
1251  *
1252  * @param cls closure
1253  * @param key current key code
1254  * @param value value in the hash map
1255  * @return GNUNET_YES if we should continue to
1256  *         iterate,
1257  *         GNUNET_NO if not.
1258  */
1259 static int
1260 remove_peer_count(void *cls, const GNUNET_HashCode * key, void *value)
1261 {
1262   struct PeerCount *peer_count = value;
1263   GNUNET_CONTAINER_heap_remove_node (peer_count->heap_node);
1264   GNUNET_free(peer_count);
1265
1266   return GNUNET_YES;
1267 }
1268
1269 /**
1270  * Connect to all peers in the peer group and iterate over their
1271  * connections.
1272  */
1273 static void
1274 count_new_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1275 {
1276   struct FindPeerContext *find_peer_context = cls;
1277   find_peer_context->previous_peers = find_peer_context->current_peers;
1278   find_peer_context->current_peers = 0;
1279   GNUNET_TESTING_get_topology (pg, find_peer_context->count_peers_cb,
1280                                find_peer_context);
1281 }
1282
1283 static void
1284 decrement_find_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1285 {
1286   struct TestFindPeer *test_find_peer = cls;
1287   GNUNET_assert(test_find_peer->find_peer_context->outstanding > 0);
1288   test_find_peer->find_peer_context->outstanding--;
1289   test_find_peer->find_peer_context->total--;
1290   if (0 == test_find_peer->find_peer_context->total)
1291     {
1292       GNUNET_SCHEDULER_add_now (&count_new_peers,
1293                                 test_find_peer->find_peer_context);
1294     }
1295   GNUNET_free(test_find_peer);
1296 }
1297
1298 /**
1299  * A find peer request has been sent to the server, now we will schedule a task
1300  * to wait the appropriate time to allow the request to go out and back.
1301  *
1302  * @param cls closure - a TestFindPeer struct
1303  * @param tc context the task is being called with
1304  */
1305 static void
1306 handle_find_peer_sent(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1307 {
1308   struct TestFindPeer *test_find_peer = cls;
1309
1310   GNUNET_DHT_disconnect (test_find_peer->dht_handle);
1311   GNUNET_SCHEDULER_add_delayed (
1312                                 GNUNET_TIME_relative_divide (find_peer_delay, 2),
1313                                 &decrement_find_peers, test_find_peer);
1314 }
1315
1316 static void
1317 send_find_peer_request(void *cls,
1318     const struct GNUNET_SCHEDULER_TaskContext * tc)
1319 {
1320   struct TestFindPeer *test_find_peer = cls;
1321
1322   if (test_find_peer->find_peer_context->outstanding
1323       > max_outstanding_find_peers)
1324     {
1325       GNUNET_SCHEDULER_add_delayed (find_peer_offset, &send_find_peer_request,
1326                                     test_find_peer);
1327       return;
1328     }
1329
1330   test_find_peer->find_peer_context->outstanding++;
1331   if (GNUNET_TIME_absolute_get_remaining (
1332                                           test_find_peer->find_peer_context->endtime).rel_value
1333       == 0)
1334     {
1335       GNUNET_SCHEDULER_add_now (&decrement_find_peers, test_find_peer);
1336       return;
1337     }
1338
1339   test_find_peer->dht_handle = GNUNET_DHT_connect (test_find_peer->daemon->cfg,
1340                                                    1);
1341   GNUNET_assert(test_find_peer->dht_handle != NULL);
1342   GNUNET_DHT_find_peers (test_find_peer->dht_handle, &handle_find_peer_sent,
1343                          test_find_peer);
1344 }
1345
1346 /**
1347  * Add a connection to the find_peer_context given.  This may
1348  * be complete overkill, but allows us to choose the peers with
1349  * the least connections to initiate find peer requests from.
1350  */
1351 static void
1352 add_new_connection(struct FindPeerContext *find_peer_context,
1353     const struct GNUNET_PeerIdentity *first,
1354     const struct GNUNET_PeerIdentity *second)
1355 {
1356   struct PeerCount *first_count;
1357   struct PeerCount *second_count;
1358
1359   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1360                                               &first->hashPubKey))
1361     {
1362       first_count
1363           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1364                                                &first->hashPubKey);
1365       GNUNET_assert(first_count != NULL);
1366       first_count->count++;
1367       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1368                                          first_count->heap_node,
1369                                          first_count->count);
1370     }
1371   else
1372     {
1373       first_count = GNUNET_malloc(sizeof(struct PeerCount));
1374       first_count->count = 1;
1375       memcpy (&first_count->peer_id, first, sizeof(struct GNUNET_PeerIdentity));
1376       first_count->heap_node
1377           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1378                                           first_count, first_count->count);
1379       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1380                                          &first->hashPubKey, first_count,
1381                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1382     }
1383
1384   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1385                                               &second->hashPubKey))
1386     {
1387       second_count
1388           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1389                                                &second->hashPubKey);
1390       GNUNET_assert(second_count != NULL);
1391       second_count->count++;
1392       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1393                                          second_count->heap_node,
1394                                          second_count->count);
1395     }
1396   else
1397     {
1398       second_count = GNUNET_malloc(sizeof(struct PeerCount));
1399       second_count->count = 1;
1400       memcpy (&second_count->peer_id, second,
1401               sizeof(struct GNUNET_PeerIdentity));
1402       second_count->heap_node
1403           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1404                                           second_count, second_count->count);
1405       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1406                                          &second->hashPubKey, second_count,
1407                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1408     }
1409 }
1410
1411 /**
1412  * Iterate over min heap of connections per peer.  For any
1413  * peer that has 0 connections, attempt to connect them to
1414  * some random peer.
1415  *
1416  * @param cls closure a struct FindPeerContext
1417  * @param node internal node of the heap
1418  * @param element value stored, a struct PeerCount
1419  * @param cost cost associated with the node
1420  * @return GNUNET_YES if we should continue to iterate,
1421  *         GNUNET_NO if not.
1422  */
1423 static int
1424 iterate_min_heap_peers(void *cls, struct GNUNET_CONTAINER_HeapNode *node,
1425     void *element, GNUNET_CONTAINER_HeapCostType cost)
1426 {
1427   struct FindPeerContext *find_peer_context = cls;
1428   struct PeerCount *peer_count = element;
1429   struct GNUNET_TESTING_Daemon *d1;
1430   struct GNUNET_TESTING_Daemon *d2;
1431   struct GNUNET_TIME_Relative timeout;
1432   if (cost == 0)
1433     {
1434       d1 = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1435       GNUNET_assert(d1 != NULL);
1436       d2 = d1;
1437       while ((d2 == d1) || (GNUNET_YES != GNUNET_TESTING_daemon_running (d2)))
1438         {
1439           d2
1440               = GNUNET_TESTING_daemon_get (
1441                                            pg,
1442                                            GNUNET_CRYPTO_random_u32 (
1443                                                                      GNUNET_CRYPTO_QUALITY_WEAK,
1444                                                                      num_peers));
1445           GNUNET_assert(d2 != NULL);
1446         }
1447
1448       /** Just try to connect the peers, don't worry about callbacks, etc. **/
1449       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1450                   "Peer %s has 0 connections.  Trying to connect to %s...\n",
1451                   GNUNET_i2s (&peer_count->peer_id), d2->shortname);
1452       timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
1453                                                DEFAULT_CONNECT_TIMEOUT);
1454       if (GNUNET_TIME_relative_to_absolute (timeout).abs_value
1455           > find_peer_context->endtime.abs_value)
1456         {
1457           timeout
1458               = GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime);
1459         }
1460       GNUNET_TESTING_daemons_connect (d1, d2, timeout,
1461                                       DEFAULT_RECONNECT_ATTEMPTS, GNUNET_YES,
1462                                       NULL, NULL);
1463     }
1464   if (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1465       > 0)
1466     return GNUNET_YES;
1467   else
1468     return GNUNET_NO;
1469 }
1470
1471 /**
1472  * Forward declaration.
1473  */
1474 static void
1475 schedule_churn_find_peer_requests(void *cls,
1476     const struct GNUNET_SCHEDULER_TaskContext * tc);
1477
1478 /**
1479  * Callback for iterating over all the peer connections of a peer group.
1480  * Used after we have churned on some peers to find which ones have zero
1481  * connections so we can make them issue find peer requests.
1482  */
1483 static void
1484 count_peers_churn_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1485     const struct GNUNET_PeerIdentity *second, const char *emsg)
1486 {
1487   struct FindPeerContext *find_peer_context = cls;
1488   struct TopologyIteratorContext *topo_ctx;
1489   struct PeerCount *peer_count;
1490
1491   if ((first != NULL) && (second != NULL))
1492     {
1493       add_new_connection (find_peer_context, first, second);
1494       find_peer_context->current_peers++;
1495     }
1496   else
1497     {
1498       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1499                   "Peer count finished (%u connections)\n",
1500                   find_peer_context->current_peers);
1501       peer_count
1502           = GNUNET_CONTAINER_heap_peek (find_peer_context->peer_min_heap);
1503       GNUNET_assert(peer_count != NULL);
1504       /* WAIT. When peers are churned they will come back with their peers (at least in peerinfo), because the HOSTS file doesn't likely get removed. CRAP. */
1505       /* NO they won't, because we have disabled peerinfo writing to disk (remember?) so we WILL have to give them new connections */
1506       /* Best course of action: have DHT automatically try to add peers from peerinfo on startup. This way IF peerinfo writes to file
1507        * then some peers will end up connected.
1508        *
1509        * Also, find any peers that have zero connections here and set up a task to choose at random another peer in the network to
1510        * connect to.  Of course, if they are blacklisted from that peer they won't be able to connect, so we will have to keep trying
1511        * until they get a peer.
1512        */
1513       /* However, they won't automatically be connected to any of their previous peers... How can we handle that? */
1514       /* So now we have choices: do we want them to come back with all their connections?  Probably not, but it solves this mess. */
1515
1516       /* Second problem, which is still a problem, is that a FIND_PEER request won't work when a peer has no connections */
1517
1518       /**
1519        * Okay, so here's how this *should* work now.
1520        *
1521        * 1. We check the min heap for any peers that have 0 connections.
1522        *    a. If any are found, we iterate over the heap and just randomly
1523        *       choose another peer and ask testing to please connect the two.
1524        *       This takes care of the case that a peer just randomly joins the
1525        *       network.  However, if there are strict topology restrictions
1526        *       (imagine a ring) choosing randomly most likely won't help.
1527        *       We make sure the connection attempt doesn't take longer than
1528        *       the total timeout, but don't care too much about the result.
1529        *    b. After that, we still schedule the find peer requests (concurrently
1530        *       with the connect attempts most likely).  This handles the case
1531        *       that the DHT iterates over peerinfo and just needs to try to send
1532        *       a message to get connected.  This should handle the case that the
1533        *       topology is very strict.
1534        *
1535        * 2. If all peers have > 0 connections, we still send find peer requests
1536        *    as long as possible (until timeout is reached) to help out those
1537        *    peers that were newly churned and need more connections.  This is because
1538        *    once all new peers have established a single connection, they won't be
1539        *    well connected.
1540        *
1541        * 3. Once we reach the timeout, we can do no more.  We must schedule the
1542        *    next iteration of get requests regardless of connections that peers
1543        *    may or may not have.
1544        *
1545        * Caveat: it would be nice to get peers to take data offline with them and
1546        *         come back with it (or not) based on the testing framework.  The
1547        *         same goes for remembering previous connections, but putting either
1548        *         into the general testing churn options seems like overkill because
1549        *         these are very specialized cases.
1550        */
1551       GNUNET_log (
1552                   GNUNET_ERROR_TYPE_WARNING,
1553                   "Out of %u peers, fewest connections is %d\n",
1554                   GNUNET_CONTAINER_heap_get_size (
1555                                                   find_peer_context->peer_min_heap),
1556                   peer_count->count);
1557       if ((peer_count->count == 0)
1558           && (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1559               > 0))
1560         {
1561           GNUNET_log (
1562                       GNUNET_ERROR_TYPE_WARNING,
1563                       "Found peer with no connections, will choose some peer(s) at random to connect to!\n");
1564           GNUNET_CONTAINER_heap_iterate (find_peer_context->peer_min_heap,
1565                                          &iterate_min_heap_peers,
1566                                          find_peer_context);
1567           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1568                                     find_peer_context);
1569         }
1570       else if ((GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1571           > 0) && (find_peer_context->last_sent != 0))
1572         {
1573           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1574                                     find_peer_context);
1575         }
1576       else
1577         {
1578           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
1579                                                  &remove_peer_count,
1580                                                  find_peer_context);
1581           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
1582           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
1583           GNUNET_free(find_peer_context);
1584           GNUNET_log (
1585                       GNUNET_ERROR_TYPE_WARNING,
1586                       "Churn round %u of %llu finished, scheduling next GET round.\n",
1587                       current_churn_round, churn_rounds);
1588           if (dhtlog_handle != NULL)
1589             {
1590               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1591               topo_ctx->cont = &do_get;
1592               topo_ctx->cls = all_gets;
1593               topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1594               topo_ctx->peers_seen
1595                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1596               die_task
1597                   = GNUNET_SCHEDULER_add_delayed (
1598                                                   GNUNET_TIME_relative_add (
1599                                                                             GNUNET_TIME_relative_add (
1600                                                                                                       DEFAULT_GET_TIMEOUT,
1601                                                                                                       all_get_timeout),
1602                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1603                                                   &end_badly,
1604                                                   "from do gets (count_peers_churn_cb)");
1605               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1606             }
1607           else
1608             {
1609               die_task
1610                   = GNUNET_SCHEDULER_add_delayed (
1611                                                   GNUNET_TIME_relative_add (
1612                                                                             GNUNET_TIME_relative_add (
1613                                                                                                       DEFAULT_GET_TIMEOUT,
1614                                                                                                       all_get_timeout),
1615                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1616                                                   &end_badly,
1617                                                   "from do gets (count_peers_churn_cb)");
1618               GNUNET_SCHEDULER_add_now (&do_get, all_gets);
1619             }
1620         }
1621     }
1622 }
1623
1624 /**
1625  * Set up a single find peer request for each peer in the topology.  Do this
1626  * until the settle time is over, limited by the number of outstanding requests
1627  * and the time allowed for each one!
1628  */
1629 static void
1630 schedule_churn_find_peer_requests(void *cls,
1631     const struct GNUNET_SCHEDULER_TaskContext * tc)
1632 {
1633   struct FindPeerContext *find_peer_ctx = cls;
1634   struct TestFindPeer *test_find_peer;
1635   struct PeerCount *peer_count;
1636   uint32_t i;
1637
1638   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
1639     find_peer_ctx->total = 1;
1640   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1641       < MIN_FIND_PEER_CUTOFF)
1642     find_peer_ctx->total = find_peer_ctx->total / 2;
1643   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1644       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
1645     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
1646         / 4);
1647   else
1648     find_peer_ctx->total = find_peer_ctx->last_sent * 4;
1649
1650   if (find_peer_ctx->total > max_outstanding_find_peers)
1651     find_peer_ctx->total = max_outstanding_find_peers;
1652
1653   find_peer_ctx->last_sent = find_peer_ctx->total;
1654   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1655               "Sending %u find peer messages (after churn)\n",
1656               find_peer_ctx->total);
1657
1658   if (find_peer_ctx->total > 0)
1659     find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
1660                                                     find_peer_ctx->total);
1661   else
1662     {
1663       find_peer_ctx->previous_peers = find_peer_ctx->current_peers;
1664       find_peer_ctx->current_peers = 0;
1665       GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_ctx);
1666     }
1667
1668   for (i = 0; i < find_peer_ctx->total; i++)
1669     {
1670       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
1671       /* If we have sent requests, choose peers with a low number of connections to send requests from */
1672       peer_count
1673           = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
1674       GNUNET_assert(peer_count != NULL);
1675       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1676                   "Sending find peer request from peer with %u connections\n",
1677                   peer_count->count);
1678       GNUNET_CONTAINER_multihashmap_remove (find_peer_ctx->peer_hash,
1679                                             &peer_count->peer_id.hashPubKey,
1680                                             peer_count);
1681       test_find_peer->daemon
1682           = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1683       GNUNET_assert(test_find_peer->daemon != NULL);
1684       test_find_peer->find_peer_context = find_peer_ctx;
1685       GNUNET_SCHEDULER_add_delayed (
1686                                     GNUNET_TIME_relative_multiply (
1687                                                                    find_peer_offset,
1688                                                                    i),
1689                                     &send_find_peer_request, test_find_peer);
1690     }
1691
1692   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
1693       == NULL))
1694     {
1695       find_peer_ctx->peer_hash
1696           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1697       find_peer_ctx->peer_min_heap
1698           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1699     }
1700   else
1701     {
1702       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
1703                                              &remove_peer_count, find_peer_ctx);
1704       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
1705       find_peer_ctx->peer_hash
1706           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1707     }
1708
1709   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
1710   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
1711 }
1712
1713 static void
1714 schedule_churn_get_topology(void *cls,
1715     const struct GNUNET_SCHEDULER_TaskContext * tc)
1716 {
1717   struct FindPeerContext *find_peer_context = cls;
1718   GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_context);
1719 }
1720
1721 /**
1722  * Called when churning of the topology has finished.
1723  *
1724  * @param cls closure unused
1725  * @param emsg NULL on success, or a printable error on failure
1726  */
1727 static void
1728 churn_complete(void *cls, const char *emsg)
1729 {
1730   struct FindPeerContext *find_peer_context = cls;
1731   struct PeerCount *peer_count;
1732   unsigned int i;
1733   struct GNUNET_TESTING_Daemon *temp_daemon;
1734   struct TopologyIteratorContext *topo_ctx;
1735   struct GNUNET_TIME_Relative calc_timeout;
1736   int count_added;
1737
1738   if (emsg != NULL)
1739     {
1740       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1741                   "Ending test, churning of peers failed with error `%s'", emsg);
1742       GNUNET_SCHEDULER_add_now (&end_badly, (void *) emsg);
1743       return;
1744     }
1745
1746   /**
1747    * If we switched any peers on, we have to somehow force connect the new peer to
1748    * SOME bootstrap peer in the network.  First schedule a task to find all peers
1749    * with no connections, then choose a random peer for each and connect them.
1750    */
1751   if (find_peer_context != NULL)
1752     {
1753       GNUNET_log (
1754                   GNUNET_ERROR_TYPE_WARNING,
1755                   "We have churned on some peers, so we must schedule find peer requests for them!\n");
1756       count_added = 0;
1757       for (i = 0; i < num_peers; i++)
1758         {
1759           temp_daemon = GNUNET_TESTING_daemon_get (pg, i);
1760           if (GNUNET_YES == GNUNET_TESTING_daemon_running (temp_daemon))
1761             {
1762               peer_count = GNUNET_malloc (sizeof(struct PeerCount));
1763               memcpy (&peer_count->peer_id, &temp_daemon->id,
1764                       sizeof(struct GNUNET_PeerIdentity));
1765               GNUNET_assert(peer_count->count == 0);
1766               peer_count->heap_node
1767                   = GNUNET_CONTAINER_heap_insert (
1768                                                   find_peer_context->peer_min_heap,
1769                                                   peer_count, peer_count->count);
1770               GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1771                                                  &temp_daemon->id.hashPubKey,
1772                                                  peer_count,
1773                                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1774               count_added++;
1775             }
1776         }
1777       GNUNET_log (
1778                   GNUNET_ERROR_TYPE_WARNING,
1779                   "Added %d peers to heap, total size %d\n",
1780                   count_added,
1781                   GNUNET_CONTAINER_heap_get_size (
1782                                                   find_peer_context->peer_min_heap));
1783       GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1784                                     &schedule_churn_get_topology,
1785                                     find_peer_context);
1786     }
1787   else
1788     {
1789       GNUNET_log (
1790                   GNUNET_ERROR_TYPE_WARNING,
1791                   "Only churned off peers, no find peer requests, scheduling more gets (after allowing time for peers to disconnect properly!)...\n");
1792       if (dhtlog_handle != NULL)
1793         {
1794           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1795           topo_ctx->cont = &do_get;
1796           topo_ctx->cls = all_gets;
1797           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1798           topo_ctx->peers_seen
1799               = GNUNET_CONTAINER_multihashmap_create (num_peers);
1800           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1801                                                    all_get_timeout);
1802           calc_timeout
1803               = GNUNET_TIME_relative_add (calc_timeout,
1804                                           DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT);
1805           calc_timeout
1806               = GNUNET_TIME_relative_add (calc_timeout,
1807                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1808           die_task
1809               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1810                                               "from do gets (churn_complete)");
1811           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1812                                         &capture_current_topology, topo_ctx);
1813           dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1814         }
1815       else
1816         {
1817           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1818                                                    all_get_timeout);
1819           calc_timeout
1820               = GNUNET_TIME_relative_add (calc_timeout,
1821                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1822           die_task
1823               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1824                                               "from do gets (churn_complete)");
1825           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1826                                         &do_get, all_gets);
1827         }
1828     }
1829 }
1830
1831 /**
1832  * Decide how many peers to turn on or off in this round, make sure the
1833  * numbers actually make sense, then do so.  This function sets in motion
1834  * churn, find peer requests for newly joined peers, and issuing get
1835  * requests once the new peers have done so.
1836  *
1837  * @param cls closure (unused)
1838  * @param tc task context (unused)
1839  */
1840 static void
1841 churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1842 {
1843   unsigned int count_running;
1844   unsigned int churn_up;
1845   unsigned int churn_down;
1846   struct GNUNET_TIME_Relative timeout;
1847   struct FindPeerContext *find_peer_context;
1848
1849   churn_up = churn_down = 0;
1850   count_running = GNUNET_TESTING_daemons_running (pg);
1851   if (count_running > churn_array[current_churn_round])
1852     churn_down = count_running - churn_array[current_churn_round];
1853   else if (count_running < churn_array[current_churn_round])
1854     churn_up = churn_array[current_churn_round] - count_running;
1855   else
1856     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1857                 "Not churning any peers, topology unchanged.\n");
1858
1859   if (churn_up > num_peers - count_running)
1860     {
1861       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1862                   "Churn file specified %u peers (up); only have %u!",
1863                   churn_array[current_churn_round], num_peers);
1864       churn_up = num_peers - count_running;
1865     }
1866   else if (churn_down > count_running)
1867     {
1868       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1869                   "Churn file specified %u peers (down); only have %u!",
1870                   churn_array[current_churn_round], count_running);
1871       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1872                   "This will leave NO peers running (mistake in churn configuration?)!");
1873       churn_down = count_running;
1874     }
1875   //timeout = GNUNET_TIME_relative_multiply(seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1876   //timeout = GNUNET_TIME_relative_multiply (seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1877   timeout = GNUNET_TIME_relative_multiply (DEFAULT_TIMEOUT, 2); /* FIXME: Lack of intelligent choice here */
1878   find_peer_context = NULL;
1879   if (churn_up > 0) /* Only need to do find peer requests if we turned new peers on */
1880     {
1881       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
1882       find_peer_context->count_peers_cb = &count_peers_churn_cb;
1883       find_peer_context->previous_peers = 0;
1884       find_peer_context->current_peers = 0;
1885       find_peer_context->endtime = GNUNET_TIME_relative_to_absolute (timeout);
1886       find_peer_context->peer_hash
1887           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1888       find_peer_context->peer_min_heap
1889           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1890     }
1891   GNUNET_log (
1892               GNUNET_ERROR_TYPE_WARNING,
1893               "churn_peers: want %u total, %u running, starting %u, stopping %u\n",
1894               churn_array[current_churn_round], count_running, churn_up,
1895               churn_down);
1896   GNUNET_TESTING_daemons_churn (pg, churn_down, churn_up, timeout,
1897                                 &churn_complete, find_peer_context);
1898   current_churn_round++;
1899 }
1900
1901 /**
1902  * Task to release DHT handle associated with GET request.
1903  */
1904 static void
1905 get_stop_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1906 {
1907   struct TestGetContext *test_get = cls;
1908   struct TopologyIteratorContext *topo_ctx;
1909
1910   /* The dht_handle may be null if this get was scheduled from a down peer */
1911   if (test_get->dht_handle != NULL)
1912     {
1913       GNUNET_DHT_disconnect (test_get->dht_handle);
1914       outstanding_gets--; /* GET is really finished */
1915       test_get->dht_handle = NULL;
1916     }
1917
1918   /* Reset the uid (which item to search for) and the daemon (which peer to search from) for later get request iterations */
1919   if (get_from_same == GNUNET_NO)
1920     {
1921       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1922                                                 num_puts);
1923       test_get->daemon
1924           = GNUNET_TESTING_daemon_get (
1925                                        pg,
1926                                        GNUNET_CRYPTO_random_u32 (
1927                                                                  GNUNET_CRYPTO_QUALITY_WEAK,
1928                                                                  num_peers));
1929     }
1930
1931 #if VERBOSE > 1
1932   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
1933 #endif
1934   update_meter (get_meter);
1935   if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0))
1936     {
1937       fprintf (
1938                stderr,
1939                "Canceling die task (get_stop_finished) %llu gets completed, %llu gets failed\n",
1940                gets_completed, gets_failed);
1941       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
1942         dhtlog_handle->insert_round_details (DHT_ROUND_GET, rounds_finished,
1943                                              num_gets, gets_completed);
1944       GNUNET_SCHEDULER_cancel (die_task);
1945       reset_meter (put_meter);
1946       reset_meter (get_meter);
1947       /**
1948        *  Handle all cases:
1949        *    1) Testing is completely finished, call the topology iteration dealy and die
1950        *    2) Testing is not finished, churn the network and do gets again (current_churn_round < churn_rounds)
1951        *    3) Testing is not finished, reschedule all the PUTS *and* GETS again (num_rounds > 1)
1952        */
1953       if (rounds_finished == total_rounds - 1) /* Everything is finished, end testing */
1954         {
1955           if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
1956             {
1957               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1958               topo_ctx->cont = &log_dht_statistics;
1959               topo_ctx->peers_seen
1960                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1961               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1962             }
1963           else
1964             GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1965         }
1966       else if (current_churn_round < churns_per_round * (rounds_finished + 1)) /* Do next round of churn */
1967         {
1968           GNUNET_log (
1969                       GNUNET_ERROR_TYPE_WARNING,
1970                       "Current churn round %u, real round %u, scheduling next round of churn.\n",
1971                       current_churn_round, rounds_finished + 1);
1972           gets_completed = 0;
1973           gets_failed = 0;
1974
1975           if (dhtlog_handle != NULL)
1976             dhtlog_handle->insert_round (DHT_ROUND_CHURN, rounds_finished);
1977
1978           GNUNET_SCHEDULER_add_now (&churn_peers, NULL);
1979         }
1980       else if (rounds_finished < total_rounds - 1) /* Start a new complete round */
1981         {
1982           rounds_finished++;
1983           gets_completed = 0;
1984           gets_failed = 0;
1985           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1986                       "Round %u of %llu finished, scheduling next round.\n",
1987                       rounds_finished, total_rounds);
1988
1989           /** We reset the peer daemon for puts and gets on each disconnect, so all we need to do is start another round! */
1990           if (GNUNET_YES == in_dht_replication) /* Replication done in DHT, don't redo puts! */
1991             {
1992               if (dhtlog_handle != NULL)
1993                 dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1994
1995               die_task
1996                   = GNUNET_SCHEDULER_add_delayed (
1997                                                   GNUNET_TIME_relative_add (
1998                                                                             GNUNET_TIME_relative_add (
1999                                                                                                       GNUNET_TIME_relative_multiply (
2000                                                                                                                                      GNUNET_TIME_UNIT_SECONDS,
2001                                                                                                                                      round_delay),
2002                                                                                                       all_get_timeout),
2003                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2004                                                   &end_badly,
2005                                                   "from do gets (next round)");
2006               GNUNET_SCHEDULER_add_delayed (
2007                                             GNUNET_TIME_relative_multiply (
2008                                                                            GNUNET_TIME_UNIT_SECONDS,
2009                                                                            round_delay),
2010                                             &do_get, all_gets);
2011             }
2012           else
2013             {
2014               if (dhtlog_handle != NULL)
2015                 dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2016               die_task
2017                   = GNUNET_SCHEDULER_add_delayed (
2018                                                   GNUNET_TIME_relative_add (
2019                                                                             GNUNET_TIME_relative_multiply (
2020                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2021                                                                                                            round_delay),
2022                                                                             GNUNET_TIME_relative_multiply (
2023                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2024                                                                                                            num_puts
2025                                                                                                                * 2)),
2026                                                   &end_badly, "from do puts");
2027               GNUNET_SCHEDULER_add_delayed (
2028                                             GNUNET_TIME_relative_multiply (
2029                                                                            GNUNET_TIME_UNIT_SECONDS,
2030                                                                            round_delay),
2031                                             &do_put, all_puts);
2032             }
2033         }
2034     }
2035 }
2036
2037 /**
2038  * Task to release get handle.
2039  */
2040 static void
2041 get_stop_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2042 {
2043   struct TestGetContext *test_get = cls;
2044
2045   if ( (tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT) != 0)
2046     gets_failed++;
2047   else
2048     cumulative_successful_gets++;
2049
2050   GNUNET_assert(test_get->get_handle != NULL);
2051   GNUNET_DHT_get_stop (test_get->get_handle);
2052   test_get->get_handle = NULL;
2053   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2054   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2055 }
2056
2057 /**
2058  * Iterator called if the GET request initiated returns a response.
2059  *
2060  * @param cls closure
2061  * @param exp when will this value expire
2062  * @param key key of the result
2063  * @param get_path NULL-terminated array of pointers
2064  *                 to the peers on reverse GET path (or NULL if not recorded)
2065  * @param put_path NULL-terminated array of pointers
2066  *                 to the peers on the PUT path (or NULL if not recorded)
2067  * @param type type of the result
2068  * @param size number of bytes in data
2069  * @param data pointer to the result data
2070  */
2071 static void
2072 get_result_iterator(void *cls, struct GNUNET_TIME_Absolute exp,
2073     const GNUNET_HashCode * key,
2074     const struct GNUNET_PeerIdentity * const *get_path,
2075     const struct GNUNET_PeerIdentity * const *put_path,
2076     enum GNUNET_BLOCK_Type type, size_t size, const void *data)
2077 {
2078   struct TestGetContext *test_get = cls;
2079
2080   if (test_get->succeeded == GNUNET_YES)
2081     return; /* Get has already been successful, probably ending now */
2082
2083   if (0 != memcmp (&known_keys[test_get->uid], key, sizeof(GNUNET_HashCode))) /* || (0 != memcmp(original_data, data, sizeof(original_data))))*/
2084     {
2085       gets_completed++;
2086       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2087                   "Key or data is not the same as was inserted!\n");
2088     }
2089   else
2090     {
2091       gets_completed++;
2092       test_get->succeeded = GNUNET_YES;
2093     }
2094 #if VERBOSE > 1
2095   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
2096 #endif
2097   GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
2098   GNUNET_SCHEDULER_add_continuation (&get_stop_task, test_get,
2099                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
2100 }
2101
2102 /**
2103  * Set up some data, and call API PUT function
2104  */
2105 static void
2106 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2107 {
2108   struct TestGetContext *test_get = cls;
2109
2110   if (num_gets == 0)
2111     {
2112       GNUNET_SCHEDULER_cancel (die_task);
2113       GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
2114     }
2115
2116   if (test_get == NULL)
2117     return; /* End of the list */
2118
2119   /* Set this here in case we are re-running gets */
2120   test_get->succeeded = GNUNET_NO;
2121
2122   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_get->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2123     {
2124       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2125                   "Peer we should issue get request from is down, skipping.\n");
2126       gets_failed++;
2127       GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2128       GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2129       return;
2130     }
2131
2132   /* Check if more gets are outstanding than should be */
2133   if (outstanding_gets > max_outstanding_gets)
2134     {
2135       GNUNET_SCHEDULER_add_delayed (
2136                                     GNUNET_TIME_relative_multiply (
2137                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2138                                                                    200),
2139                                     &do_get, test_get);
2140       return;
2141     }
2142
2143   /* Connect to the first peer's DHT */
2144   test_get->dht_handle = GNUNET_DHT_connect (test_get->daemon->cfg, 10);
2145   GNUNET_assert(test_get->dht_handle != NULL);
2146   outstanding_gets++;
2147
2148   cumulative_num_gets++;
2149   /* Insert the data at the first peer */
2150   test_get->get_handle = GNUNET_DHT_get_start (test_get->dht_handle, get_delay,
2151                                                GNUNET_BLOCK_TYPE_TEST,
2152                                                &known_keys[test_get->uid],
2153                                                get_replication,
2154                                                GNUNET_DHT_RO_NONE, NULL, 0,
2155                                                NULL, 0, &get_result_iterator,
2156                                                test_get);
2157
2158 #if VERBOSE > 1
2159   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
2160       test_get->uid,
2161       test_get->daemon->shortname);
2162 #endif
2163   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed (get_timeout,
2164                                                             &get_stop_task,
2165                                                             test_get);
2166
2167   /* Schedule the next request in the linked list of get requests */
2168   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2169 }
2170
2171 /**
2172  * Called when the PUT request has been transmitted to the DHT service.
2173  * Schedule the GET request for some time in the future.
2174  */
2175 static void
2176 put_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2177 {
2178   struct TestPutContext *test_put = cls;
2179   struct TopologyIteratorContext *topo_ctx;
2180   outstanding_puts--;
2181   puts_completed++;
2182
2183   if ( (tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT) != 0)
2184     fprintf (stderr, "PUT Request failed!\n");
2185
2186   /* Reset the daemon (which peer to insert at) for later put request iterations */
2187   if (replicate_same == GNUNET_NO)
2188     test_put->daemon
2189         = GNUNET_TESTING_daemon_get (
2190                                      pg,
2191                                      GNUNET_CRYPTO_random_u32 (
2192                                                                GNUNET_CRYPTO_QUALITY_WEAK,
2193                                                                num_peers));
2194
2195   GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
2196   test_put->disconnect_task = GNUNET_SCHEDULER_add_now (&put_disconnect_task,
2197                                                         test_put);
2198   if (GNUNET_YES == update_meter (put_meter))
2199     {
2200       GNUNET_assert(outstanding_puts == 0);
2201       GNUNET_SCHEDULER_cancel (die_task);
2202       if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2203         {
2204           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2205           topo_ctx->cont = &do_get;
2206           topo_ctx->cls = all_gets;
2207           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
2208           topo_ctx->peers_seen
2209               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2210           die_task
2211               = GNUNET_SCHEDULER_add_delayed (
2212                                               GNUNET_TIME_relative_add (
2213                                                                         GNUNET_TIME_relative_add (
2214                                                                                                   DEFAULT_GET_TIMEOUT,
2215                                                                                                   all_get_timeout),
2216                                                                         DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2217                                               &end_badly,
2218                                               "from do gets (put finished)");
2219           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
2220         }
2221       else
2222         {
2223           fprintf (stderr, "Scheduling die task (put finished)\n");
2224           die_task
2225               = GNUNET_SCHEDULER_add_delayed (
2226                                               GNUNET_TIME_relative_add (
2227                                                                         DEFAULT_GET_TIMEOUT,
2228                                                                         all_get_timeout),
2229                                               &end_badly,
2230                                               "from do gets (put finished)");
2231           GNUNET_SCHEDULER_add_delayed (DEFAULT_GET_TIMEOUT, &do_get, all_gets);
2232         }
2233       return;
2234     }
2235 }
2236
2237 /**
2238  * Set up some data, and call API PUT function
2239  */
2240 static void
2241 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2242 {
2243   struct TestPutContext *test_put = cls;
2244   char data[test_data_size]; /* Made up data to store */
2245   uint32_t rand;
2246   int i;
2247
2248   if (test_put == NULL)
2249     return; /* End of list */
2250
2251   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_put->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2252     {
2253       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2254                   "Peer we should issue put request at is down, skipping.\n");
2255       update_meter (put_meter);
2256       GNUNET_SCHEDULER_add_now (&do_put, test_put->next);
2257       return;
2258     }
2259
2260   for (i = 0; i < sizeof(data); i++)
2261     {
2262       memset (&data[i], GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2263                                                   UINT32_MAX), 1);
2264     }
2265
2266   if (outstanding_puts > max_outstanding_puts)
2267     {
2268       GNUNET_SCHEDULER_add_delayed (
2269                                     GNUNET_TIME_relative_multiply (
2270                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2271                                                                    200),
2272                                     &do_put, test_put);
2273       return;
2274     }
2275
2276 #if VERBOSE > 1
2277   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
2278       test_put->uid,
2279       test_put->daemon->shortname);
2280 #endif
2281   test_put->dht_handle = GNUNET_DHT_connect (test_put->daemon->cfg, 10);
2282
2283   GNUNET_assert(test_put->dht_handle != NULL);
2284   outstanding_puts++;
2285   GNUNET_DHT_put (test_put->dht_handle, &known_keys[test_put->uid],
2286                   put_replication, GNUNET_DHT_RO_NONE, GNUNET_BLOCK_TYPE_TEST,
2287                   sizeof(data), data, GNUNET_TIME_UNIT_FOREVER_ABS, put_delay,
2288                   &put_finished, test_put);
2289   test_put->disconnect_task
2290       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever (),
2291                                       &put_disconnect_task, test_put);
2292   rand = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2);
2293   GNUNET_SCHEDULER_add_delayed (
2294                                 GNUNET_TIME_relative_multiply (
2295                                                                GNUNET_TIME_UNIT_SECONDS,
2296                                                                rand), &do_put,
2297                                 test_put->next);
2298 }
2299
2300 static void
2301 schedule_find_peer_requests(void *cls,
2302     const struct GNUNET_SCHEDULER_TaskContext * tc);
2303
2304 static void
2305     setup_malicious_peers(void *cls,
2306         const struct GNUNET_SCHEDULER_TaskContext * tc);
2307
2308 /**
2309  * Given a number of total peers and a bucket size, estimate the number of
2310  * connections in a perfect kademlia topology.
2311  */
2312 static unsigned int
2313 connection_estimate(unsigned int peer_count, unsigned int bucket_size)
2314 {
2315   unsigned int i;
2316   unsigned int filled;
2317   i = num_peers;
2318
2319   filled = 0;
2320   while (i >= bucket_size)
2321     {
2322       filled++;
2323       i = i / 2;
2324     }
2325   filled++; /* Add one filled bucket to account for one "half full" and some miscellaneous */
2326   return filled * bucket_size * peer_count;
2327
2328 }
2329
2330 /**
2331  * Callback for iterating over all the peer connections of a peer group.
2332  */
2333 static void
2334 count_peers_cb(void *cls, const struct GNUNET_PeerIdentity *first,
2335     const struct GNUNET_PeerIdentity *second, const char *emsg)
2336 {
2337   struct FindPeerContext *find_peer_context = cls;
2338   if ((first != NULL) && (second != NULL))
2339     {
2340       add_new_connection (find_peer_context, first, second);
2341       find_peer_context->current_peers++;
2342     }
2343   else
2344     {
2345       GNUNET_log (
2346                   GNUNET_ERROR_TYPE_WARNING,
2347                   "Peer count finished (%u connections), %u new peers, connection estimate %u (target %u)\n",
2348                   find_peer_context->current_peers,
2349                   find_peer_context->current_peers
2350                       - find_peer_context->previous_peers,
2351                   connection_estimate (num_peers, DEFAULT_BUCKET_SIZE),
2352                   target_total_connections);
2353
2354       if ((find_peer_context->last_sent < 8)
2355           || ((find_peer_context->current_peers < 2
2356               * connection_estimate (num_peers, DEFAULT_BUCKET_SIZE))
2357               && (GNUNET_TIME_absolute_get_remaining (
2358                                                       find_peer_context->endtime).rel_value
2359                   > 0) && (find_peer_context->current_peers
2360               < target_total_connections)))
2361         {
2362           GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests,
2363                                     find_peer_context);
2364         }
2365       else
2366         {
2367           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
2368                                                  &remove_peer_count,
2369                                                  find_peer_context);
2370           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
2371           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
2372           GNUNET_free(find_peer_context);
2373           fprintf (stderr, "Not sending any more find peer requests.\n");
2374
2375 #if HAVE_MALICIOUS
2376           if (GNUNET_YES == malicious_after_settle)
2377             {
2378               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "calling setup_malicious_peers\n");
2379               GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2380             }
2381 #endif
2382         }
2383     }
2384 }
2385
2386 /**
2387  * Set up a single find peer request for each peer in the topology.  Do this
2388  * until the settle time is over, limited by the number of outstanding requests
2389  * and the time allowed for each one!
2390  */
2391 static void
2392 schedule_find_peer_requests(void *cls,
2393     const struct GNUNET_SCHEDULER_TaskContext * tc)
2394 {
2395   struct FindPeerContext *find_peer_ctx = cls;
2396   struct TestFindPeer *test_find_peer;
2397   struct PeerCount *peer_count;
2398   uint32_t i;
2399   uint32_t random;
2400
2401   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
2402     find_peer_ctx->total = 1;
2403   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
2404       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
2405     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
2406         / 8);
2407   else
2408     find_peer_ctx->total = find_peer_ctx->last_sent * 2;
2409
2410   if (find_peer_ctx->total > max_outstanding_find_peers)
2411     find_peer_ctx->total = max_outstanding_find_peers;
2412
2413   if (find_peer_ctx->total > num_peers) /* Don't try to send more messages than we have peers! */
2414     find_peer_ctx->total = num_peers;
2415
2416   find_peer_ctx->last_sent = find_peer_ctx->total;
2417   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2418               "Sending %u find peer messages (goal at least %u connections)\n",
2419               find_peer_ctx->total, target_total_connections);
2420
2421   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
2422                                                   find_peer_ctx->total);
2423   for (i = 0; i < find_peer_ctx->total; i++)
2424     {
2425       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
2426       /* If we haven't sent any requests yet, choose random peers */
2427       /* Also choose random in _half_ of all cases, so we don't
2428        * get stuck choosing topologically restricted peers with
2429        * few connections that will never be able to find any new
2430        * peers! */
2431       if ((find_peer_ctx->previous_peers == 0) || (i % 2 == 0))
2432         {
2433           /**
2434            * Attempt to spread find peer requests across even sections of the peer address
2435            * space.  Choose basically 1 peer in every num_peers / max_outstanding_requests
2436            * each time, then offset it by a randomish value.
2437            *
2438            * For instance, if num_peers is 100 and max_outstanding is 10, first chosen peer
2439            * will be between 0 - 10, second between 10 - 20, etc.
2440            */
2441           random = (num_peers / find_peer_ctx->total) * i;
2442           random = random
2443               + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2444                                           (num_peers / find_peer_ctx->total));
2445           if (random >= num_peers)
2446             {
2447               random = random - num_peers;
2448             }
2449 #if REAL_RANDOM
2450           random = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2451 #endif
2452           test_find_peer->daemon = GNUNET_TESTING_daemon_get (pg, random);
2453         }
2454       else /* If we have sent requests, choose peers with a low number of connections to send requests from */
2455         {
2456           peer_count
2457               = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
2458           GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(find_peer_ctx->peer_hash, &peer_count->peer_id.hashPubKey, peer_count));
2459           test_find_peer->daemon
2460               = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
2461           GNUNET_assert(test_find_peer->daemon != NULL);
2462         }
2463
2464       test_find_peer->find_peer_context = find_peer_ctx;
2465       GNUNET_SCHEDULER_add_delayed (
2466                                     GNUNET_TIME_relative_multiply (
2467                                                                    find_peer_offset,
2468                                                                    i),
2469                                     &send_find_peer_request, test_find_peer);
2470     }
2471
2472   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
2473       == NULL))
2474     {
2475       find_peer_ctx->peer_hash
2476           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2477       find_peer_ctx->peer_min_heap
2478           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2479     }
2480   else
2481     {
2482       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
2483                                              &remove_peer_count, find_peer_ctx);
2484       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
2485       find_peer_ctx->peer_hash
2486           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2487     }
2488
2489   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
2490   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
2491
2492 }
2493
2494 /**
2495  * Convert unique ID to hash code.
2496  *
2497  * @param uid unique ID to convert
2498  * @param hash set to uid (extended with zeros)
2499  */
2500 static void
2501 hash_from_uid(uint32_t uid, GNUNET_HashCode *hash)
2502 {
2503   memset (hash, 0, sizeof(GNUNET_HashCode));
2504   *((uint32_t *) hash) = uid;
2505 }
2506
2507 /**
2508  * Set up all of the put and get operations we want to do
2509  * in the current round.  Allocate data structure for each,
2510  * add to list, then schedule the actual PUT operations.
2511  */
2512 static void
2513 setup_puts_and_gets(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2514 {
2515   int i;
2516   struct TestPutContext *test_put;
2517   struct TestGetContext *test_get;
2518   uint32_t temp_peer;
2519   GNUNET_HashCode uid_hash;
2520   int count;
2521 #if REMEMBER
2522   int remember[num_puts][num_peers];
2523   memset(&remember, 0, sizeof(int) * num_puts * num_peers);
2524 #endif
2525   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "in setup_puts_and_gets\n");
2526   known_keys = GNUNET_malloc(sizeof(GNUNET_HashCode) * num_puts);
2527   for (i = 0; i < num_puts; i++)
2528     {
2529       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
2530       test_put->uid = i;
2531       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
2532                                         &known_keys[i]);
2533       /* Set first X bits to match the chosen sybil location if we want to do the sybil attack! */
2534       if (GNUNET_YES == malicious_sybil)
2535         {
2536           memcpy (&known_keys[i], &sybil_target, sizeof(GNUNET_HashCode) / 2);
2537           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2538                       "Distance between sybil location and key is %d\n",
2539                       GNUNET_CRYPTO_hash_matching_bits (&known_keys[i],
2540                                                         &sybil_target));
2541         }
2542       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2543                                             num_peers);
2544       test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2545       /* Don't start PUTs at malicious peers! */
2546       if (malicious_bloom != NULL)
2547         {
2548           count = 0;
2549           hash_from_uid (temp_peer, &uid_hash);
2550           while ((GNUNET_YES
2551               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2552               && (count < num_peers))
2553             {
2554               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2555                                                     num_peers);
2556               hash_from_uid (temp_peer, &uid_hash);
2557               test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2558               count++;
2559             }
2560           if (count == num_peers)
2561             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2562                         "Couldn't find peer not in malicious bloom to select!\n");
2563         }
2564
2565       test_put->next = all_puts;
2566       all_puts = test_put;
2567     }
2568
2569   for (i = 0; i < num_gets; i++)
2570     {
2571       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
2572       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2573                                                 num_puts);
2574 #if REMEMBER
2575       while (remember[test_get->uid][temp_daemon] == 1)
2576       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2577       remember[test_get->uid][temp_daemon] = 1;
2578 #endif
2579       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2580                                             num_peers);
2581       test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2582       /* Don't start GETs at malicious peers! */
2583       if (malicious_bloom != NULL)
2584         {
2585           hash_from_uid (temp_peer, &uid_hash);
2586           count = 0;
2587           while ((GNUNET_YES
2588               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2589               && (count < num_peers))
2590             {
2591               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2592                                                     num_peers);
2593               hash_from_uid (temp_peer, &uid_hash);
2594               test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2595               count++;
2596             }
2597           if (count == num_peers)
2598             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2599                         "Couldn't find peer not in malicious bloom to select!\n");
2600         }
2601       test_get->next = all_gets;
2602       all_gets = test_get;
2603     }
2604
2605   /*GNUNET_SCHEDULER_cancel (die_task);*/
2606   die_task
2607       = GNUNET_SCHEDULER_add_delayed (
2608                                       GNUNET_TIME_relative_multiply (
2609                                                                      GNUNET_TIME_UNIT_SECONDS,
2610                                                                      num_puts
2611                                                                          * 2),
2612                                       &end_badly, "from do puts");
2613   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
2614
2615 }
2616
2617 /**
2618  * Set up some all of the put and get operations we want
2619  * to do.  Allocate data structure for each, add to list,
2620  * then call actual insert functions.
2621  */
2622 static void
2623 continue_puts_and_gets(void *cls,
2624     const struct GNUNET_SCHEDULER_TaskContext * tc)
2625 {
2626   int i;
2627   int max;
2628   struct TopologyIteratorContext *topo_ctx;
2629   struct FindPeerContext *find_peer_context;
2630   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "In continue_puts_and_gets\n");
2631   if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2632     {
2633       if (settle_time >= 180 * 2)
2634         max = (settle_time / 180) - 2;
2635       else
2636         max = 1;
2637       for (i = 1; i < max; i++)
2638         {
2639           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2640           topo_ctx->current_iteration = i;
2641           topo_ctx->total_iterations = max;
2642           topo_ctx->peers_seen
2643               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2644           //fprintf(stderr, "scheduled topology iteration in %d minutes\n", i);
2645           GNUNET_SCHEDULER_add_delayed (
2646                                         GNUNET_TIME_relative_multiply (
2647                                                                        GNUNET_TIME_UNIT_MINUTES,
2648                                                                        i * 3),
2649                                         &capture_current_topology, topo_ctx);
2650         }
2651       topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2652       topo_ctx->cont = &setup_puts_and_gets;
2653       topo_ctx->peers_seen = GNUNET_CONTAINER_multihashmap_create (num_peers);
2654       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2655                   "setting setup_puts_and_gets for %d seconds in the future\n",
2656                   settle_time + 10);
2657       GNUNET_SCHEDULER_add_delayed (
2658                                     GNUNET_TIME_relative_multiply (
2659                                                                    GNUNET_TIME_UNIT_SECONDS,
2660                                                                    (settle_time
2661                                                                        + 10)),
2662                                     &capture_current_topology, topo_ctx);
2663     }
2664   else
2665     GNUNET_SCHEDULER_add_delayed (
2666                                   GNUNET_TIME_relative_multiply (
2667                                                                  GNUNET_TIME_UNIT_SECONDS,
2668                                                                  (settle_time
2669                                                                      + 10)),
2670                                   &setup_puts_and_gets, NULL);
2671
2672   if (dhtlog_handle != NULL)
2673     dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2674
2675 #if HAVE_MALICIOUS
2676   if ((GNUNET_YES != malicious_after_settle) || (settle_time == 0))
2677     {
2678       GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2679     }
2680 #endif
2681
2682   if ((GNUNET_YES == do_find_peer) && (settle_time > 0))
2683     {
2684       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2685                   "Scheduling find peer requests during \"settle\" time.\n");
2686       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
2687       find_peer_context->count_peers_cb = &count_peers_cb;
2688       find_peer_context->endtime
2689           = GNUNET_TIME_relative_to_absolute (
2690                                               GNUNET_TIME_relative_multiply (
2691                                                                              GNUNET_TIME_UNIT_SECONDS,
2692                                                                              settle_time));
2693       GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests, find_peer_context);
2694     }
2695   else
2696     {
2697       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2698                   "Assuming automatic DHT find peer requests.\n");
2699     }
2700 }
2701
2702 /**
2703  * Task to release DHT handles
2704  */
2705 static void
2706 malicious_disconnect_task(void *cls,
2707                           const struct GNUNET_SCHEDULER_TaskContext * tc)
2708 {
2709   struct MaliciousContext *ctx = cls;
2710   outstanding_malicious--;
2711   malicious_completed++;
2712   ctx->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2713   GNUNET_DHT_disconnect (ctx->dht_handle);
2714   ctx->dht_handle = NULL;
2715   GNUNET_free(ctx);
2716
2717   if (malicious_completed == malicious_getters + malicious_putters
2718       + malicious_droppers)
2719     {
2720       fprintf (stderr, "Finished setting all malicious peers up!\n");
2721     }
2722 }
2723
2724 /**
2725  * Task to release DHT handles
2726  */
2727 static void
2728 malicious_done_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2729 {
2730   struct MaliciousContext *ctx = cls;
2731   GNUNET_SCHEDULER_cancel (ctx->disconnect_task);
2732   GNUNET_SCHEDULER_add_now (&malicious_disconnect_task, ctx);
2733 }
2734
2735 /**
2736  * Set up some data, and call API PUT function
2737  */
2738 static void
2739 set_malicious(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2740 {
2741   struct MaliciousContext *ctx = cls;
2742
2743   if (outstanding_malicious > DEFAULT_MAX_OUTSTANDING_GETS)
2744     {
2745       GNUNET_SCHEDULER_add_delayed (
2746                                     GNUNET_TIME_relative_multiply (
2747                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2748                                                                    100),
2749                                     &set_malicious, ctx);
2750       return;
2751     }
2752
2753   if (ctx->dht_handle == NULL)
2754     {
2755       ctx->dht_handle = GNUNET_DHT_connect (ctx->daemon->cfg, 1);
2756       outstanding_malicious++;
2757     }
2758
2759   GNUNET_assert(ctx->dht_handle != NULL);
2760
2761 #if VERBOSE > 1
2762   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Setting peer %s malicious type %d\n",
2763       ctx->daemon->shortname, ctx->malicious_type);
2764 #endif
2765
2766   switch (ctx->malicious_type)
2767     {
2768   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET:
2769     GNUNET_DHT_set_malicious_getter (ctx->dht_handle, malicious_get_frequency, &malicious_done_task, ctx);
2770     break;
2771   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT:
2772     GNUNET_DHT_set_malicious_putter (ctx->dht_handle, malicious_put_frequency, &malicious_done_task, ctx);
2773     break;
2774   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP:
2775     GNUNET_DHT_set_malicious_dropper (ctx->dht_handle, &malicious_done_task, ctx);
2776     break;
2777   default:
2778     break;
2779     }
2780
2781   ctx->disconnect_task
2782       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
2783                                       &malicious_disconnect_task, ctx);
2784 }
2785
2786 #if HAVE_MALICIOUS
2787 /**
2788  * Choose the next peer from the peer group to set as malicious.
2789  * If we are doing a sybil attack, find the nearest peer to the
2790  * sybil location that has not already been set malicious.  Otherwise
2791  * just choose a random not already chosen peer.
2792  *
2793  * @param pg the peer group
2794  * @param bloom the bloomfilter which contains all peer already
2795  *        chosen to be malicious
2796  */
2797 static uint32_t
2798 choose_next_malicious (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_CONTAINER_BloomFilter *bloom)
2799   {
2800     int i;
2801     int nearest;
2802     int bits_match;
2803     int curr_distance;
2804     int count;
2805     struct GNUNET_TESTING_Daemon *temp_daemon;
2806     GNUNET_HashCode uid_hash;
2807
2808     curr_distance = 0;
2809     nearest = 0;
2810     GNUNET_assert (bloom != NULL);
2811
2812     if (GNUNET_YES == malicious_sybil)
2813       {
2814         for (i = 0; i < num_peers; i++)
2815           {
2816             temp_daemon = GNUNET_TESTING_daemon_get(pg, i);
2817             hash_from_uid(i, &uid_hash);
2818             /* Check if this peer matches the bloomfilter */
2819             if ((GNUNET_NO == GNUNET_TESTING_daemon_running(temp_daemon)) || (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)))
2820             continue;
2821
2822             bits_match = GNUNET_CRYPTO_hash_matching_bits (&temp_daemon->id.hashPubKey, &sybil_target);
2823             if (bits_match >= curr_distance)
2824               {
2825                 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found nearer peer %s to %s, old matching bits %d, new %d\n", GNUNET_i2s(&temp_daemon->id), GNUNET_h2s(&sybil_target), curr_distance, bits_match);
2826                 nearest = i;
2827                 curr_distance = bits_match;
2828               }
2829           }
2830       }
2831     else
2832       {
2833         nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2834         hash_from_uid(nearest, &uid_hash);
2835         count = 0;
2836         while ((GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)) && (count < num_peers))
2837           {
2838             GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Peer %d already in bloom (tried %d times)\n", nearest, count);
2839             nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2840             hash_from_uid(nearest, &uid_hash);
2841             count++;
2842           }
2843         if (count == num_peers)
2844         GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Tried %d times to find a peer, selecting %d at random!!\n", count, nearest);
2845       }
2846
2847     return nearest;
2848   }
2849
2850 /**
2851  * Select randomly from set of known peers,
2852  * set the desired number of peers to the
2853  * proper malicious types.
2854  */
2855 static void
2856 setup_malicious_peers (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2857   {
2858     struct MaliciousContext *ctx;
2859     int i;
2860     uint32_t temp_daemon;
2861     GNUNET_HashCode uid_hash;
2862
2863     for (i = 0; i < malicious_getters; i++)
2864       {
2865         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2866         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2867         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2868         hash_from_uid(temp_daemon, &uid_hash);
2869         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2870         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET;
2871         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2872
2873       }
2874
2875     for (i = 0; i < malicious_putters; i++)
2876       {
2877         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2878         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2879         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2880         hash_from_uid(temp_daemon, &uid_hash);
2881         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2882         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT;
2883         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2884
2885       }
2886
2887     for (i = 0; i < malicious_droppers; i++)
2888       {
2889         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2890         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2891         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2892         hash_from_uid(temp_daemon, &uid_hash);
2893         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2894         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP;
2895         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2896       }
2897   }
2898 #endif
2899
2900 #if ONLY_TESTING
2901 /* Forward declaration */
2902 static void
2903 topology_callback (void *cls,
2904     const struct GNUNET_PeerIdentity *first,
2905     const struct GNUNET_PeerIdentity *second,
2906     uint32_t distance,
2907     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2908     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2909     struct GNUNET_TESTING_Daemon *first_daemon,
2910     struct GNUNET_TESTING_Daemon *second_daemon,
2911     const char *emsg);
2912
2913 /**
2914  * Retry connecting two specific peers until they connect,
2915  * at a specific interval.  These two peers previously failed
2916  * to connect, and we hope they continue to so that we can
2917  * debug the reason they are having issues.
2918  */
2919 static void
2920 repeat_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2921   {
2922
2923     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Repeating connect attempt between %s and %s.\n", repeat_connect_peer1->shortname, repeat_connect_peer2->shortname);
2924     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 1 configuration `%s'\n", repeat_connect_peer1->cfgfile);
2925     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 2 configuration `%s'\n", repeat_connect_peer2->cfgfile);
2926
2927     repeat_connect_task = GNUNET_SCHEDULER_NO_TASK;
2928     GNUNET_TESTING_daemons_connect(repeat_connect_peer1,
2929         repeat_connect_peer2,
2930         GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60),
2931         2, &topology_callback, NULL);
2932   }
2933 #endif
2934
2935 /**
2936  * This function is called whenever a connection attempt is finished between two of
2937  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
2938  * number of times this function is called should equal the number returned
2939  * from the GNUNET_TESTING_connect_topology call.
2940  *
2941  * The emsg variable is NULL on success (peers connected), and non-NULL on
2942  * failure (peers failed to connect).
2943  */
2944 static void
2945 topology_callback(void *cls, const struct GNUNET_PeerIdentity *first,
2946                   const struct GNUNET_PeerIdentity *second, uint32_t distance,
2947                   const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2948                   const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2949                   struct GNUNET_TESTING_Daemon *first_daemon,
2950                   struct GNUNET_TESTING_Daemon *second_daemon, const char *emsg)
2951 {
2952   struct TopologyIteratorContext *topo_ctx;
2953   unsigned long long duration;
2954   unsigned long long total_duration;
2955   unsigned int new_connections;
2956   unsigned int new_failed_connections;
2957   long double conns_per_sec_recent;
2958   long double conns_per_sec_total;
2959   long double failed_conns_per_sec_recent;
2960   long double failed_conns_per_sec_total;
2961   char *temp_conn_string;
2962   char *temp_conn_failed_string;
2963   char *revision_str;
2964
2965 #if ONLY_TESTING
2966   if (repeat_connect_mode == GNUNET_YES)
2967     {
2968       if ((first_daemon == repeat_connect_peer1) &&
2969           (second_daemon == repeat_connect_peer2))
2970         {
2971           if (emsg != NULL) /* Peers failed to connect again! */
2972             {
2973               GNUNET_assert(repeat_connect_task == GNUNET_SCHEDULER_NO_TASK);
2974               repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
2975               return;
2976             }
2977           else /* Repeat peers actually connected! */
2978             {
2979               if (repeat_connect_task != GNUNET_SCHEDULER_NO_TASK)
2980               GNUNET_SCHEDULER_cancel(repeat_connect_task);
2981               repeat_connect_peer1 = NULL;
2982               repeat_connect_peer2 = NULL;
2983               repeat_connect_mode = GNUNET_NO;
2984               GNUNET_TESTING_resume_connections(pg);
2985               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Resuming normal connection mode, debug connection was successful!\n");
2986             }
2987         }
2988     }
2989 #endif
2990
2991   if (GNUNET_TIME_absolute_get_difference (connect_last_time,
2992                                            GNUNET_TIME_absolute_get ()).rel_value
2993       > GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
2994                                        CONN_UPDATE_DURATION).rel_value)
2995     {
2996       /* Get number of new connections */
2997       new_connections = total_connections - previous_connections;
2998
2999       /* Get number of new FAILED connections */
3000       new_failed_connections = failed_connections - previous_failed_connections;
3001
3002       /* Get duration in seconds */
3003       duration
3004           = GNUNET_TIME_absolute_get_difference (connect_last_time,
3005                                                  GNUNET_TIME_absolute_get ()).rel_value
3006               / 1000;
3007       total_duration
3008           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3009                                                  GNUNET_TIME_absolute_get ()).rel_value
3010               / 1000;
3011
3012       failed_conns_per_sec_recent = (long double) new_failed_connections / duration;
3013       failed_conns_per_sec_total = (long double) failed_connections / total_duration;
3014       conns_per_sec_recent = (long double) new_connections / duration;
3015       conns_per_sec_total = (long double) total_connections / total_duration;
3016       GNUNET_log (
3017                   GNUNET_ERROR_TYPE_WARNING,
3018                   "Recent: %.2f/s, Total: %.2f/s, Recent failed: %.2f/s, total failed %.2f/s\n",
3019                   conns_per_sec_recent,
3020                   conns_per_sec_total, failed_conns_per_sec_recent,
3021                   failed_conns_per_sec_total);
3022       connect_last_time = GNUNET_TIME_absolute_get ();
3023       previous_connections = total_connections;
3024       previous_failed_connections = failed_connections;
3025       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3026                   "have %u total_connections, %u failed\n", total_connections,
3027                   failed_connections);
3028 #if ONLY_TESTING
3029       /* These conditions likely mean we've entered the death spiral of doom */
3030       if ((total_connections > 20000) &&
3031           (conns_per_sec_recent < 5.0) &&
3032           (conns_per_sec_total > 10.0) &&
3033           (emsg != NULL) &&
3034           (repeat_connect_mode == GNUNET_NO))
3035         {
3036           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Entering repeat connection attempt mode!\n");
3037           repeat_connect_peer1 = first_daemon;
3038           repeat_connect_peer2 = second_daemon;
3039           repeat_connect_mode = GNUNET_YES;
3040           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping NEW connections from being scheduled!\n");
3041           GNUNET_TESTING_stop_connections(pg);
3042           repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
3043         }
3044 #endif
3045     }
3046
3047   if (emsg == NULL)
3048     {
3049       total_connections++;
3050 #if VERBOSE > 1
3051       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
3052           first_daemon->shortname,
3053           second_daemon->shortname,
3054           distance);
3055 #endif
3056     }
3057   else
3058     {
3059       failed_connections++;
3060 #if VERBOSE
3061       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peer %s to peer %s with error :\n%s\n",
3062           first_daemon->shortname,
3063           second_daemon->shortname, emsg);
3064
3065       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
3066           first_daemon->shortname,
3067           second_daemon->shortname, emsg);
3068 #endif
3069     }
3070
3071 #if ONLY_TESTING
3072   if ((repeat_connect_mode == GNUNET_YES) )
3073   return;
3074 #endif
3075
3076   GNUNET_assert(peer_connect_meter != NULL);
3077   if (GNUNET_YES == update_meter (peer_connect_meter))
3078     {
3079 #if VERBOSE
3080       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3081           "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
3082           total_connections);
3083 #endif
3084       if (failed_connections > 0)
3085         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3086                     "While connecting, had %u failed connections.\n",
3087                     failed_connections);
3088       if (dhtlog_handle != NULL)
3089         {
3090           dhtlog_handle->update_connections (total_connections);
3091           dhtlog_handle->insert_topology (expected_connections);
3092         }
3093
3094       total_duration
3095           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3096                                                  GNUNET_TIME_absolute_get ()).rel_value
3097               / 1000;
3098       failed_conns_per_sec_total = (long double) failed_connections / total_duration;
3099       conns_per_sec_total = (long double) total_connections / total_duration;
3100       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3101                   "Overall connection info --- Total: %u, Total Failed %u/s\n",
3102                   total_connections, failed_connections);
3103       GNUNET_log (
3104                   GNUNET_ERROR_TYPE_WARNING,
3105                   "Overall connection info --- Total: %.2f/s, Total Failed %.2f/s\n",
3106                   conns_per_sec_total, failed_conns_per_sec_total);
3107
3108       GNUNET_asprintf (&temp_conn_string,
3109                        "DHT Profiler Connection/s",
3110                        trial_to_run);
3111       GNUNET_asprintf (&temp_conn_failed_string,
3112                        "DHT Profiler Connection/s failed",
3113                        trial_to_run);
3114       GNUNET_asprintf (&revision_str, "%llu", revision);
3115
3116       if (GNUNET_YES == insert_gauger_data)
3117         GAUGER_ID("DHT_TESTING", temp_conn_string, (float)conns_per_sec_total, "conns/s", revision_str);
3118       if (GNUNET_YES == insert_gauger_data)
3119         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, (float)failed_conns_per_sec_total, "failed_conns", revision_str);
3120
3121       GNUNET_free(temp_conn_string);
3122       GNUNET_free(temp_conn_failed_string);
3123       GNUNET_asprintf (&temp_conn_string,
3124                        "DHT Profiler Total Connections",
3125                        trial_to_run);
3126       GNUNET_asprintf (
3127                        &temp_conn_failed_string,
3128                        "DHT Profiler Total Connections failed",
3129                        trial_to_run);
3130       if (GNUNET_YES == insert_gauger_data)
3131         GAUGER_ID("DHT_TESTING", temp_conn_string, total_connections, "conns", revision_str);
3132       if (GNUNET_YES == insert_gauger_data)
3133         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_connections, "failed conns", revision_str);
3134       GNUNET_free(temp_conn_string);
3135       GNUNET_free(temp_conn_failed_string);
3136       GNUNET_free(revision_str);
3137
3138       GNUNET_SCHEDULER_cancel (die_task);
3139
3140       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
3141         {
3142           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
3143           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3144                       "Setting continue gets and puts as topo_cont\n");
3145           topo_ctx->cont = &continue_puts_and_gets;
3146           topo_ctx->peers_seen
3147               = GNUNET_CONTAINER_multihashmap_create (num_peers);
3148           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
3149         }
3150       else
3151         {
3152           GNUNET_log (
3153                       GNUNET_ERROR_TYPE_WARNING,
3154                       "For some reason, NOT scheduling final topology capture (settle_time %d, dhtlog_handle %s)!\n",
3155                       settle_time, dhtlog_handle);
3156           GNUNET_SCHEDULER_add_now (&continue_puts_and_gets, NULL);
3157         }
3158     }
3159   else if (total_connections + failed_connections == expected_connections)
3160     {
3161       GNUNET_SCHEDULER_cancel (die_task);
3162       die_task
3163           = GNUNET_SCHEDULER_add_now (&end_badly,
3164                                       "from topology_callback (too many failed connections)");
3165     }
3166 }
3167
3168 static void
3169 peers_started_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3170     const struct GNUNET_CONFIGURATION_Handle *cfg,
3171     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3172 {
3173   char *revision_str;
3174   if (emsg != NULL)
3175     {
3176       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3177                   "Failed to start daemon with error: `%s'\n", emsg);
3178       return;
3179     }
3180   GNUNET_assert (id != NULL);
3181
3182 #if VERBOSE > 1
3183   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
3184       (num_peers - peers_left) + 1, num_peers);
3185 #endif
3186
3187   peers_left--;
3188
3189   if (GNUNET_YES == update_meter (peer_start_meter))
3190     {
3191 #if VERBOSE
3192       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3193           "All %d daemons started, now connecting peers!\n",
3194           num_peers);
3195 #endif
3196       GNUNET_SCHEDULER_cancel (die_task);
3197
3198       GNUNET_asprintf (&revision_str, "%llu", revision);
3199       if (GNUNET_YES == insert_gauger_data)
3200         GAUGER_ID("DHT_TESTING",
3201                   "peer_startup_time",
3202                   GNUNET_TIME_absolute_get_duration(peer_start_time).rel_value / (double)num_peers,
3203                   "ms/peer", revision_str);
3204       GNUNET_free(revision_str);
3205
3206       expected_connections = UINT_MAX;
3207       if ((pg != NULL) && (peers_left == 0))
3208         {
3209           connect_start_time = GNUNET_TIME_absolute_get ();
3210           expected_connections
3211               = GNUNET_TESTING_connect_topology (
3212                                                  pg,
3213                                                  connect_topology,
3214                                                  connect_topology_option,
3215                                                  connect_topology_option_modifier,
3216                                                  connect_timeout,
3217                                                  connect_attempts, NULL, NULL);
3218
3219           peer_connect_meter = create_meter (expected_connections,
3220                                              "Peer connection ", GNUNET_YES);
3221           fprintf (stderr, "Have %d expected connections\n",
3222                    expected_connections);
3223         }
3224
3225       if (expected_connections == 0)
3226         {
3227           die_task
3228               = GNUNET_SCHEDULER_add_now (&end_badly,
3229                                           "from connect topology (bad return)");
3230         }
3231
3232       die_task
3233           = GNUNET_SCHEDULER_add_delayed (
3234                                           GNUNET_TIME_relative_multiply (
3235                                                                          GNUNET_TIME_UNIT_SECONDS,
3236                                                                          DEFAULT_CONNECT_TIMEOUT
3237                                                                              * expected_connections),
3238                                           &end_badly,
3239                                           "from connect topology (timeout)");
3240
3241       ok = 0;
3242     }
3243 }
3244
3245 static void
3246 create_topology()
3247 {
3248   unsigned int create_expected_connections;
3249   peers_left = num_peers; /* Reset counter */
3250   create_expected_connections = GNUNET_TESTING_create_topology (pg, topology, blacklist_topology,
3251                                                                 blacklist_transports);
3252   if (create_expected_connections > 0)
3253     {
3254       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3255                   "Topology set up, have %u expected connections, now starting peers!\n", create_expected_connections);
3256       GNUNET_TESTING_daemons_continue_startup (pg);
3257       peer_start_time = GNUNET_TIME_absolute_get();
3258     }
3259   else
3260     {
3261       GNUNET_SCHEDULER_cancel (die_task);
3262       die_task = GNUNET_SCHEDULER_add_now (&end_badly,
3263                                            "from create topology (bad return)");
3264     }
3265   GNUNET_free_non_null(blacklist_transports);
3266   GNUNET_SCHEDULER_cancel (die_task);
3267   die_task
3268       = GNUNET_SCHEDULER_add_delayed (
3269                                       GNUNET_TIME_relative_multiply (
3270                                                                      seconds_per_peer_start,
3271                                                                      num_peers),
3272                                       &end_badly,
3273                                       "from continue startup (timeout)");
3274 }
3275
3276 /**
3277  * Callback indicating that the hostkey was created for a peer.
3278  *
3279  * @param cls NULL
3280  * @param id the peer identity
3281  * @param d the daemon handle (pretty useless at this point, remove?)
3282  * @param emsg non-null on failure
3283  */
3284 static void
3285 hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3286     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3287 {
3288   char * revision_str;
3289   if (emsg != NULL)
3290     {
3291       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3292                   "Hostkey callback received error: %s\n", emsg);
3293     }
3294
3295 #if VERBOSE > 1
3296   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3297       "Hostkey (%d/%d) created for peer `%s'\n",
3298       num_peers - peers_left, num_peers, GNUNET_i2s(id));
3299 #endif
3300
3301   peers_left--;
3302   if (GNUNET_YES == update_meter (hostkey_meter))
3303     {
3304       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3305                   "All %d hostkeys created, now creating topology!\n",
3306                   num_peers);
3307
3308       GNUNET_asprintf (&revision_str, "%llu", revision);
3309       if (GNUNET_YES == insert_gauger_data)
3310         {
3311           if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (config, "TESTING",
3312                                                                    "HOSTKEYSFILE"))
3313             {
3314               GAUGER_ID("DHT_TESTING",
3315                         "HOSTKEY_GENERATION",
3316                         GNUNET_TIME_absolute_get_duration(hostkey_start_time).rel_value / (double)num_peers,
3317                         "ms/hostkey", revision_str);
3318             }
3319           else
3320             {
3321               GAUGER_ID("DHT_TESTING",
3322                         "HOSTKEY_GENERATION_REAL",
3323                         GNUNET_TIME_absolute_get_duration(hostkey_start_time).rel_value / (double)num_peers,
3324                         "ms/hostkey", revision_str);
3325             }
3326         }
3327
3328       GNUNET_free(revision_str);
3329
3330       GNUNET_SCHEDULER_cancel (die_task);
3331       /* Set up task in case topology creation doesn't finish
3332        * within a reasonable amount of time */
3333       die_task = GNUNET_SCHEDULER_add_delayed (DEFAULT_TOPOLOGY_TIMEOUT,
3334                                                &end_badly,
3335                                                "from create_topology");
3336       GNUNET_SCHEDULER_add_now (&create_topology, NULL);
3337       ok = 0;
3338     }
3339 }
3340
3341 static void
3342 run(void *cls, char * const *args, const char *cfgfile,
3343     const struct GNUNET_CONFIGURATION_Handle *cfg)
3344 {
3345   struct stat frstat;
3346   struct GNUNET_DHTLOG_TrialInfo trial_info;
3347   struct GNUNET_TESTING_Host *hosts;
3348   struct GNUNET_TESTING_Host *temphost;
3349   struct GNUNET_TESTING_Host *tempnext;
3350   char *topology_str;
3351   char *connect_topology_str;
3352   char *blacklist_topology_str;
3353   char *connect_topology_option_str;
3354   char *connect_topology_option_modifier_string;
3355   char *trialmessage;
3356   char *topology_percentage_str;
3357   float topology_percentage;
3358   char *topology_probability_str;
3359   char *hostfile;
3360   float topology_probability;
3361   unsigned long long temp_config_number;
3362   int stop_closest;
3363   int stop_found;
3364   int strict_kademlia;
3365   char *buf;
3366   char *data;
3367   char *churn_data;
3368   char *churn_filename;
3369   int count;
3370   int ret;
3371   int line_number;
3372   int k;
3373
3374   config = cfg;
3375   rounds_finished = 0;
3376   memset (&trial_info, 0, sizeof(struct GNUNET_DHTLOG_TrialInfo));
3377   /* Get path from configuration file */
3378   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "paths",
3379                                                            "servicehome",
3380                                                            &test_directory))
3381     {
3382       ok = 404;
3383       return;
3384     }
3385
3386   /* Get number of peers to start from configuration */
3387   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3388                                                               "num_peers",
3389                                                               &num_peers))
3390     {
3391       GNUNET_log (
3392                   GNUNET_ERROR_TYPE_WARNING,
3393                   "Number of peers must be specified in section %s option %s\n",
3394                   "TESTING", "NUM_PEERS");
3395     }
3396   GNUNET_assert(num_peers > 0 && num_peers < ULONG_MAX);
3397
3398   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3399                                                           "connect_timeout",
3400                                                           &temp_config_number))
3401     connect_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3402                                                      temp_config_number);
3403   else
3404     {
3405       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3406                   "testing", "connect_timeout");
3407       return;
3408     }
3409
3410   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3411                                                           "connect_attempts",
3412                                                           &connect_attempts))
3413     {
3414       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3415                   "testing", "connect_attempts");
3416       return;
3417     }
3418
3419   if (GNUNET_OK
3420       != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3421                                                 "max_outstanding_connections",
3422                                                 &max_outstanding_connections))
3423     {
3424       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3425                   "testing", "max_outstanding_connections");
3426       return;
3427     }
3428
3429   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3430                                                           "max_concurrent_ssh",
3431                                                           &max_concurrent_ssh))
3432     {
3433       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3434                   "testing", "max_concurrent_ssh");
3435       return;
3436     }
3437
3438   /**
3439    * Get DHT specific testing options.
3440    */
3441   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3442                                                            "mysql_logging"))
3443       || (GNUNET_YES
3444           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3445                                                    "mysql_logging_extended"))
3446       || (GNUNET_YES
3447           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3448                                                    "mysql_logging_minimal")))
3449     {
3450       if (GNUNET_YES
3451           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3452                                                    "mysql_logging_minimal"))
3453         dhtlog_minimal = GNUNET_YES;
3454
3455       dhtlog_handle = GNUNET_DHTLOG_connect (cfg);
3456       if (dhtlog_handle == NULL)
3457         {
3458           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3459                       "Could not connect to mysql server for logging, will NOT log dht operations!");
3460           ok = 3306;
3461           return;
3462         }
3463     }
3464
3465   stop_closest = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3466                                                        "stop_on_closest");
3467   if (stop_closest == GNUNET_SYSERR)
3468     stop_closest = GNUNET_NO;
3469
3470   stop_found = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht", "stop_found");
3471   if (stop_found == GNUNET_SYSERR)
3472     stop_found = GNUNET_NO;
3473
3474   strict_kademlia = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3475                                                           "strict_kademlia");
3476   if (strict_kademlia == GNUNET_SYSERR)
3477     strict_kademlia = GNUNET_NO;
3478
3479   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3480                                                           "comment",
3481                                                           &trialmessage))
3482     trialmessage = NULL;
3483
3484   churn_data = NULL;
3485   /** Check for a churn file to do churny simulation */
3486   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3487                                                           "churn_file",
3488                                                           &churn_filename))
3489     {
3490       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Reading churn data from %s\n",
3491                   churn_filename);
3492       if (GNUNET_OK != GNUNET_DISK_file_test (churn_filename))
3493         {
3494           GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Error reading churn file!\n");
3495           GNUNET_free_non_null(trialmessage);
3496           GNUNET_free(churn_filename);
3497           return;
3498         }
3499       if ((0 != STAT (churn_filename, &frstat)) || (frstat.st_size == 0))
3500         {
3501           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3502                       "Could not open file specified for churn data, ending test!");
3503           ok = 1119;
3504           GNUNET_free_non_null(trialmessage);
3505           GNUNET_free(churn_filename);
3506           return;
3507         }
3508
3509       churn_data = GNUNET_malloc_large (frstat.st_size);
3510       GNUNET_assert(churn_data != NULL);
3511       if (frstat.st_size != GNUNET_DISK_fn_read (churn_filename, churn_data,
3512                                                  frstat.st_size))
3513         {
3514           GNUNET_log (
3515                       GNUNET_ERROR_TYPE_ERROR,
3516                       "Could not read file %s specified for churn, ending test!",
3517                       churn_filename);
3518           GNUNET_free (churn_filename);
3519           GNUNET_free (churn_data);
3520           GNUNET_free_non_null(trialmessage);
3521           return;
3522         }
3523
3524       GNUNET_free_non_null(churn_filename);
3525
3526       buf = churn_data;
3527       count = 0;
3528       /* Read the first line */
3529       while (count < frstat.st_size)
3530         {
3531           count++;
3532           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3533             {
3534               churn_data[count] = '\0';
3535               if (1 != sscanf (buf, "%u", &churn_rounds))
3536                 {
3537                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3538                               "Failed to read number of rounds from churn file, ending test!\n");
3539                   ret = 4200;
3540                   GNUNET_free_non_null(trialmessage);
3541                   GNUNET_free_non_null(churn_data);
3542                   return;
3543                 }
3544               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3545                           "Read %u rounds from churn file\n", churn_rounds);
3546               buf = &churn_data[count + 1];
3547               churn_array = GNUNET_malloc(sizeof(unsigned int) * churn_rounds);
3548               break; /* Done with this part */
3549             }
3550         }
3551
3552       if (GNUNET_OK
3553           != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3554                                                     "churns_per_round",
3555                                                     &churns_per_round))
3556         {
3557           churns_per_round = (unsigned long long) churn_rounds;
3558         }
3559
3560       line_number = 0;
3561       while ((count < frstat.st_size) && (line_number < churn_rounds))
3562         {
3563           count++;
3564           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3565             {
3566               churn_data[count] = '\0';
3567
3568               ret = sscanf (buf, "%u", &churn_array[line_number]);
3569               if (1 == ret)
3570                 {
3571                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3572                               "Read %u peers in round %u\n",
3573                               churn_array[line_number], line_number);
3574                   line_number++;
3575                 }
3576               else
3577                 {
3578                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3579                               "Error reading line `%s' in hostfile\n", buf);
3580                   buf = &churn_data[count + 1];
3581                   continue;
3582                 }
3583               buf = &churn_data[count + 1];
3584             }
3585           else if (churn_data[count] == '\n') /* Blank line */
3586             buf = &churn_data[count + 1];
3587         }
3588     }
3589   GNUNET_free_non_null(churn_data);
3590
3591   /* Check for a hostfile containing user@host:port triples */
3592   if (GNUNET_OK
3593       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "hostfile",
3594                                                 &hostfile))
3595     hostfile = NULL;
3596
3597   hosts = NULL;
3598   temphost = NULL;
3599   data = NULL;
3600   if (hostfile != NULL)
3601     {
3602       if (GNUNET_OK != GNUNET_DISK_file_test (hostfile))
3603         GNUNET_DISK_fn_write (hostfile, NULL, 0, GNUNET_DISK_PERM_USER_READ
3604             | GNUNET_DISK_PERM_USER_WRITE);
3605       if ((0 != STAT (hostfile, &frstat)) || (frstat.st_size == 0))
3606         {
3607           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3608                       "Could not open file specified for host list, ending test!");
3609           ok = 1119;
3610           GNUNET_free_non_null(trialmessage);
3611           GNUNET_free(hostfile);
3612           return;
3613         }
3614
3615       data = GNUNET_malloc_large (frstat.st_size);
3616       GNUNET_assert(data != NULL);
3617       if (frstat.st_size
3618           != GNUNET_DISK_fn_read (hostfile, data, frstat.st_size))
3619         {
3620           GNUNET_log (
3621                       GNUNET_ERROR_TYPE_ERROR,
3622                       "Could not read file %s specified for host list, ending test!",
3623                       hostfile);
3624           GNUNET_free (hostfile);
3625           GNUNET_free (data);
3626           GNUNET_free_non_null(trialmessage);
3627           return;
3628         }
3629
3630       GNUNET_free_non_null(hostfile);
3631
3632       buf = data;
3633       count = 0;
3634       while (count < frstat.st_size - 1)
3635         {
3636           count++;
3637           if (((data[count] == '\n')) && (buf != &data[count]))
3638             {
3639               data[count] = '\0';
3640               temphost = GNUNET_malloc(sizeof(struct GNUNET_TESTING_Host));
3641               ret = sscanf (buf, "%a[a-zA-Z0-9_]@%a[a-zA-Z0-9.]:%hd",
3642                             &temphost->username, &temphost->hostname,
3643                             &temphost->port);
3644               if (3 == ret)
3645                 {
3646                   GNUNET_log (
3647                               GNUNET_ERROR_TYPE_DEBUG,
3648                               "Successfully read host %s, port %d and user %s from file\n",
3649                               temphost->hostname, temphost->port,
3650                               temphost->username);
3651                 }
3652               else
3653                 {
3654                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3655                               "Error reading line `%s' in hostfile\n", buf);
3656                   GNUNET_free(temphost);
3657                   buf = &data[count + 1];
3658                   continue;
3659                 }
3660               temphost->next = hosts;
3661               hosts = temphost;
3662               buf = &data[count + 1];
3663             }
3664           else if ((data[count] == '\n') || (data[count] == '\0'))
3665             buf = &data[count + 1];
3666         }
3667     }
3668   GNUNET_free_non_null(data);
3669   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3670                                                           "malicious_getters",
3671                                                           &malicious_getters))
3672     malicious_getters = 0;
3673
3674   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3675                                                           "malicious_putters",
3676                                                           &malicious_putters))
3677     malicious_putters = 0;
3678
3679   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3680                                                           "malicious_droppers",
3681                                                           &malicious_droppers))
3682     malicious_droppers = 0;
3683
3684   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3685                                                           "settle_time",
3686                                                           &settle_time))
3687     settle_time = 0;
3688
3689   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3690                                                               "dht_testing",
3691                                                               "num_puts",
3692                                                               &num_puts))
3693     num_puts = num_peers;
3694
3695   if (GNUNET_SYSERR
3696       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3697                                                 "put_replication",
3698                                                 &put_replication))
3699     put_replication = DEFAULT_PUT_REPLICATION;
3700
3701   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3702                                                               "dht_testing",
3703                                                               "num_gets",
3704                                                               &num_gets))
3705     num_gets = num_peers;
3706
3707   if (GNUNET_SYSERR
3708       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3709                                                 "get_replication",
3710                                                 &get_replication))
3711     get_replication = DEFAULT_GET_REPLICATION;
3712
3713   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3714                                                           "find_peer_delay",
3715                                                           &temp_config_number))
3716     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3717                                                      temp_config_number);
3718   else
3719     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3720
3721   if (GNUNET_OK
3722       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3723                                                 "concurrent_find_peers",
3724                                                 &temp_config_number))
3725     max_outstanding_find_peers = temp_config_number;
3726   else
3727     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3728
3729   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3730                                                           "get_timeout",
3731                                                           &temp_config_number))
3732     get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3733                                                  temp_config_number);
3734   else
3735     get_timeout = DEFAULT_GET_TIMEOUT;
3736
3737   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3738                                                           "concurrent_puts",
3739                                                           &temp_config_number))
3740     max_outstanding_puts = temp_config_number;
3741   else
3742     max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
3743
3744   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3745                                                           "concurrent_gets",
3746                                                           &temp_config_number))
3747     max_outstanding_gets = temp_config_number;
3748   else
3749     max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
3750
3751   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3752                                                           "timeout",
3753                                                           &temp_config_number))
3754     all_get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3755                                                      temp_config_number);
3756   else
3757     all_get_timeout.rel_value = get_timeout.rel_value * num_gets;
3758
3759   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3760                                                           "get_delay",
3761                                                           &temp_config_number))
3762     get_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3763                                                temp_config_number);
3764   else
3765     get_delay = DEFAULT_GET_DELAY;
3766
3767   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3768                                                           "put_delay",
3769                                                           &temp_config_number))
3770     put_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3771                                                temp_config_number);
3772   else
3773     put_delay = DEFAULT_PUT_DELAY;
3774
3775   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3776                                                           "peer_start_timeout",
3777                                                           &temp_config_number))
3778     seconds_per_peer_start
3779         = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3780                                          temp_config_number);
3781   else
3782     seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START;
3783
3784   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3785                                                           "data_size",
3786                                                           &temp_config_number))
3787     test_data_size = temp_config_number;
3788   else
3789     test_data_size = DEFAULT_TEST_DATA_SIZE;
3790
3791   /**
3792    * Get testing related options.
3793    */
3794   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3795                                                           "REPLICATE_SAME"))
3796     replicate_same = GNUNET_YES;
3797
3798   /**
3799    * Get testing related options.
3800    */
3801   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3802                                                           "GET_FROM_SAME"))
3803     get_from_same = GNUNET_YES;
3804
3805   if (GNUNET_NO
3806       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3807                                               "MALICIOUS_GET_FREQUENCY",
3808                                               &malicious_get_frequency))
3809     malicious_get_frequency = DEFAULT_MALICIOUS_GET_FREQUENCY;
3810
3811   if (GNUNET_NO
3812       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3813                                               "MALICIOUS_PUT_FREQUENCY",
3814                                               &malicious_put_frequency))
3815     malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY;
3816
3817   if (GNUNET_YES
3818       == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3819                                                "MALICIOUS_AFTER_SETTLE"))
3820     malicious_after_settle = GNUNET_YES;
3821
3822   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3823                                                           "MALICIOUS_SYBIL"))
3824     {
3825       /* Set up the malicious target at random for this round */
3826       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
3827                                         &sybil_target);
3828       malicious_sybil = GNUNET_YES;
3829     }
3830
3831   /* Create the bloomfilter for choosing which peers to set malicious */
3832
3833   /* Bloomfilter size must be 2^k for some integer k */
3834   k = 1;
3835   while (1 << k < malicious_droppers)
3836     k++;
3837   if (malicious_droppers > 0)
3838     malicious_bloom = GNUNET_CONTAINER_bloomfilter_init (NULL, 1 << k,
3839                                                          DHT_BLOOM_K);
3840
3841   /* The normal behavior of the DHT is to do find peer requests
3842    * on its own.  Only if this is explicitly turned off should
3843    * the testing driver issue find peer requests (even though
3844    * this is likely the default when testing).
3845    */
3846   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3847                                                          "do_find_peer"))
3848     do_find_peer = GNUNET_YES;
3849
3850   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3851                                                            "insert_gauger_data"))
3852     insert_gauger_data = GNUNET_YES;
3853
3854   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3855                                                           "republish"))
3856     in_dht_replication = GNUNET_YES;
3857
3858   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3859                                                            "TRIAL_TO_RUN",
3860                                                            &trial_to_run))
3861     trial_to_run = 0;
3862
3863   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3864                                                            "REVISION",
3865                                                            &revision))
3866     revision = 0;
3867
3868   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3869                                                            "FIND_PEER_DELAY",
3870                                                            &temp_config_number))
3871     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3872                                                      temp_config_number);
3873   else
3874     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3875
3876   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3877                                                            "ROUND_DELAY",
3878                                                            &round_delay))
3879     round_delay = 0;
3880
3881   if (GNUNET_NO
3882       == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3883                                                 "OUTSTANDING_FIND_PEERS",
3884                                                 &max_outstanding_find_peers))
3885     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3886
3887   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3888                                                           "strict_kademlia"))
3889     max_outstanding_find_peers = max_outstanding_find_peers * 1;
3890
3891   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
3892                                                   max_outstanding_find_peers);
3893
3894   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3895                                                               "dht_testing",
3896                                                               "num_rounds",
3897                                                               &total_rounds))
3898     total_rounds = 1;
3899
3900   if ((GNUNET_SYSERR
3901       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3902                                                 "target_total_connections",
3903                                                 &target_total_connections))
3904       || (target_total_connections == 0))
3905     target_total_connections = connection_estimate (num_peers,
3906                                                     DEFAULT_BUCKET_SIZE);
3907
3908   topology_str = NULL;
3909   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3910                                                             "topology",
3911                                                             &topology_str))
3912       && (GNUNET_NO == GNUNET_TESTING_topology_get (&topology, topology_str)))
3913     {
3914       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3915                   "Invalid topology `%s' given for section %s option %s\n",
3916                   topology_str, "TESTING", "TOPOLOGY");
3917       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
3918     }
3919
3920   if (GNUNET_OK
3921       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage",
3922                                                 &topology_percentage_str))
3923     topology_percentage = 0.5;
3924   else
3925     {
3926       topology_percentage = atof (topology_percentage_str);
3927       GNUNET_free(topology_percentage_str);
3928     }
3929
3930   if (GNUNET_OK
3931       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability",
3932                                                 &topology_probability_str))
3933     topology_probability = 0.5;
3934   else
3935     {
3936       topology_probability = atof (topology_probability_str);
3937       GNUNET_free(topology_probability_str);
3938     }
3939
3940   if ((GNUNET_YES
3941       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3942                                                 "connect_topology",
3943                                                 &connect_topology_str))
3944       && (GNUNET_NO == GNUNET_TESTING_topology_get (&connect_topology,
3945                                                     connect_topology_str)))
3946     {
3947       GNUNET_log (
3948                   GNUNET_ERROR_TYPE_WARNING,
3949                   "Invalid connect topology `%s' given for section %s option %s\n",
3950                   connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
3951     }
3952   GNUNET_free_non_null(connect_topology_str);
3953
3954   if ((GNUNET_YES
3955       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3956                                                 "connect_topology_option",
3957                                                 &connect_topology_option_str))
3958       && (GNUNET_NO
3959           == GNUNET_TESTING_topology_option_get (&connect_topology_option,
3960                                                  connect_topology_option_str)))
3961     {
3962       GNUNET_log (
3963                   GNUNET_ERROR_TYPE_WARNING,
3964                   "Invalid connect topology option `%s' given for section %s option %s\n",
3965                   connect_topology_option_str, "TESTING",
3966                   "CONNECT_TOPOLOGY_OPTION");
3967       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
3968     }
3969   GNUNET_free_non_null(connect_topology_option_str);
3970
3971   if (GNUNET_YES
3972       == GNUNET_CONFIGURATION_get_value_string (
3973                                                 cfg,
3974                                                 "testing",
3975                                                 "connect_topology_option_modifier",
3976                                                 &connect_topology_option_modifier_string))
3977     {
3978       if (sscanf (connect_topology_option_modifier_string, "%lf",
3979                   &connect_topology_option_modifier) != 1)
3980         {
3981           GNUNET_log (
3982                       GNUNET_ERROR_TYPE_WARNING,
3983                       _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
3984                       connect_topology_option_modifier_string,
3985                       "connect_topology_option_modifier", "TESTING");
3986         }
3987       GNUNET_free (connect_topology_option_modifier_string);
3988     }
3989
3990   if (GNUNET_YES
3991       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3992                                                 "blacklist_transports",
3993                                                 &blacklist_transports))
3994     blacklist_transports = NULL;
3995
3996   if ((GNUNET_YES
3997       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3998                                                 "blacklist_topology",
3999                                                 &blacklist_topology_str))
4000       && (GNUNET_NO == GNUNET_TESTING_topology_get (&blacklist_topology,
4001                                                     blacklist_topology_str)))
4002     {
4003       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4004                   "Invalid topology `%s' given for section %s option %s\n",
4005                   topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
4006     }
4007   GNUNET_free_non_null(topology_str);
4008   GNUNET_free_non_null(blacklist_topology_str);
4009
4010   /* Set peers_left so we know when all peers started */
4011   peers_left = num_peers;
4012
4013   /* Set up a task to end testing if peer start fails */
4014   die_task
4015       = GNUNET_SCHEDULER_add_delayed (
4016                                       GNUNET_TIME_relative_multiply (
4017                                                                      seconds_per_peer_start,
4018                                                                      num_peers),
4019                                       &end_badly,
4020                                       "didn't generate all hostkeys within allowed startup time!");
4021
4022   if (dhtlog_handle == NULL)
4023     GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "dhtlog_handle is NULL!");
4024
4025   trial_info.other_identifier = (unsigned int) trial_to_run;
4026   trial_info.num_nodes = peers_left;
4027   trial_info.topology = topology;
4028   trial_info.blacklist_topology = blacklist_topology;
4029   trial_info.connect_topology = connect_topology;
4030   trial_info.connect_topology_option = connect_topology_option;
4031   trial_info.connect_topology_option_modifier
4032       = connect_topology_option_modifier;
4033   trial_info.topology_percentage = topology_percentage;
4034   trial_info.topology_probability = topology_probability;
4035   trial_info.puts = num_puts;
4036   trial_info.gets = num_gets;
4037   trial_info.concurrent = max_outstanding_gets;
4038   trial_info.settle_time = settle_time;
4039   trial_info.num_rounds = total_rounds;
4040   trial_info.malicious_getters = malicious_getters;
4041   trial_info.malicious_putters = malicious_putters;
4042   trial_info.malicious_droppers = malicious_droppers;
4043   trial_info.malicious_get_frequency = malicious_get_frequency.rel_value;
4044   trial_info.malicious_put_frequency = malicious_put_frequency.rel_value;
4045   trial_info.stop_closest = stop_closest;
4046   trial_info.stop_found = stop_found;
4047   trial_info.strict_kademlia = strict_kademlia;
4048
4049   if (trialmessage != NULL)
4050     trial_info.message = trialmessage;
4051   else
4052     trial_info.message = "";
4053
4054   if (dhtlog_handle != NULL)
4055     dhtlog_handle->insert_trial (&trial_info);
4056
4057   GNUNET_free_non_null(trialmessage);
4058
4059   hostkey_meter = create_meter (peers_left, "Hostkeys created ", GNUNET_YES);
4060   peer_start_meter = create_meter (peers_left, "Peers started ", GNUNET_YES);
4061
4062   put_meter = create_meter (num_puts, "Puts completed ", GNUNET_YES);
4063   get_meter = create_meter (num_gets, "Gets completed ", GNUNET_YES);
4064   hostkey_start_time = GNUNET_TIME_absolute_get();
4065   pg
4066       = GNUNET_TESTING_daemons_start (
4067                                       cfg,
4068                                       peers_left,
4069                                       max_outstanding_connections,
4070                                       max_concurrent_ssh,
4071                                       GNUNET_TIME_relative_multiply (
4072                                                                      seconds_per_peer_start,
4073                                                                      num_peers),
4074                                       &hostkey_callback, NULL,
4075                                       &peers_started_callback, NULL,
4076                                       &topology_callback, NULL, hosts);
4077   temphost = hosts;
4078   while (temphost != NULL)
4079     {
4080       tempnext = temphost->next;
4081       GNUNET_free (temphost->username);
4082       GNUNET_free (temphost->hostname);
4083       GNUNET_free (temphost);
4084       temphost = tempnext;
4085     }
4086 }
4087
4088 int
4089 main(int argc, char *argv[])
4090 {
4091   int ret;
4092   struct GNUNET_GETOPT_CommandLineOption options[] =
4093     {
4094     GNUNET_GETOPT_OPTION_END };
4095
4096   ret = GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-driver", "nohelp", options,
4097                             &run, &ok);
4098
4099   if (malicious_bloom != NULL)
4100     GNUNET_CONTAINER_bloomfilter_free (malicious_bloom);
4101
4102   if (ret != GNUNET_OK)
4103     {
4104       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4105                   "`gnunet-dht-driver': Failed with error code %d\n", ret);
4106     }
4107
4108   /**
4109    * Need to remove base directory, subdirectories taken care
4110    * of by the testing framework.
4111    */
4112   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
4113     {
4114       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4115                   "Failed to remove testing directory %s\n", test_directory);
4116     }
4117   return ret;
4118 }
4119
4120 /* end of gnunet-dht-driver.c */