src/dht/gnunet-dht-driver.c

   1 /*
   2  This file is part of GNUnet.
   3  (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5  GNUnet is free software; you can redistribute it and/or modify
   6  it under the terms of the GNU General Public License as published
   7  by the Free Software Foundation; either version 3, or (at your
   8  option) any later version.
   9
  10  GNUnet is distributed in the hope that it will be useful, but
  11  WITHOUT ANY WARRANTY; without even the implied warranty of
  12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  General Public License for more details.
  14
  15  You should have received a copy of the GNU General Public License
  16  along with GNUnet; see the file COPYING.  If not, write to the
  17  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18  Boston, MA 02111-1307, USA.
  19  */
  20 /**
  21  * @file dht/gnunet-dht-driver.c
  22  * @brief Driver for setting up a group of gnunet peers and
  23  *        then issuing GETS and PUTS on the DHT.  Coarse results
  24  *        are reported, fine grained results (if requested) are
  25  *        logged to a (mysql) database, or to file.
  26  * @author Nathan Evans (who to blame)
  27  */
  28 #include "platform.h"
  29 #ifndef HAVE_MALICIOUS
  30 #error foo
  31 #endif
  32 #include "gnunet_testing_lib.h"
  33 #include "gnunet_core_service.h"
  34 #include "gnunet_dht_service.h"
  35 #include "dhtlog.h"
  36 #include "dht.h"
  37 #include "gauger.h"
  38
  39 /* Specific DEBUG hack, do not use normally (may leak memory, segfault, or eat children.) */
  40 #define ONLY_TESTING GNUNET_NO
  41
  42 /* DEFINES */
  43 #define VERBOSE GNUNET_NO
  44
  45 /* Timeout for entire driver to run */
  46 #define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  47
  48 /* Timeout for waiting for (individual) replies to get requests */
  49 #define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  50
  51 #define DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  52
  53 /* Timeout for waiting for gets to be sent to the service */
  54 #define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  55
  56 /* Timeout for waiting for puts to be sent to the service */
  57 #define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  58
  59 /* Time to allow a find peer request to take */
  60 #define DEFAULT_FIND_PEER_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40)
  61
  62 /* Time to wait for all peers disconnected due to to churn to actually be removed from system */
  63 #define DEFAULT_PEER_DISCONNECT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  64
  65 #define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45)
  66
  67 #define DEFAULT_TEST_DATA_SIZE 8
  68
  69 #define DEFAULT_BUCKET_SIZE 4
  70
  71 /* If more than this many peers are added, slow down sending */
  72 #define MAX_FIND_PEER_CUTOFF 2000
  73
  74 /* If less than this many peers are added, speed up sending */
  75 #define MIN_FIND_PEER_CUTOFF 500
  76
  77 /* How often (in seconds) to print out connection information */
  78 #define CONN_UPDATE_DURATION 10
  79
  80 #define DEFAULT_MAX_OUTSTANDING_PUTS 10
  81
  82 #define DEFAULT_MAX_OUTSTANDING_FIND_PEERS 64
  83
  84 #define DEFAULT_FIND_PEER_OFFSET GNUNET_TIME_relative_divide (DEFAULT_FIND_PEER_DELAY, DEFAULT_MAX_OUTSTANDING_FIND_PEERS)
  85
  86 #define DEFAULT_MAX_OUTSTANDING_GETS 10
  87
  88 #define DEFAULT_CONNECT_TIMEOUT 60
  89
  90 #define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8)
  91
  92 #define DEFAULT_RECONNECT_ATTEMPTS 8
  93
  94 /*
  95  * Default frequency for sending malicious get messages
  96  */
  97 #define DEFAULT_MALICIOUS_GET_FREQUENCY GNUNET_TIME_UNIT_SECONDS
  98
  99 /*
 100  * Default frequency for sending malicious put messages
 101  */
 102 #define DEFAULT_MALICIOUS_PUT_FREQUENCY GNUNET_TIME_UNIT_SECONDS
 103
 104 /* Structs */
 105
 106 struct MaliciousContext
 107 {
 108   /**
 109    * Handle to DHT service (via the API)
 110    */
 111   struct GNUNET_DHT_Handle *dht_handle;
 112
 113   /**
 114    *  Handle to the peer daemon
 115    */
 116   struct GNUNET_TESTING_Daemon *daemon;
 117
 118   /**
 119    * Task for disconnecting DHT handles
 120    */
 121   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 122
 123   /**
 124    * What type of malicious to set this peer to.
 125    */
 126   int malicious_type;
 127 };
 128
 129 struct TestFindPeer
 130 {
 131   /* This is a linked list */
 132   struct TestFindPeer *next;
 133
 134   /* Handle to the bigger context */
 135   struct FindPeerContext *find_peer_context;
 136
 137   /**
 138    * Handle to the peer's DHT service (via the API)
 139    */
 140   struct GNUNET_DHT_Handle *dht_handle;
 141
 142   /**
 143    *  Handle to the peer daemon
 144    */
 145   struct GNUNET_TESTING_Daemon *daemon;
 146
 147   /**
 148    * Task for disconnecting DHT handles
 149    */
 150   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 151 };
 152
 153 struct TestPutContext
 154 {
 155   /* This is a linked list */
 156   struct TestPutContext *next;
 157
 158   /**
 159    * Handle to the first peers DHT service (via the API)
 160    */
 161   struct GNUNET_DHT_Handle *dht_handle;
 162
 163   /**
 164    *  Handle to the PUT peer daemon
 165    */
 166   struct GNUNET_TESTING_Daemon *daemon;
 167
 168   /**
 169    *  Identifier for this PUT
 170    */
 171   uint32_t uid;
 172
 173   /**
 174    * Task for disconnecting DHT handles
 175    */
 176   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 177 };
 178
 179 struct TestGetContext
 180 {
 181   /* This is a linked list */
 182   struct TestGetContext *next;
 183
 184   /**
 185    * Handle to the first peers DHT service (via the API)
 186    */
 187   struct GNUNET_DHT_Handle *dht_handle;
 188
 189   /**
 190    * Handle for the DHT get request
 191    */
 192   struct GNUNET_DHT_GetHandle *get_handle;
 193
 194   /**
 195    *  Handle to the GET peer daemon
 196    */
 197   struct GNUNET_TESTING_Daemon *daemon;
 198
 199   /**
 200    *  Identifier for this GET
 201    */
 202   uint32_t uid;
 203
 204   /**
 205    * Task for disconnecting DHT handles (and stopping GET)
 206    */
 207   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 208
 209   /**
 210    * Whether or not this request has been fulfilled already.
 211    */
 212   int succeeded;
 213 };
 214
 215 /**
 216  * Simple struct to keep track of progress, and print a
 217  * nice little percentage meter for long running tasks.
 218  */
 219 struct ProgressMeter
 220 {
 221   unsigned int total;
 222
 223   unsigned int modnum;
 224
 225   unsigned int dotnum;
 226
 227   unsigned int completed;
 228
 229   int print;
 230
 231   char *startup_string;
 232 };
 233
 234 /**
 235  * Linked list of information for populating statistics
 236  * before ending trial.
 237  */
 238 struct StatisticsIteratorContext
 239 {
 240   const struct GNUNET_PeerIdentity *peer;
 241   unsigned int stat_routes;
 242   unsigned int stat_route_forwards;
 243   unsigned int stat_results;
 244   unsigned int stat_results_to_client;
 245   unsigned int stat_result_forwards;
 246   unsigned int stat_gets;
 247   unsigned int stat_puts;
 248   unsigned int stat_puts_inserted;
 249   unsigned int stat_find_peer;
 250   unsigned int stat_find_peer_start;
 251   unsigned int stat_get_start;
 252   unsigned int stat_put_start;
 253   unsigned int stat_find_peer_reply;
 254   unsigned int stat_get_reply;
 255   unsigned int stat_find_peer_answer;
 256   unsigned int stat_get_response_start;
 257 };
 258
 259 /**
 260  * Context for getting a topology, logging it, and continuing
 261  * on with some next operation.
 262  */
 263 struct TopologyIteratorContext
 264 {
 265   unsigned int total_iterations;
 266   unsigned int current_iteration;
 267   unsigned int total_connections;
 268   unsigned int total_peers;
 269   struct GNUNET_CONTAINER_MultiHashMap *peers_seen;
 270   struct GNUNET_PeerIdentity *peer;
 271   GNUNET_SCHEDULER_Task cont;
 272   void *cls;
 273   struct GNUNET_TIME_Relative timeout;
 274 };
 275
 276 struct PeerCount
 277 {
 278   /** Node in the heap */
 279   struct GNUNET_CONTAINER_HeapNode *heap_node;
 280
 281   /** Peer the count refers to */
 282   struct GNUNET_PeerIdentity peer_id;
 283
 284   /** Count of connections this peer has */
 285   unsigned int count;
 286 };
 287
 288 /**
 289  * Context for sending out find peer requests.
 290  */
 291 struct FindPeerContext
 292 {
 293   /**
 294    * How long to send find peer requests, once the settle time
 295    * is over don't send any more out!
 296    */
 297   struct GNUNET_TIME_Absolute endtime;
 298
 299   /**
 300    * Number of connections in the current topology
 301    * (after this round of find peer requests has ended).
 302    */
 303   unsigned int current_peers;
 304
 305   /**
 306    * Number of connections in the current topology
 307    * (before this round of find peer requests started).
 308    */
 309   unsigned int previous_peers;
 310
 311   /**
 312    * Number of find peer requests we have currently
 313    * outstanding.
 314    */
 315   unsigned int outstanding;
 316
 317   /**
 318    * Number of find peer requests to send in this round.
 319    */
 320   unsigned int total;
 321
 322   /**
 323    * Number of find peer requests sent last time around.
 324    */
 325   unsigned int last_sent;
 326
 327   /**
 328    * Hashmap of peers in the current topology, value
 329    * is a PeerCount, with the number of connections
 330    * this peer has.
 331    */
 332   struct GNUNET_CONTAINER_MultiHashMap *peer_hash;
 333
 334   /**
 335    * Min heap which orders values in the peer_hash for
 336    * easy lookup.
 337    */
 338   struct GNUNET_CONTAINER_Heap *peer_min_heap;
 339
 340   /**
 341    * Callback for counting the peers in the current topology.
 342    */
 343   GNUNET_TESTING_NotifyTopology count_peers_cb;
 344 };
 345
 346 enum DHT_ROUND_TYPES
 347 {
 348   /**
 349    * Next full round (puts + gets).
 350    */
 351   DHT_ROUND_NORMAL,
 352
 353   /**
 354    * Next round of gets.
 355    */
 356   DHT_ROUND_GET,
 357
 358   /**
 359    * Next round of puts.
 360    */
 361   DHT_ROUND_PUT,
 362
 363   /**
 364    * Next round of churn.
 365    */
 366   DHT_ROUND_CHURN
 367 };
 368
 369 /* Globals */
 370
 371 /**
 372  * How long to try to connect two peers.
 373  */
 374 struct GNUNET_TIME_Relative connect_timeout;
 375
 376 /**
 377  * How many times to re-attempt connecting two peers.
 378  */
 379 static unsigned long long connect_attempts;
 380
 381 /**
 382  * Timeout to let all GET requests happen.
 383  */
 384 static struct GNUNET_TIME_Relative all_get_timeout;
 385
 386 /**
 387  * Per get timeout
 388  */
 389 static struct GNUNET_TIME_Relative get_timeout;
 390
 391 /**
 392  * Time to allow for GET requests to be sent to service.
 393  */
 394 static struct GNUNET_TIME_Relative get_delay;
 395
 396 /**
 397  * Time to allow for PUT requests to be sent to service.
 398  */
 399 static struct GNUNET_TIME_Relative put_delay;
 400
 401 /**
 402  * Delay between sending find peer requests (if
 403  * handled by the driver, no effect if sent by service).
 404  */
 405 static struct GNUNET_TIME_Relative find_peer_delay;
 406
 407 /**
 408  * Time between find peer requests
 409  * (find_peer_delay / max_outstanding_find_peer)
 410  */
 411 static struct GNUNET_TIME_Relative find_peer_offset;
 412
 413 /**
 414  * How many seconds to allow each peer to start.
 415  */
 416 static struct GNUNET_TIME_Relative seconds_per_peer_start;
 417
 418 /**
 419  * At what time did we start the connection process.
 420  */
 421 static struct GNUNET_TIME_Absolute connect_start_time;
 422
 423 /**
 424  * What was the last time we updated connection/second information.
 425  */
 426 static struct GNUNET_TIME_Absolute connect_last_time;
 427
 428 /**
 429  * Boolean value, should the driver issue find peer requests
 430  * (GNUNET_YES) or should it be left to the service (GNUNET_NO)
 431  */
 432 static unsigned int do_find_peer;
 433
 434 /**
 435  * Whether or not to insert gauger data.
 436  */
 437 static unsigned int insert_gauger_data;
 438
 439 #if ONLY_TESTING
 440 /**
 441  * Are we currently trying to connect two peers repeatedly?
 442  */
 443 static unsigned int repeat_connect_mode;
 444
 445 /**
 446  * Task for repeating connects.
 447  */
 448 GNUNET_SCHEDULER_TaskIdentifier repeat_connect_task;
 449
 450 struct GNUNET_TESTING_Daemon *repeat_connect_peer1;
 451 struct GNUNET_TESTING_Daemon *repeat_connect_peer2;
 452 #endif
 453
 454 /**
 455  * Boolean value, should replication be done by the dht
 456  * service (GNUNET_YES) or by the driver (GNUNET_NO)
 457  */
 458 static unsigned int in_dht_replication;
 459
 460 /**
 461  * Size of test data to insert/retrieve during testing.
 462  */
 463 static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE;
 464
 465 /**
 466  * Maximum number of concurrent connections to peers.
 467  */
 468 static unsigned long long max_outstanding_connections;
 469
 470 /**
 471  * Maximum number of concurrent ssh instances to peers.
 472  */
 473 static unsigned long long max_concurrent_ssh;
 474
 475 /**
 476  * Maximum number of concurrent PUT requests.
 477  */
 478 static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
 479
 480 /**
 481  * Maximum number of concurrent GET requests.
 482  */
 483 static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
 484
 485 /**
 486  * Number of nodes issuing malicious GET messages.
 487  */
 488 static unsigned long long malicious_getters;
 489
 490 /**
 491  * Maximum number of concurrent find peer messages being sent.
 492  */
 493 static unsigned long long max_outstanding_find_peers;
 494
 495 /**
 496  * Number of nodes issuing malicious PUT messages.
 497  */
 498 static unsigned long long malicious_putters;
 499
 500 /**
 501  * Time (in seconds) to delay between rounds.
 502  */
 503 static unsigned long long round_delay;
 504
 505 /**
 506  * The identifier for this trial (if we have one)
 507  * for external data collection.
 508  */
 509 static unsigned long long trial_to_run;
 510
 511 /**
 512  * How many malicious droppers to seed in the network.
 513  */
 514 static unsigned long long malicious_droppers;
 515
 516 /**
 517  * Bloom filter to restrict malicious nodes chosen.
 518  */
 519 struct GNUNET_CONTAINER_BloomFilter *malicious_bloom;
 520
 521 /**
 522  * Whether malicious droppers should be chosen based on proximity to a key.
 523  */
 524 static int malicious_sybil;
 525
 526 /**
 527  * Target for the malicious sybil nodes (choose the closest to this key).
 528  */
 529 static GNUNET_HashCode sybil_target;
 530
 531 /**
 532  * How often to send malicious GET messages.
 533  */
 534 static struct GNUNET_TIME_Relative malicious_get_frequency;
 535
 536 /**
 537  * How often to send malicious PUT messages.
 538  */
 539 static struct GNUNET_TIME_Relative malicious_put_frequency;
 540
 541 /**
 542  * How long to send find peer requests.
 543  */
 544 static unsigned long long settle_time;
 545
 546 /**
 547  * Handle to the dhtlog service.
 548  */
 549 static struct GNUNET_DHTLOG_Handle *dhtlog_handle;
 550
 551 /**
 552  * Replication value for GET requests.
 553  */
 554 static unsigned long long get_replication;
 555
 556 /**
 557  * Replication value for PUT requests.
 558  */
 559 static unsigned long long put_replication;
 560
 561 /**
 562  * If GNUNET_YES, insert data at the same peers every time.
 563  * Otherwise, choose a new random peer to insert at each time.
 564  */
 565 static unsigned int replicate_same;
 566
 567 /**
 568  * If GNUNET_YES, issue GET requests at the same peers every time.
 569  * Otherwise, choose a new random peer/data combination to search
 570  * each time.
 571  */
 572 static unsigned int get_from_same;
 573
 574 /**
 575  * Should malicious peers be set after allowing for settle time?
 576  * Default is to set them malicious after initial connection setup.
 577  */
 578 static unsigned int malicious_after_settle;
 579
 580 /**
 581  * Number of rounds for testing (PUTS + GETS)
 582  */
 583 static unsigned long long total_rounds;
 584
 585 /**
 586  * Target number of connections (will stop sending find peer
 587  * messages when this number is exceeded)
 588  */
 589 static unsigned long long target_total_connections;
 590
 591 /**
 592  * Number of rounds already run
 593  */
 594 static unsigned int rounds_finished;
 595
 596 /**
 597  * Number of rounds of churn to read from the file (first line, should be a single number).
 598  */
 599 static unsigned int churn_rounds;
 600
 601 /**
 602  * Current round we are in for churn, tells us how many peers to connect/disconnect.
 603  */
 604 static unsigned int current_churn_round;
 605
 606 /**
 607  * Number of times to churn per round
 608  */
 609 static unsigned long long churns_per_round;
 610
 611 /**
 612  * Array of churn values.
 613  */
 614 static unsigned int *churn_array;
 615
 616 /**
 617  * Hash map of stats contexts.
 618  */
 619 static struct GNUNET_CONTAINER_MultiHashMap *stats_map;
 620
 621 /**
 622  * LL of malicious settings.
 623  */
 624 struct MaliciousContext *all_malicious;
 625
 626 /**
 627  * List of GETS to perform
 628  */
 629 struct TestGetContext *all_gets;
 630
 631 /**
 632  * List of PUTS to perform
 633  */
 634 struct TestPutContext *all_puts;
 635
 636 /**
 637  * Directory to store temporary data in, defined in config file
 638  */
 639 static char *test_directory;
 640
 641 /**
 642  * Variable used to store the number of connections we should wait for.
 643  */
 644 static unsigned int expected_connections;
 645
 646 /**
 647  * Variable used to keep track of how many peers aren't yet started.
 648  */
 649 static unsigned long long peers_left;
 650
 651 /**
 652  * Handle to the set of all peers run for this test.
 653  */
 654 static struct GNUNET_TESTING_PeerGroup *pg;
 655
 656 /**
 657  * Global config handle.
 658  */
 659 static const struct GNUNET_CONFIGURATION_Handle *config;
 660
 661 /**
 662  * Total number of peers to run, set based on config file.
 663  */
 664 static unsigned long long num_peers;
 665
 666 /**
 667  * Total number of items to insert.
 668  */
 669 static unsigned long long num_puts;
 670
 671 /**
 672  * How many puts do we currently have in flight?
 673  */
 674 static unsigned long long outstanding_puts;
 675
 676 /**
 677  * How many puts are done?
 678  */
 679 static unsigned long long puts_completed;
 680
 681 /**
 682  * Total number of items to attempt to get.
 683  */
 684 static unsigned long long num_gets;
 685
 686 /**
 687  * How many puts do we currently have in flight?
 688  */
 689 static unsigned long long outstanding_gets;
 690
 691 /**
 692  * How many gets are done?
 693  */
 694 static unsigned long long gets_completed;
 695
 696 /**
 697  * Total number of items to attempt to get.
 698  */
 699 static unsigned long long cumulative_num_gets;
 700
 701 /**
 702  * How many gets are done?
 703  */
 704 static unsigned long long cumulative_successful_gets;
 705
 706 /**
 707  * How many gets failed?
 708  */
 709 static unsigned long long gets_failed;
 710
 711 /**
 712  * How many malicious control messages do
 713  * we currently have in flight?
 714  */
 715 static unsigned long long outstanding_malicious;
 716
 717 /**
 718  * How many set malicious peers are done?
 719  */
 720 static unsigned int malicious_completed;
 721
 722 /**
 723  * For gauger logging, what specific identifier (svn revision)
 724  * should be used?
 725  */
 726 static unsigned long long revision;
 727
 728 /**
 729  * Global used to count how many connections we have currently
 730  * been notified about (how many times has topology_callback been called
 731  * with success?)
 732  */
 733 static unsigned int total_connections;
 734
 735 /**
 736  * Previous connections, for counting new connections during some duration.
 737  */
 738 static unsigned int previous_connections;
 739
 740 /**
 741  * For counting failed connections during some duration.
 742  */
 743 static unsigned int previous_failed_connections;
 744
 745 /**
 746  * Global used to count how many failed connections we have
 747  * been notified about (how many times has topology_callback
 748  * been called with failure?)
 749  */
 750 static unsigned int failed_connections;
 751
 752 /**
 753  * If GNUNET_YES, only log PUT/GET round data to mysql, otherwise
 754  * log everything (including each dht service logging).
 755  */
 756 static unsigned int dhtlog_minimal;
 757
 758 /* Task handle to use to schedule shutdown if something goes wrong */
 759 GNUNET_SCHEDULER_TaskIdentifier die_task;
 760
 761 static char *blacklist_transports;
 762
 763 static enum GNUNET_TESTING_Topology topology;
 764
 765 static enum GNUNET_TESTING_Topology blacklist_topology =
 766     GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 767
 768 static enum GNUNET_TESTING_Topology connect_topology =
 769     GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 770
 771 static enum GNUNET_TESTING_TopologyOption connect_topology_option =
 772     GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 773
 774 static double connect_topology_option_modifier = 0.0;
 775
 776 static struct ProgressMeter *hostkey_meter;
 777
 778 static struct ProgressMeter *peer_start_meter;
 779
 780 static struct ProgressMeter *peer_connect_meter;
 781
 782 static struct ProgressMeter *put_meter;
 783
 784 static struct ProgressMeter *get_meter;
 785
 786 static GNUNET_HashCode *known_keys;
 787
 788 /* Global return value (0 for success, anything else for failure) */
 789 static int ok;
 790
 791 /**
 792  * Create a meter to keep track of the progress of some task.
 793  *
 794  * @param total the total number of items to complete
 795  * @param start_string a string to prefix the meter with (if printing)
 796  * @param print GNUNET_YES to print the meter, GNUNET_NO to count
 797  *              internally only
 798  *
 799  * @return the progress meter
 800  */
 801 static struct ProgressMeter *
 802 create_meter(unsigned int total, char * start_string, int print)
 803 {
 804   struct ProgressMeter *ret;
 805   ret = GNUNET_malloc(sizeof(struct ProgressMeter));
 806   ret->print = print;
 807   ret->total = total;
 808   ret->modnum = total / 4;
 809   ret->dotnum = (total / 50) + 1;
 810   if (start_string != NULL)
 811     ret->startup_string = GNUNET_strdup(start_string);
 812   else
 813     ret->startup_string = GNUNET_strdup("");
 814
 815   return ret;
 816 }
 817
 818 /**
 819  * Update progress meter (increment by one).
 820  *
 821  * @param meter the meter to update and print info for
 822  *
 823  * @return GNUNET_YES if called the total requested,
 824  *         GNUNET_NO if more items expected
 825  */
 826 static int
 827 update_meter(struct ProgressMeter *meter)
 828 {
 829   if (meter->print == GNUNET_YES)
 830     {
 831       if (meter->completed % meter->modnum == 0)
 832         {
 833           if (meter->completed == 0)
 834             {
 835               fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
 836             }
 837           else
 838             fprintf (stdout, "%d%%", (int) (((float) meter->completed
 839                 / meter->total) * 100));
 840         }
 841       else if (meter->completed % meter->dotnum == 0)
 842         fprintf (stdout, ".");
 843
 844       if (meter->completed + 1 == meter->total)
 845         fprintf (stdout, "%d%%]\n", 100);
 846       fflush (stdout);
 847     }
 848   meter->completed++;
 849
 850   if (meter->completed == meter->total)
 851     return GNUNET_YES;
 852   return GNUNET_NO;
 853 }
 854
 855 /**
 856  * Reset progress meter.
 857  *
 858  * @param meter the meter to reset
 859  *
 860  * @return GNUNET_YES if meter reset,
 861  *         GNUNET_SYSERR on error
 862  */
 863 static int
 864 reset_meter(struct ProgressMeter *meter)
 865 {
 866   if (meter == NULL)
 867     return GNUNET_SYSERR;
 868
 869   meter->completed = 0;
 870   return GNUNET_YES;
 871 }
 872
 873 /**
 874  * Release resources for meter
 875  *
 876  * @param meter the meter to free
 877  */
 878 static void
 879 free_meter(struct ProgressMeter *meter)
 880 {
 881   GNUNET_free_non_null (meter->startup_string);
 882   GNUNET_free (meter);
 883 }
 884
 885 /**
 886  * Check whether peers successfully shut down.
 887  */
 888 static void
 889 shutdown_callback(void *cls, const char *emsg)
 890 {
 891   if (emsg != NULL)
 892     {
 893       if (ok == 0)
 894         ok = 2;
 895     }
 896 }
 897
 898 /**
 899  * Task to release DHT handles for PUT
 900  */
 901 static void
 902 put_disconnect_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 903 {
 904   struct TestPutContext *test_put = cls;
 905   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 906   GNUNET_DHT_disconnect (test_put->dht_handle);
 907   test_put->dht_handle = NULL;
 908   if (replicate_same == GNUNET_NO)
 909     test_put->daemon
 910         = GNUNET_TESTING_daemon_get (
 911                                      pg,
 912                                      GNUNET_CRYPTO_random_u32 (
 913                                                                GNUNET_CRYPTO_QUALITY_WEAK,
 914                                                                num_peers));
 915 }
 916
 917 /**
 918  * Function scheduled to be run on the successful completion of this
 919  * testcase.
 920  */
 921 static void
 922 finish_testing(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 923 {
 924   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ending test normally!\n",
 925               (char *) cls);
 926   GNUNET_assert (pg != NULL);
 927   struct TestPutContext *test_put = all_puts;
 928   struct TestGetContext *test_get = all_gets;
 929   char *temp_get_string;
 930   char *revision_str;
 931
 932   while (test_put != NULL)
 933     {
 934       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 935         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
 936       if (test_put->dht_handle != NULL)
 937         GNUNET_DHT_disconnect (test_put->dht_handle);
 938       test_put = test_put->next;
 939     }
 940
 941   while (test_get != NULL)
 942     {
 943       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 944         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
 945       if (test_get->get_handle != NULL)
 946         GNUNET_DHT_get_stop (test_get->get_handle);
 947       if (test_get->dht_handle != NULL)
 948         GNUNET_DHT_disconnect (test_get->dht_handle);
 949       test_get = test_get->next;
 950     }
 951
 952   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
 953
 954   if (dhtlog_handle != NULL)
 955     {
 956       fprintf (stderr, "Update trial endtime\n");
 957       dhtlog_handle->update_trial (cumulative_successful_gets);
 958       GNUNET_DHTLOG_disconnect (dhtlog_handle);
 959       dhtlog_handle = NULL;
 960     }
 961
 962   if (hostkey_meter != NULL)
 963     free_meter (hostkey_meter);
 964   if (peer_start_meter != NULL)
 965     free_meter (peer_start_meter);
 966   if (peer_connect_meter != NULL)
 967     free_meter (peer_connect_meter);
 968   if (put_meter != NULL)
 969     free_meter (put_meter);
 970   if (get_meter != NULL)
 971     free_meter (get_meter);
 972
 973   GNUNET_asprintf (&temp_get_string,
 974                    "DHT Successful GETs (trial %d)", trial_to_run);
 975   GNUNET_asprintf (&revision_str, "%llu", revision);
 976   if (GNUNET_YES == insert_gauger_data)
 977     GAUGER_ID("DHT_TESTING", temp_get_string, cumulative_successful_gets / (double)cumulative_num_gets, "percent successful", revision_str);
 978   fprintf (
 979            stderr,
 980            "Finished trial, had %llu successful gets out of %llu total, %.2f percent succeeded\n",
 981            cumulative_successful_gets, cumulative_num_gets,
 982            cumulative_successful_gets / (double) cumulative_num_gets);
 983   GNUNET_free(temp_get_string);
 984
 985   ok = 0;
 986 }
 987
 988 /**
 989  * Callback for iterating over all the peer connections of a peer group.
 990  */
 991 static void
 992 log_topology_cb(void *cls, const struct GNUNET_PeerIdentity *first,
 993     const struct GNUNET_PeerIdentity *second, const char *emsg)
 994 {
 995   struct TopologyIteratorContext *topo_ctx = cls;
 996   if ((first != NULL) && (second != NULL))
 997     {
 998       if ((topo_ctx->peers_seen != NULL) && (GNUNET_NO
 999           == GNUNET_CONTAINER_multihashmap_contains (topo_ctx->peers_seen,
1000                                                      &first->hashPubKey)))
1001         {
1002           GNUNET_CONTAINER_multihashmap_put (topo_ctx->peers_seen,
1003                                              &first->hashPubKey, NULL,
1004                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1005           topo_ctx->total_peers++;
1006         }
1007       topo_ctx->total_connections++;
1008       if ((GNUNET_NO == dhtlog_minimal) && (dhtlog_handle != NULL))
1009         dhtlog_handle->insert_extended_topology (first, second);
1010     }
1011   else
1012     {
1013       GNUNET_assert(dhtlog_handle != NULL);
1014       GNUNET_log (
1015                   GNUNET_ERROR_TYPE_WARNING,
1016                   "Topology iteration (%u/%u) finished (%u connections, %u peers)\n",
1017                   topo_ctx->current_iteration, topo_ctx->total_iterations,
1018                   topo_ctx->total_connections, topo_ctx->total_peers);
1019       dhtlog_handle->update_topology (topo_ctx->total_connections);
1020       if (topo_ctx->cont != NULL)
1021         GNUNET_SCHEDULER_add_now (topo_ctx->cont, topo_ctx->cls);
1022       if (topo_ctx->peers_seen != NULL)
1023         GNUNET_CONTAINER_multihashmap_destroy (topo_ctx->peers_seen);
1024       GNUNET_free(topo_ctx);
1025     }
1026 }
1027
1028 /**
1029  * Iterator over hash map entries.
1030  *
1031  * @param cls closure - always NULL
1032  * @param key current key code
1033  * @param value value in the hash map, a stats context
1034  * @return GNUNET_YES if we should continue to
1035  *         iterate,
1036  *         GNUNET_NO if not.
1037  */
1038 static int
1039 stats_iterate(void *cls, const GNUNET_HashCode * key, void *value)
1040 {
1041   struct StatisticsIteratorContext *stats_ctx;
1042   if (value == NULL)
1043     return GNUNET_NO;
1044   stats_ctx = value;
1045   dhtlog_handle->insert_stat (stats_ctx->peer, stats_ctx->stat_routes,
1046                               stats_ctx->stat_route_forwards,
1047                               stats_ctx->stat_results,
1048                               stats_ctx->stat_results_to_client,
1049                               stats_ctx->stat_result_forwards,
1050                               stats_ctx->stat_gets, stats_ctx->stat_puts,
1051                               stats_ctx->stat_puts_inserted,
1052                               stats_ctx->stat_find_peer,
1053                               stats_ctx->stat_find_peer_start,
1054                               stats_ctx->stat_get_start,
1055                               stats_ctx->stat_put_start,
1056                               stats_ctx->stat_find_peer_reply,
1057                               stats_ctx->stat_get_reply,
1058                               stats_ctx->stat_find_peer_answer,
1059                               stats_ctx->stat_get_response_start);
1060   GNUNET_free(stats_ctx);
1061   return GNUNET_YES;
1062 }
1063
1064 static void
1065 stats_finished(void *cls, int result)
1066 {
1067   fprintf (stderr, "Finished getting all peers statistics, iterating!\n");
1068   GNUNET_CONTAINER_multihashmap_iterate (stats_map, &stats_iterate, NULL);
1069   GNUNET_CONTAINER_multihashmap_destroy (stats_map);
1070   GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1071 }
1072
1073 /**
1074  * Callback function to process statistic values.
1075  *
1076  * @param cls closure
1077  * @param peer the peer the statistics belong to
1078  * @param subsystem name of subsystem that created the statistic
1079  * @param name the name of the datum
1080  * @param value the current value
1081  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1082  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1083  */
1084 static int
1085 stats_handle(void *cls, const struct GNUNET_PeerIdentity *peer,
1086     const char *subsystem, const char *name, uint64_t value, int is_persistent)
1087 {
1088   struct StatisticsIteratorContext *stats_ctx;
1089
1090   if (dhtlog_handle != NULL)
1091     dhtlog_handle->add_generic_stat (peer, name, subsystem, value);
1092   if (GNUNET_CONTAINER_multihashmap_contains (stats_map, &peer->hashPubKey))
1093     {
1094       stats_ctx = GNUNET_CONTAINER_multihashmap_get (stats_map,
1095                                                      &peer->hashPubKey);
1096     }
1097   else
1098     {
1099       stats_ctx = GNUNET_malloc(sizeof(struct StatisticsIteratorContext));
1100       stats_ctx->peer = peer;
1101       GNUNET_CONTAINER_multihashmap_put (stats_map, &peer->hashPubKey,
1102                                          stats_ctx,
1103                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1104     }
1105   GNUNET_assert(stats_ctx != NULL);
1106
1107   if (strcmp (name, STAT_ROUTES) == 0)
1108     stats_ctx->stat_routes = value;
1109   else if (strcmp (name, STAT_ROUTE_FORWARDS) == 0)
1110     stats_ctx->stat_route_forwards = value;
1111   else if (strcmp (name, STAT_RESULTS) == 0)
1112     stats_ctx->stat_results = value;
1113   else if (strcmp (name, STAT_RESULTS_TO_CLIENT) == 0)
1114     stats_ctx->stat_results_to_client = value;
1115   else if (strcmp (name, STAT_RESULT_FORWARDS) == 0)
1116     stats_ctx->stat_result_forwards = value;
1117   else if (strcmp (name, STAT_GETS) == 0)
1118     stats_ctx->stat_gets = value;
1119   else if (strcmp (name, STAT_PUTS) == 0)
1120     stats_ctx->stat_puts = value;
1121   else if (strcmp (name, STAT_PUTS_INSERTED) == 0)
1122     stats_ctx->stat_puts_inserted = value;
1123   else if (strcmp (name, STAT_FIND_PEER) == 0)
1124     stats_ctx->stat_find_peer = value;
1125   else if (strcmp (name, STAT_FIND_PEER_START) == 0)
1126     stats_ctx->stat_find_peer_start = value;
1127   else if (strcmp (name, STAT_GET_START) == 0)
1128     stats_ctx->stat_get_start = value;
1129   else if (strcmp (name, STAT_PUT_START) == 0)
1130     stats_ctx->stat_put_start = value;
1131   else if (strcmp (name, STAT_FIND_PEER_REPLY) == 0)
1132     stats_ctx->stat_find_peer_reply = value;
1133   else if (strcmp (name, STAT_GET_REPLY) == 0)
1134     stats_ctx->stat_get_reply = value;
1135   else if (strcmp (name, STAT_FIND_PEER_ANSWER) == 0)
1136     stats_ctx->stat_find_peer_answer = value;
1137   else if (strcmp (name, STAT_GET_RESPONSE_START) == 0)
1138     stats_ctx->stat_get_response_start = value;
1139
1140   return GNUNET_OK;
1141 }
1142
1143 /**
1144  * Connect to statistics service for each peer and get the appropriate
1145  * dht statistics for safe keeping.
1146  */
1147 static void
1148 log_dht_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1149 {
1150   stats_map = GNUNET_CONTAINER_multihashmap_create (num_peers);
1151   fprintf (stderr, "Starting statistics logging\n");
1152   GNUNET_TESTING_get_statistics (pg, &stats_finished, &stats_handle, NULL);
1153 }
1154
1155 /**
1156  * Connect to all peers in the peer group and iterate over their
1157  * connections.
1158  */
1159 static void
1160 capture_current_topology(void *cls,
1161     const struct GNUNET_SCHEDULER_TaskContext * tc)
1162 {
1163   struct TopologyIteratorContext *topo_ctx = cls;
1164   dhtlog_handle->insert_topology (0);
1165   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Called capture_current_topology\n");
1166   GNUNET_TESTING_get_topology (pg, &log_topology_cb, topo_ctx);
1167 }
1168
1169 /**
1170  * Check if the get_handle is being used, if so stop the request.  Either
1171  * way, schedule the end_badly_cont function which actually shuts down the
1172  * test.
1173  */
1174 static void
1175 end_badly(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1176 {
1177   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failing test with error: `%s'!\n",
1178               (char *) cls);
1179
1180   struct TestPutContext *test_put = all_puts;
1181   struct TestGetContext *test_get = all_gets;
1182
1183   while (test_put != NULL)
1184     {
1185       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1186         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
1187       if (test_put->dht_handle != NULL)
1188         GNUNET_DHT_disconnect (test_put->dht_handle);
1189       test_put = test_put->next;
1190     }
1191
1192   while (test_get != NULL)
1193     {
1194       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1195         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
1196       if (test_get->get_handle != NULL)
1197         GNUNET_DHT_get_stop (test_get->get_handle);
1198       if (test_get->dht_handle != NULL)
1199         GNUNET_DHT_disconnect (test_get->dht_handle);
1200       test_get = test_get->next;
1201     }
1202
1203   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
1204
1205   if (dhtlog_handle != NULL)
1206     {
1207       fprintf (stderr, "Update trial endtime\n");
1208       dhtlog_handle->update_trial (gets_completed);
1209       GNUNET_DHTLOG_disconnect (dhtlog_handle);
1210       dhtlog_handle = NULL;
1211     }
1212
1213   if (hostkey_meter != NULL)
1214     free_meter (hostkey_meter);
1215   if (peer_start_meter != NULL)
1216     free_meter (peer_start_meter);
1217   if (peer_connect_meter != NULL)
1218     free_meter (peer_connect_meter);
1219   if (put_meter != NULL)
1220     free_meter (put_meter);
1221   if (get_meter != NULL)
1222     free_meter (get_meter);
1223
1224   ok = 1;
1225 }
1226
1227 /**
1228  * Forward declaration.
1229  */
1230 static void
1231 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1232
1233 /**
1234  * Forward declaration.
1235  */
1236 static void
1237 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1238
1239 /**
1240  * Iterator over hash map entries.
1241  *
1242  * @param cls closure
1243  * @param key current key code
1244  * @param value value in the hash map
1245  * @return GNUNET_YES if we should continue to
1246  *         iterate,
1247  *         GNUNET_NO if not.
1248  */
1249 static int
1250 remove_peer_count(void *cls, const GNUNET_HashCode * key, void *value)
1251 {
1252   struct PeerCount *peer_count = value;
1253   GNUNET_CONTAINER_heap_remove_node (peer_count->heap_node);
1254   GNUNET_free(peer_count);
1255
1256   return GNUNET_YES;
1257 }
1258
1259 /**
1260  * Connect to all peers in the peer group and iterate over their
1261  * connections.
1262  */
1263 static void
1264 count_new_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1265 {
1266   struct FindPeerContext *find_peer_context = cls;
1267   find_peer_context->previous_peers = find_peer_context->current_peers;
1268   find_peer_context->current_peers = 0;
1269   GNUNET_TESTING_get_topology (pg, find_peer_context->count_peers_cb,
1270                                find_peer_context);
1271 }
1272
1273 static void
1274 decrement_find_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1275 {
1276   struct TestFindPeer *test_find_peer = cls;
1277   GNUNET_assert(test_find_peer->find_peer_context->outstanding > 0);
1278   test_find_peer->find_peer_context->outstanding--;
1279   test_find_peer->find_peer_context->total--;
1280   if (0 == test_find_peer->find_peer_context->total)
1281     {
1282       GNUNET_SCHEDULER_add_now (&count_new_peers,
1283                                 test_find_peer->find_peer_context);
1284     }
1285   GNUNET_free(test_find_peer);
1286 }
1287
1288 /**
1289  * A find peer request has been sent to the server, now we will schedule a task
1290  * to wait the appropriate time to allow the request to go out and back.
1291  *
1292  * @param cls closure - a TestFindPeer struct
1293  * @param tc context the task is being called with
1294  */
1295 static void
1296 handle_find_peer_sent(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1297 {
1298   struct TestFindPeer *test_find_peer = cls;
1299
1300   GNUNET_DHT_disconnect (test_find_peer->dht_handle);
1301   GNUNET_SCHEDULER_add_delayed (
1302                                 GNUNET_TIME_relative_divide (find_peer_delay, 2),
1303                                 &decrement_find_peers, test_find_peer);
1304 }
1305
1306 static void
1307 send_find_peer_request(void *cls,
1308     const struct GNUNET_SCHEDULER_TaskContext * tc)
1309 {
1310   struct TestFindPeer *test_find_peer = cls;
1311
1312   if (test_find_peer->find_peer_context->outstanding
1313       > max_outstanding_find_peers)
1314     {
1315       GNUNET_SCHEDULER_add_delayed (find_peer_offset, &send_find_peer_request,
1316                                     test_find_peer);
1317       return;
1318     }
1319
1320   test_find_peer->find_peer_context->outstanding++;
1321   if (GNUNET_TIME_absolute_get_remaining (
1322                                           test_find_peer->find_peer_context->endtime).rel_value
1323       == 0)
1324     {
1325       GNUNET_SCHEDULER_add_now (&decrement_find_peers, test_find_peer);
1326       return;
1327     }
1328
1329   test_find_peer->dht_handle = GNUNET_DHT_connect (test_find_peer->daemon->cfg,
1330                                                    1);
1331   GNUNET_assert(test_find_peer->dht_handle != NULL);
1332   GNUNET_DHT_find_peers (test_find_peer->dht_handle, &handle_find_peer_sent,
1333                          test_find_peer);
1334 }
1335
1336 /**
1337  * Add a connection to the find_peer_context given.  This may
1338  * be complete overkill, but allows us to choose the peers with
1339  * the least connections to initiate find peer requests from.
1340  */
1341 static void
1342 add_new_connection(struct FindPeerContext *find_peer_context,
1343     const struct GNUNET_PeerIdentity *first,
1344     const struct GNUNET_PeerIdentity *second)
1345 {
1346   struct PeerCount *first_count;
1347   struct PeerCount *second_count;
1348
1349   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1350                                               &first->hashPubKey))
1351     {
1352       first_count
1353           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1354                                                &first->hashPubKey);
1355       GNUNET_assert(first_count != NULL);
1356       first_count->count++;
1357       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1358                                          first_count->heap_node,
1359                                          first_count->count);
1360     }
1361   else
1362     {
1363       first_count = GNUNET_malloc(sizeof(struct PeerCount));
1364       first_count->count = 1;
1365       memcpy (&first_count->peer_id, first, sizeof(struct GNUNET_PeerIdentity));
1366       first_count->heap_node
1367           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1368                                           first_count, first_count->count);
1369       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1370                                          &first->hashPubKey, first_count,
1371                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1372     }
1373
1374   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1375                                               &second->hashPubKey))
1376     {
1377       second_count
1378           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1379                                                &second->hashPubKey);
1380       GNUNET_assert(second_count != NULL);
1381       second_count->count++;
1382       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1383                                          second_count->heap_node,
1384                                          second_count->count);
1385     }
1386   else
1387     {
1388       second_count = GNUNET_malloc(sizeof(struct PeerCount));
1389       second_count->count = 1;
1390       memcpy (&second_count->peer_id, second,
1391               sizeof(struct GNUNET_PeerIdentity));
1392       second_count->heap_node
1393           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1394                                           second_count, second_count->count);
1395       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1396                                          &second->hashPubKey, second_count,
1397                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1398     }
1399 }
1400
1401 /**
1402  * Iterate over min heap of connections per peer.  For any
1403  * peer that has 0 connections, attempt to connect them to
1404  * some random peer.
1405  *
1406  * @param cls closure a struct FindPeerContext
1407  * @param node internal node of the heap
1408  * @param element value stored, a struct PeerCount
1409  * @param cost cost associated with the node
1410  * @return GNUNET_YES if we should continue to iterate,
1411  *         GNUNET_NO if not.
1412  */
1413 static int
1414 iterate_min_heap_peers(void *cls, struct GNUNET_CONTAINER_HeapNode *node,
1415     void *element, GNUNET_CONTAINER_HeapCostType cost)
1416 {
1417   struct FindPeerContext *find_peer_context = cls;
1418   struct PeerCount *peer_count = element;
1419   struct GNUNET_TESTING_Daemon *d1;
1420   struct GNUNET_TESTING_Daemon *d2;
1421   struct GNUNET_TIME_Relative timeout;
1422   if (cost == 0)
1423     {
1424       d1 = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1425       GNUNET_assert(d1 != NULL);
1426       d2 = d1;
1427       while ((d2 == d1) || (GNUNET_YES != GNUNET_TESTING_daemon_running (d2)))
1428         {
1429           d2
1430               = GNUNET_TESTING_daemon_get (
1431                                            pg,
1432                                            GNUNET_CRYPTO_random_u32 (
1433                                                                      GNUNET_CRYPTO_QUALITY_WEAK,
1434                                                                      num_peers));
1435           GNUNET_assert(d2 != NULL);
1436         }
1437
1438       /** Just try to connect the peers, don't worry about callbacks, etc. **/
1439       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1440                   "Peer %s has 0 connections.  Trying to connect to %s...\n",
1441                   GNUNET_i2s (&peer_count->peer_id), d2->shortname);
1442       timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
1443                                                DEFAULT_CONNECT_TIMEOUT);
1444       if (GNUNET_TIME_relative_to_absolute (timeout).abs_value
1445           > find_peer_context->endtime.abs_value)
1446         {
1447           timeout
1448               = GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime);
1449         }
1450       GNUNET_TESTING_daemons_connect (d1, d2, timeout,
1451                                       DEFAULT_RECONNECT_ATTEMPTS, GNUNET_YES,
1452                                       NULL, NULL);
1453     }
1454   if (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1455       > 0)
1456     return GNUNET_YES;
1457   else
1458     return GNUNET_NO;
1459 }
1460
1461 /**
1462  * Forward declaration.
1463  */
1464 static void
1465 schedule_churn_find_peer_requests(void *cls,
1466     const struct GNUNET_SCHEDULER_TaskContext * tc);
1467
1468 /**
1469  * Callback for iterating over all the peer connections of a peer group.
1470  * Used after we have churned on some peers to find which ones have zero
1471  * connections so we can make them issue find peer requests.
1472  */
1473 static void
1474 count_peers_churn_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1475     const struct GNUNET_PeerIdentity *second, const char *emsg)
1476 {
1477   struct FindPeerContext *find_peer_context = cls;
1478   struct TopologyIteratorContext *topo_ctx;
1479   struct PeerCount *peer_count;
1480
1481   if ((first != NULL) && (second != NULL))
1482     {
1483       add_new_connection (find_peer_context, first, second);
1484       find_peer_context->current_peers++;
1485     }
1486   else
1487     {
1488       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1489                   "Peer count finished (%u connections)\n",
1490                   find_peer_context->current_peers);
1491       peer_count
1492           = GNUNET_CONTAINER_heap_peek (find_peer_context->peer_min_heap);
1493       GNUNET_assert(peer_count != NULL);
1494       /* WAIT. When peers are churned they will come back with their peers (at least in peerinfo), because the HOSTS file doesn't likely get removed. CRAP. */
1495       /* NO they won't, because we have disabled peerinfo writing to disk (remember?) so we WILL have to give them new connections */
1496       /* Best course of action: have DHT automatically try to add peers from peerinfo on startup. This way IF peerinfo writes to file
1497        * then some peers will end up connected.
1498        *
1499        * Also, find any peers that have zero connections here and set up a task to choose at random another peer in the network to
1500        * connect to.  Of course, if they are blacklisted from that peer they won't be able to connect, so we will have to keep trying
1501        * until they get a peer.
1502        */
1503       /* However, they won't automatically be connected to any of their previous peers... How can we handle that? */
1504       /* So now we have choices: do we want them to come back with all their connections?  Probably not, but it solves this mess. */
1505
1506       /* Second problem, which is still a problem, is that a FIND_PEER request won't work when a peer has no connections */
1507
1508       /**
1509        * Okay, so here's how this *should* work now.
1510        *
1511        * 1. We check the min heap for any peers that have 0 connections.
1512        *    a. If any are found, we iterate over the heap and just randomly
1513        *       choose another peer and ask testing to please connect the two.
1514        *       This takes care of the case that a peer just randomly joins the
1515        *       network.  However, if there are strict topology restrictions
1516        *       (imagine a ring) choosing randomly most likely won't help.
1517        *       We make sure the connection attempt doesn't take longer than
1518        *       the total timeout, but don't care too much about the result.
1519        *    b. After that, we still schedule the find peer requests (concurrently
1520        *       with the connect attempts most likely).  This handles the case
1521        *       that the DHT iterates over peerinfo and just needs to try to send
1522        *       a message to get connected.  This should handle the case that the
1523        *       topology is very strict.
1524        *
1525        * 2. If all peers have > 0 connections, we still send find peer requests
1526        *    as long as possible (until timeout is reached) to help out those
1527        *    peers that were newly churned and need more connections.  This is because
1528        *    once all new peers have established a single connection, they won't be
1529        *    well connected.
1530        *
1531        * 3. Once we reach the timeout, we can do no more.  We must schedule the
1532        *    next iteration of get requests regardless of connections that peers
1533        *    may or may not have.
1534        *
1535        * Caveat: it would be nice to get peers to take data offline with them and
1536        *         come back with it (or not) based on the testing framework.  The
1537        *         same goes for remembering previous connections, but putting either
1538        *         into the general testing churn options seems like overkill because
1539        *         these are very specialized cases.
1540        */
1541       GNUNET_log (
1542                   GNUNET_ERROR_TYPE_WARNING,
1543                   "Out of %u peers, fewest connections is %d\n",
1544                   GNUNET_CONTAINER_heap_get_size (
1545                                                   find_peer_context->peer_min_heap),
1546                   peer_count->count);
1547       if ((peer_count->count == 0)
1548           && (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1549               > 0))
1550         {
1551           GNUNET_log (
1552                       GNUNET_ERROR_TYPE_WARNING,
1553                       "Found peer with no connections, will choose some peer(s) at random to connect to!\n");
1554           GNUNET_CONTAINER_heap_iterate (find_peer_context->peer_min_heap,
1555                                          &iterate_min_heap_peers,
1556                                          find_peer_context);
1557           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1558                                     find_peer_context);
1559         }
1560       else if ((GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1561           > 0) && (find_peer_context->last_sent != 0))
1562         {
1563           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1564                                     find_peer_context);
1565         }
1566       else
1567         {
1568           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
1569                                                  &remove_peer_count,
1570                                                  find_peer_context);
1571           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
1572           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
1573           GNUNET_free(find_peer_context);
1574           GNUNET_log (
1575                       GNUNET_ERROR_TYPE_WARNING,
1576                       "Churn round %u of %llu finished, scheduling next GET round.\n",
1577                       current_churn_round, churn_rounds);
1578           if (dhtlog_handle != NULL)
1579             {
1580               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1581               topo_ctx->cont = &do_get;
1582               topo_ctx->cls = all_gets;
1583               topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1584               topo_ctx->peers_seen
1585                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1586               die_task
1587                   = GNUNET_SCHEDULER_add_delayed (
1588                                                   GNUNET_TIME_relative_add (
1589                                                                             GNUNET_TIME_relative_add (
1590                                                                                                       DEFAULT_GET_TIMEOUT,
1591                                                                                                       all_get_timeout),
1592                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1593                                                   &end_badly,
1594                                                   "from do gets (count_peers_churn_cb)");
1595               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1596             }
1597           else
1598             {
1599               die_task
1600                   = GNUNET_SCHEDULER_add_delayed (
1601                                                   GNUNET_TIME_relative_add (
1602                                                                             GNUNET_TIME_relative_add (
1603                                                                                                       DEFAULT_GET_TIMEOUT,
1604                                                                                                       all_get_timeout),
1605                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1606                                                   &end_badly,
1607                                                   "from do gets (count_peers_churn_cb)");
1608               GNUNET_SCHEDULER_add_now (&do_get, all_gets);
1609             }
1610         }
1611     }
1612 }
1613
1614 /**
1615  * Set up a single find peer request for each peer in the topology.  Do this
1616  * until the settle time is over, limited by the number of outstanding requests
1617  * and the time allowed for each one!
1618  */
1619 static void
1620 schedule_churn_find_peer_requests(void *cls,
1621     const struct GNUNET_SCHEDULER_TaskContext * tc)
1622 {
1623   struct FindPeerContext *find_peer_ctx = cls;
1624   struct TestFindPeer *test_find_peer;
1625   struct PeerCount *peer_count;
1626   uint32_t i;
1627
1628   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
1629     find_peer_ctx->total = 1;
1630   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1631       < MIN_FIND_PEER_CUTOFF)
1632     find_peer_ctx->total = find_peer_ctx->total / 2;
1633   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1634       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
1635     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
1636         / 4);
1637   else
1638     find_peer_ctx->total = find_peer_ctx->last_sent * 4;
1639
1640   if (find_peer_ctx->total > max_outstanding_find_peers)
1641     find_peer_ctx->total = max_outstanding_find_peers;
1642
1643   find_peer_ctx->last_sent = find_peer_ctx->total;
1644   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1645               "Sending %u find peer messages (after churn)\n",
1646               find_peer_ctx->total);
1647
1648   if (find_peer_ctx->total > 0)
1649     find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
1650                                                     find_peer_ctx->total);
1651   else
1652     {
1653       find_peer_ctx->previous_peers = find_peer_ctx->current_peers;
1654       find_peer_ctx->current_peers = 0;
1655       GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_ctx);
1656     }
1657
1658   for (i = 0; i < find_peer_ctx->total; i++)
1659     {
1660       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
1661       /* If we have sent requests, choose peers with a low number of connections to send requests from */
1662       peer_count
1663           = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
1664       GNUNET_assert(peer_count != NULL);
1665       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1666                   "Sending find peer request from peer with %u connections\n",
1667                   peer_count->count);
1668       GNUNET_CONTAINER_multihashmap_remove (find_peer_ctx->peer_hash,
1669                                             &peer_count->peer_id.hashPubKey,
1670                                             peer_count);
1671       test_find_peer->daemon
1672           = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1673       GNUNET_assert(test_find_peer->daemon != NULL);
1674       test_find_peer->find_peer_context = find_peer_ctx;
1675       GNUNET_SCHEDULER_add_delayed (
1676                                     GNUNET_TIME_relative_multiply (
1677                                                                    find_peer_offset,
1678                                                                    i),
1679                                     &send_find_peer_request, test_find_peer);
1680     }
1681
1682   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
1683       == NULL))
1684     {
1685       find_peer_ctx->peer_hash
1686           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1687       find_peer_ctx->peer_min_heap
1688           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1689     }
1690   else
1691     {
1692       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
1693                                              &remove_peer_count, find_peer_ctx);
1694       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
1695       find_peer_ctx->peer_hash
1696           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1697     }
1698
1699   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
1700   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
1701 }
1702
1703 static void
1704 schedule_churn_get_topology(void *cls,
1705     const struct GNUNET_SCHEDULER_TaskContext * tc)
1706 {
1707   struct FindPeerContext *find_peer_context = cls;
1708   GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_context);
1709 }
1710
1711 /**
1712  * Called when churning of the topology has finished.
1713  *
1714  * @param cls closure unused
1715  * @param emsg NULL on success, or a printable error on failure
1716  */
1717 static void
1718 churn_complete(void *cls, const char *emsg)
1719 {
1720   struct FindPeerContext *find_peer_context = cls;
1721   struct PeerCount *peer_count;
1722   unsigned int i;
1723   struct GNUNET_TESTING_Daemon *temp_daemon;
1724   struct TopologyIteratorContext *topo_ctx;
1725   struct GNUNET_TIME_Relative calc_timeout;
1726   int count_added;
1727
1728   if (emsg != NULL)
1729     {
1730       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1731                   "Ending test, churning of peers failed with error `%s'", emsg);
1732       GNUNET_SCHEDULER_add_now (&end_badly, (void *) emsg);
1733       return;
1734     }
1735
1736   /**
1737    * If we switched any peers on, we have to somehow force connect the new peer to
1738    * SOME bootstrap peer in the network.  First schedule a task to find all peers
1739    * with no connections, then choose a random peer for each and connect them.
1740    */
1741   if (find_peer_context != NULL)
1742     {
1743       GNUNET_log (
1744                   GNUNET_ERROR_TYPE_WARNING,
1745                   "We have churned on some peers, so we must schedule find peer requests for them!\n");
1746       count_added = 0;
1747       for (i = 0; i < num_peers; i++)
1748         {
1749           temp_daemon = GNUNET_TESTING_daemon_get (pg, i);
1750           if (GNUNET_YES == GNUNET_TESTING_daemon_running (temp_daemon))
1751             {
1752               peer_count = GNUNET_malloc (sizeof(struct PeerCount));
1753               memcpy (&peer_count->peer_id, &temp_daemon->id,
1754                       sizeof(struct GNUNET_PeerIdentity));
1755               GNUNET_assert(peer_count->count == 0);
1756               peer_count->heap_node
1757                   = GNUNET_CONTAINER_heap_insert (
1758                                                   find_peer_context->peer_min_heap,
1759                                                   peer_count, peer_count->count);
1760               GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1761                                                  &temp_daemon->id.hashPubKey,
1762                                                  peer_count,
1763                                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1764               count_added++;
1765             }
1766         }
1767       GNUNET_log (
1768                   GNUNET_ERROR_TYPE_WARNING,
1769                   "Added %d peers to heap, total size %d\n",
1770                   count_added,
1771                   GNUNET_CONTAINER_heap_get_size (
1772                                                   find_peer_context->peer_min_heap));
1773       GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1774                                     &schedule_churn_get_topology,
1775                                     find_peer_context);
1776     }
1777   else
1778     {
1779       GNUNET_log (
1780                   GNUNET_ERROR_TYPE_WARNING,
1781                   "Only churned off peers, no find peer requests, scheduling more gets (after allowing time for peers to disconnect properly!)...\n");
1782       if (dhtlog_handle != NULL)
1783         {
1784           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1785           topo_ctx->cont = &do_get;
1786           topo_ctx->cls = all_gets;
1787           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1788           topo_ctx->peers_seen
1789               = GNUNET_CONTAINER_multihashmap_create (num_peers);
1790           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1791                                                    all_get_timeout);
1792           calc_timeout
1793               = GNUNET_TIME_relative_add (calc_timeout,
1794                                           DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT);
1795           calc_timeout
1796               = GNUNET_TIME_relative_add (calc_timeout,
1797                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1798           die_task
1799               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1800                                               "from do gets (churn_complete)");
1801           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1802                                         &capture_current_topology, topo_ctx);
1803           dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1804         }
1805       else
1806         {
1807           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1808                                                    all_get_timeout);
1809           calc_timeout
1810               = GNUNET_TIME_relative_add (calc_timeout,
1811                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1812           die_task
1813               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1814                                               "from do gets (churn_complete)");
1815           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1816                                         &do_get, all_gets);
1817         }
1818     }
1819 }
1820
1821 /**
1822  * Decide how many peers to turn on or off in this round, make sure the
1823  * numbers actually make sense, then do so.  This function sets in motion
1824  * churn, find peer requests for newly joined peers, and issuing get
1825  * requests once the new peers have done so.
1826  *
1827  * @param cls closure (unused)
1828  * @param tc task context (unused)
1829  */
1830 static void
1831 churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1832 {
1833   unsigned int count_running;
1834   unsigned int churn_up;
1835   unsigned int churn_down;
1836   struct GNUNET_TIME_Relative timeout;
1837   struct FindPeerContext *find_peer_context;
1838
1839   churn_up = churn_down = 0;
1840   count_running = GNUNET_TESTING_daemons_running (pg);
1841   if (count_running > churn_array[current_churn_round])
1842     churn_down = count_running - churn_array[current_churn_round];
1843   else if (count_running < churn_array[current_churn_round])
1844     churn_up = churn_array[current_churn_round] - count_running;
1845   else
1846     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1847                 "Not churning any peers, topology unchanged.\n");
1848
1849   if (churn_up > num_peers - count_running)
1850     {
1851       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1852                   "Churn file specified %u peers (up); only have %u!",
1853                   churn_array[current_churn_round], num_peers);
1854       churn_up = num_peers - count_running;
1855     }
1856   else if (churn_down > count_running)
1857     {
1858       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1859                   "Churn file specified %u peers (down); only have %u!",
1860                   churn_array[current_churn_round], count_running);
1861       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1862                   "This will leave NO peers running (mistake in churn configuration?)!");
1863       churn_down = count_running;
1864     }
1865   //timeout = GNUNET_TIME_relative_multiply(seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1866   //timeout = GNUNET_TIME_relative_multiply (seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1867   timeout = GNUNET_TIME_relative_multiply (DEFAULT_TIMEOUT, 2); /* FIXME: Lack of intelligent choice here */
1868   find_peer_context = NULL;
1869   if (churn_up > 0) /* Only need to do find peer requests if we turned new peers on */
1870     {
1871       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
1872       find_peer_context->count_peers_cb = &count_peers_churn_cb;
1873       find_peer_context->previous_peers = 0;
1874       find_peer_context->current_peers = 0;
1875       find_peer_context->endtime = GNUNET_TIME_relative_to_absolute (timeout);
1876       find_peer_context->peer_hash
1877           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1878       find_peer_context->peer_min_heap
1879           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1880     }
1881   GNUNET_log (
1882               GNUNET_ERROR_TYPE_WARNING,
1883               "churn_peers: want %u total, %u running, starting %u, stopping %u\n",
1884               churn_array[current_churn_round], count_running, churn_up,
1885               churn_down);
1886   GNUNET_TESTING_daemons_churn (pg, churn_down, churn_up, timeout,
1887                                 &churn_complete, find_peer_context);
1888   current_churn_round++;
1889 }
1890
1891 /**
1892  * Task to release DHT handle associated with GET request.
1893  */
1894 static void
1895 get_stop_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1896 {
1897   struct TestGetContext *test_get = cls;
1898   struct TopologyIteratorContext *topo_ctx;
1899
1900   /* The dht_handle may be null if this get was scheduled from a down peer */
1901   if (test_get->dht_handle != NULL)
1902     {
1903       GNUNET_DHT_disconnect (test_get->dht_handle);
1904       outstanding_gets--; /* GET is really finished */
1905       test_get->dht_handle = NULL;
1906     }
1907
1908   /* Reset the uid (which item to search for) and the daemon (which peer to search from) for later get request iterations */
1909   if (get_from_same == GNUNET_NO)
1910     {
1911       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1912                                                 num_puts);
1913       test_get->daemon
1914           = GNUNET_TESTING_daemon_get (
1915                                        pg,
1916                                        GNUNET_CRYPTO_random_u32 (
1917                                                                  GNUNET_CRYPTO_QUALITY_WEAK,
1918                                                                  num_peers));
1919     }
1920
1921 #if VERBOSE > 1
1922   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
1923 #endif
1924   update_meter (get_meter);
1925   if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0))
1926     {
1927       fprintf (
1928                stderr,
1929                "Canceling die task (get_stop_finished) %llu gets completed, %llu gets failed\n",
1930                gets_completed, gets_failed);
1931       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
1932         dhtlog_handle->insert_round_details (DHT_ROUND_GET, rounds_finished,
1933                                              num_gets, gets_completed);
1934       GNUNET_SCHEDULER_cancel (die_task);
1935       reset_meter (put_meter);
1936       reset_meter (get_meter);
1937       /**
1938        *  Handle all cases:
1939        *    1) Testing is completely finished, call the topology iteration dealy and die
1940        *    2) Testing is not finished, churn the network and do gets again (current_churn_round < churn_rounds)
1941        *    3) Testing is not finished, reschedule all the PUTS *and* GETS again (num_rounds > 1)
1942        */
1943       if (rounds_finished == total_rounds - 1) /* Everything is finished, end testing */
1944         {
1945           if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
1946             {
1947               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1948               topo_ctx->cont = &log_dht_statistics;
1949               topo_ctx->peers_seen
1950                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1951               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1952             }
1953           else
1954             GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1955         }
1956       else if (current_churn_round < churns_per_round * (rounds_finished + 1)) /* Do next round of churn */
1957         {
1958           GNUNET_log (
1959                       GNUNET_ERROR_TYPE_WARNING,
1960                       "Current churn round %u, real round %u, scheduling next round of churn.\n",
1961                       current_churn_round, rounds_finished + 1);
1962           gets_completed = 0;
1963           gets_failed = 0;
1964
1965           if (dhtlog_handle != NULL)
1966             dhtlog_handle->insert_round (DHT_ROUND_CHURN, rounds_finished);
1967
1968           GNUNET_SCHEDULER_add_now (&churn_peers, NULL);
1969         }
1970       else if (rounds_finished < total_rounds - 1) /* Start a new complete round */
1971         {
1972           rounds_finished++;
1973           gets_completed = 0;
1974           gets_failed = 0;
1975           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1976                       "Round %u of %llu finished, scheduling next round.\n",
1977                       rounds_finished, total_rounds);
1978
1979           /** We reset the peer daemon for puts and gets on each disconnect, so all we need to do is start another round! */
1980           if (GNUNET_YES == in_dht_replication) /* Replication done in DHT, don't redo puts! */
1981             {
1982               if (dhtlog_handle != NULL)
1983                 dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1984
1985               die_task
1986                   = GNUNET_SCHEDULER_add_delayed (
1987                                                   GNUNET_TIME_relative_add (
1988                                                                             GNUNET_TIME_relative_add (
1989                                                                                                       GNUNET_TIME_relative_multiply (
1990                                                                                                                                      GNUNET_TIME_UNIT_SECONDS,
1991                                                                                                                                      round_delay),
1992                                                                                                       all_get_timeout),
1993                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1994                                                   &end_badly,
1995                                                   "from do gets (next round)");
1996               GNUNET_SCHEDULER_add_delayed (
1997                                             GNUNET_TIME_relative_multiply (
1998                                                                            GNUNET_TIME_UNIT_SECONDS,
1999                                                                            round_delay),
2000                                             &do_get, all_gets);
2001             }
2002           else
2003             {
2004               if (dhtlog_handle != NULL)
2005                 dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2006               die_task
2007                   = GNUNET_SCHEDULER_add_delayed (
2008                                                   GNUNET_TIME_relative_add (
2009                                                                             GNUNET_TIME_relative_multiply (
2010                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2011                                                                                                            round_delay),
2012                                                                             GNUNET_TIME_relative_multiply (
2013                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2014                                                                                                            num_puts
2015                                                                                                                * 2)),
2016                                                   &end_badly, "from do puts");
2017               GNUNET_SCHEDULER_add_delayed (
2018                                             GNUNET_TIME_relative_multiply (
2019                                                                            GNUNET_TIME_UNIT_SECONDS,
2020                                                                            round_delay),
2021                                             &do_put, all_puts);
2022             }
2023         }
2024     }
2025 }
2026
2027 /**
2028  * Task to release get handle.
2029  */
2030 static void
2031 get_stop_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2032 {
2033   struct TestGetContext *test_get = cls;
2034
2035   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2036     gets_failed++;
2037   else
2038     cumulative_successful_gets++;
2039
2040   GNUNET_assert(test_get->get_handle != NULL);
2041   GNUNET_DHT_get_stop (test_get->get_handle);
2042   test_get->get_handle = NULL;
2043   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2044   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2045 }
2046
2047 /**
2048  * Iterator called if the GET request initiated returns a response.
2049  *
2050  * @param cls closure
2051  * @param exp when will this value expire
2052  * @param key key of the result
2053  * @param get_path NULL-terminated array of pointers
2054  *                 to the peers on reverse GET path (or NULL if not recorded)
2055  * @param put_path NULL-terminated array of pointers
2056  *                 to the peers on the PUT path (or NULL if not recorded)
2057  * @param type type of the result
2058  * @param size number of bytes in data
2059  * @param data pointer to the result data
2060  */
2061 static void
2062 get_result_iterator(void *cls, struct GNUNET_TIME_Absolute exp,
2063     const GNUNET_HashCode * key,
2064     const struct GNUNET_PeerIdentity * const *get_path,
2065     const struct GNUNET_PeerIdentity * const *put_path,
2066     enum GNUNET_BLOCK_Type type, size_t size, const void *data)
2067 {
2068   struct TestGetContext *test_get = cls;
2069
2070   if (test_get->succeeded == GNUNET_YES)
2071     return; /* Get has already been successful, probably ending now */
2072
2073   if (0 != memcmp (&known_keys[test_get->uid], key, sizeof(GNUNET_HashCode))) /* || (0 != memcmp(original_data, data, sizeof(original_data))))*/
2074     {
2075       gets_completed++;
2076       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2077                   "Key or data is not the same as was inserted!\n");
2078     }
2079   else
2080     {
2081       gets_completed++;
2082       test_get->succeeded = GNUNET_YES;
2083     }
2084 #if VERBOSE > 1
2085   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
2086 #endif
2087   GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
2088   GNUNET_SCHEDULER_add_continuation (&get_stop_task, test_get,
2089                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
2090 }
2091
2092 /**
2093  * Set up some data, and call API PUT function
2094  */
2095 static void
2096 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2097 {
2098   struct TestGetContext *test_get = cls;
2099
2100   if (num_gets == 0)
2101     {
2102       GNUNET_SCHEDULER_cancel (die_task);
2103       GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
2104     }
2105
2106   if (test_get == NULL)
2107     return; /* End of the list */
2108
2109   /* Set this here in case we are re-running gets */
2110   test_get->succeeded = GNUNET_NO;
2111
2112   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_get->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2113     {
2114       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2115                   "Peer we should issue get request from is down, skipping.\n");
2116       gets_failed++;
2117       GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2118       GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2119       return;
2120     }
2121
2122   /* Check if more gets are outstanding than should be */
2123   if (outstanding_gets > max_outstanding_gets)
2124     {
2125       GNUNET_SCHEDULER_add_delayed (
2126                                     GNUNET_TIME_relative_multiply (
2127                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2128                                                                    200),
2129                                     &do_get, test_get);
2130       return;
2131     }
2132
2133   /* Connect to the first peer's DHT */
2134   test_get->dht_handle = GNUNET_DHT_connect (test_get->daemon->cfg, 10);
2135   GNUNET_assert(test_get->dht_handle != NULL);
2136   outstanding_gets++;
2137
2138   cumulative_num_gets++;
2139   /* Insert the data at the first peer */
2140   test_get->get_handle = GNUNET_DHT_get_start (test_get->dht_handle, get_delay,
2141                                                GNUNET_BLOCK_TYPE_TEST,
2142                                                &known_keys[test_get->uid],
2143                                                get_replication,
2144                                                GNUNET_DHT_RO_NONE, NULL, 0,
2145                                                NULL, 0, &get_result_iterator,
2146                                                test_get);
2147
2148 #if VERBOSE > 1
2149   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
2150       test_get->uid,
2151       test_get->daemon->shortname);
2152 #endif
2153   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed (get_timeout,
2154                                                             &get_stop_task,
2155                                                             test_get);
2156
2157   /* Schedule the next request in the linked list of get requests */
2158   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2159 }
2160
2161 /**
2162  * Called when the PUT request has been transmitted to the DHT service.
2163  * Schedule the GET request for some time in the future.
2164  */
2165 static void
2166 put_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2167 {
2168   struct TestPutContext *test_put = cls;
2169   struct TopologyIteratorContext *topo_ctx;
2170   outstanding_puts--;
2171   puts_completed++;
2172
2173   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
2174     fprintf (stderr, "PUT Request failed!\n");
2175
2176   /* Reset the daemon (which peer to insert at) for later put request iterations */
2177   if (replicate_same == GNUNET_NO)
2178     test_put->daemon
2179         = GNUNET_TESTING_daemon_get (
2180                                      pg,
2181                                      GNUNET_CRYPTO_random_u32 (
2182                                                                GNUNET_CRYPTO_QUALITY_WEAK,
2183                                                                num_peers));
2184
2185   GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
2186   test_put->disconnect_task = GNUNET_SCHEDULER_add_now (&put_disconnect_task,
2187                                                         test_put);
2188   if (GNUNET_YES == update_meter (put_meter))
2189     {
2190       GNUNET_assert(outstanding_puts == 0);
2191       GNUNET_SCHEDULER_cancel (die_task);
2192       if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2193         {
2194           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2195           topo_ctx->cont = &do_get;
2196           topo_ctx->cls = all_gets;
2197           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
2198           topo_ctx->peers_seen
2199               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2200           die_task
2201               = GNUNET_SCHEDULER_add_delayed (
2202                                               GNUNET_TIME_relative_add (
2203                                                                         GNUNET_TIME_relative_add (
2204                                                                                                   DEFAULT_GET_TIMEOUT,
2205                                                                                                   all_get_timeout),
2206                                                                         DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2207                                               &end_badly,
2208                                               "from do gets (put finished)");
2209           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
2210         }
2211       else
2212         {
2213           fprintf (stderr, "Scheduling die task (put finished)\n");
2214           die_task
2215               = GNUNET_SCHEDULER_add_delayed (
2216                                               GNUNET_TIME_relative_add (
2217                                                                         DEFAULT_GET_TIMEOUT,
2218                                                                         all_get_timeout),
2219                                               &end_badly,
2220                                               "from do gets (put finished)");
2221           GNUNET_SCHEDULER_add_delayed (DEFAULT_GET_TIMEOUT, &do_get, all_gets);
2222         }
2223       return;
2224     }
2225 }
2226
2227 /**
2228  * Set up some data, and call API PUT function
2229  */
2230 static void
2231 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2232 {
2233   struct TestPutContext *test_put = cls;
2234   char data[test_data_size]; /* Made up data to store */
2235   uint32_t rand;
2236   int i;
2237
2238   if (test_put == NULL)
2239     return; /* End of list */
2240
2241   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_put->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2242     {
2243       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2244                   "Peer we should issue put request at is down, skipping.\n");
2245       update_meter (put_meter);
2246       GNUNET_SCHEDULER_add_now (&do_put, test_put->next);
2247       return;
2248     }
2249
2250   for (i = 0; i < sizeof(data); i++)
2251     {
2252       memset (&data[i], GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2253                                                   UINT32_MAX), 1);
2254     }
2255
2256   if (outstanding_puts > max_outstanding_puts)
2257     {
2258       GNUNET_SCHEDULER_add_delayed (
2259                                     GNUNET_TIME_relative_multiply (
2260                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2261                                                                    200),
2262                                     &do_put, test_put);
2263       return;
2264     }
2265
2266 #if VERBOSE > 1
2267   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
2268       test_put->uid,
2269       test_put->daemon->shortname);
2270 #endif
2271   test_put->dht_handle = GNUNET_DHT_connect (test_put->daemon->cfg, 10);
2272
2273   GNUNET_assert(test_put->dht_handle != NULL);
2274   outstanding_puts++;
2275   GNUNET_DHT_put (test_put->dht_handle, &known_keys[test_put->uid],
2276                   put_replication, GNUNET_DHT_RO_NONE, GNUNET_BLOCK_TYPE_TEST,
2277                   sizeof(data), data, GNUNET_TIME_UNIT_FOREVER_ABS, put_delay,
2278                   &put_finished, test_put);
2279   test_put->disconnect_task
2280       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever (),
2281                                       &put_disconnect_task, test_put);
2282   rand = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2);
2283   GNUNET_SCHEDULER_add_delayed (
2284                                 GNUNET_TIME_relative_multiply (
2285                                                                GNUNET_TIME_UNIT_SECONDS,
2286                                                                rand), &do_put,
2287                                 test_put->next);
2288 }
2289
2290 static void
2291 schedule_find_peer_requests(void *cls,
2292     const struct GNUNET_SCHEDULER_TaskContext * tc);
2293
2294 static void
2295     setup_malicious_peers(void *cls,
2296         const struct GNUNET_SCHEDULER_TaskContext * tc);
2297
2298 /**
2299  * Given a number of total peers and a bucket size, estimate the number of
2300  * connections in a perfect kademlia topology.
2301  */
2302 static unsigned int
2303 connection_estimate(unsigned int peer_count, unsigned int bucket_size)
2304 {
2305   unsigned int i;
2306   unsigned int filled;
2307   i = num_peers;
2308
2309   filled = 0;
2310   while (i >= bucket_size)
2311     {
2312       filled++;
2313       i = i / 2;
2314     }
2315   filled++; /* Add one filled bucket to account for one "half full" and some miscellaneous */
2316   return filled * bucket_size * peer_count;
2317
2318 }
2319
2320 /**
2321  * Callback for iterating over all the peer connections of a peer group.
2322  */
2323 static void
2324 count_peers_cb(void *cls, const struct GNUNET_PeerIdentity *first,
2325     const struct GNUNET_PeerIdentity *second, const char *emsg)
2326 {
2327   struct FindPeerContext *find_peer_context = cls;
2328   if ((first != NULL) && (second != NULL))
2329     {
2330       add_new_connection (find_peer_context, first, second);
2331       find_peer_context->current_peers++;
2332     }
2333   else
2334     {
2335       GNUNET_log (
2336                   GNUNET_ERROR_TYPE_WARNING,
2337                   "Peer count finished (%u connections), %u new peers, connection estimate %u (target %u)\n",
2338                   find_peer_context->current_peers,
2339                   find_peer_context->current_peers
2340                       - find_peer_context->previous_peers,
2341                   connection_estimate (num_peers, DEFAULT_BUCKET_SIZE),
2342                   target_total_connections);
2343
2344       if ((find_peer_context->last_sent < 8)
2345           || ((find_peer_context->current_peers < 2
2346               * connection_estimate (num_peers, DEFAULT_BUCKET_SIZE))
2347               && (GNUNET_TIME_absolute_get_remaining (
2348                                                       find_peer_context->endtime).rel_value
2349                   > 0) && (find_peer_context->current_peers
2350               < target_total_connections)))
2351         {
2352           GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests,
2353                                     find_peer_context);
2354         }
2355       else
2356         {
2357           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
2358                                                  &remove_peer_count,
2359                                                  find_peer_context);
2360           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
2361           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
2362           GNUNET_free(find_peer_context);
2363           fprintf (stderr, "Not sending any more find peer requests.\n");
2364
2365 #if HAVE_MALICIOUS
2366           if (GNUNET_YES == malicious_after_settle)
2367             {
2368               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "calling setup_malicious_peers\n");
2369               GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2370             }
2371 #endif
2372         }
2373     }
2374 }
2375
2376 /**
2377  * Set up a single find peer request for each peer in the topology.  Do this
2378  * until the settle time is over, limited by the number of outstanding requests
2379  * and the time allowed for each one!
2380  */
2381 static void
2382 schedule_find_peer_requests(void *cls,
2383     const struct GNUNET_SCHEDULER_TaskContext * tc)
2384 {
2385   struct FindPeerContext *find_peer_ctx = cls;
2386   struct TestFindPeer *test_find_peer;
2387   struct PeerCount *peer_count;
2388   uint32_t i;
2389   uint32_t random;
2390
2391   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
2392     find_peer_ctx->total = 1;
2393   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
2394       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
2395     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
2396         / 8);
2397   else
2398     find_peer_ctx->total = find_peer_ctx->last_sent * 2;
2399
2400   if (find_peer_ctx->total > max_outstanding_find_peers)
2401     find_peer_ctx->total = max_outstanding_find_peers;
2402
2403   if (find_peer_ctx->total > num_peers) /* Don't try to send more messages than we have peers! */
2404     find_peer_ctx->total = num_peers;
2405
2406   find_peer_ctx->last_sent = find_peer_ctx->total;
2407   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2408               "Sending %u find peer messages (goal at least %u connections)\n",
2409               find_peer_ctx->total, target_total_connections);
2410
2411   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
2412                                                   find_peer_ctx->total);
2413   for (i = 0; i < find_peer_ctx->total; i++)
2414     {
2415       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
2416       /* If we haven't sent any requests yet, choose random peers */
2417       /* Also choose random in _half_ of all cases, so we don't
2418        * get stuck choosing topologically restricted peers with
2419        * few connections that will never be able to find any new
2420        * peers! */
2421       if ((find_peer_ctx->previous_peers == 0) || (i % 2 == 0))
2422         {
2423           /**
2424            * Attempt to spread find peer requests across even sections of the peer address
2425            * space.  Choose basically 1 peer in every num_peers / max_outstanding_requests
2426            * each time, then offset it by a randomish value.
2427            *
2428            * For instance, if num_peers is 100 and max_outstanding is 10, first chosen peer
2429            * will be between 0 - 10, second between 10 - 20, etc.
2430            */
2431           random = (num_peers / find_peer_ctx->total) * i;
2432           random = random
2433               + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2434                                           (num_peers / find_peer_ctx->total));
2435           if (random >= num_peers)
2436             {
2437               random = random - num_peers;
2438             }
2439 #if REAL_RANDOM
2440           random = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2441 #endif
2442           test_find_peer->daemon = GNUNET_TESTING_daemon_get (pg, random);
2443         }
2444       else /* If we have sent requests, choose peers with a low number of connections to send requests from */
2445         {
2446           peer_count
2447               = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
2448           GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(find_peer_ctx->peer_hash, &peer_count->peer_id.hashPubKey, peer_count));
2449           test_find_peer->daemon
2450               = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
2451           GNUNET_assert(test_find_peer->daemon != NULL);
2452         }
2453
2454       test_find_peer->find_peer_context = find_peer_ctx;
2455       GNUNET_SCHEDULER_add_delayed (
2456                                     GNUNET_TIME_relative_multiply (
2457                                                                    find_peer_offset,
2458                                                                    i),
2459                                     &send_find_peer_request, test_find_peer);
2460     }
2461
2462   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
2463       == NULL))
2464     {
2465       find_peer_ctx->peer_hash
2466           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2467       find_peer_ctx->peer_min_heap
2468           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2469     }
2470   else
2471     {
2472       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
2473                                              &remove_peer_count, find_peer_ctx);
2474       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
2475       find_peer_ctx->peer_hash
2476           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2477     }
2478
2479   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
2480   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
2481
2482 }
2483
2484 /**
2485  * Convert unique ID to hash code.
2486  *
2487  * @param uid unique ID to convert
2488  * @param hash set to uid (extended with zeros)
2489  */
2490 static void
2491 hash_from_uid(uint32_t uid, GNUNET_HashCode *hash)
2492 {
2493   memset (hash, 0, sizeof(GNUNET_HashCode));
2494   *((uint32_t *) hash) = uid;
2495 }
2496
2497 /**
2498  * Set up all of the put and get operations we want to do
2499  * in the current round.  Allocate data structure for each,
2500  * add to list, then schedule the actual PUT operations.
2501  */
2502 static void
2503 setup_puts_and_gets(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2504 {
2505   int i;
2506   struct TestPutContext *test_put;
2507   struct TestGetContext *test_get;
2508   uint32_t temp_peer;
2509   GNUNET_HashCode uid_hash;
2510   int count;
2511 #if REMEMBER
2512   int remember[num_puts][num_peers];
2513   memset(&remember, 0, sizeof(int) * num_puts * num_peers);
2514 #endif
2515   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "in setup_puts_and_gets\n");
2516   known_keys = GNUNET_malloc(sizeof(GNUNET_HashCode) * num_puts);
2517   for (i = 0; i < num_puts; i++)
2518     {
2519       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
2520       test_put->uid = i;
2521       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
2522                                         &known_keys[i]);
2523       /* Set first X bits to match the chosen sybil location if we want to do the sybil attack! */
2524       if (GNUNET_YES == malicious_sybil)
2525         {
2526           memcpy (&known_keys[i], &sybil_target, sizeof(GNUNET_HashCode) / 2);
2527           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2528                       "Distance between sybil location and key is %d\n",
2529                       GNUNET_CRYPTO_hash_matching_bits (&known_keys[i],
2530                                                         &sybil_target));
2531         }
2532       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2533                                             num_peers);
2534       test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2535       /* Don't start PUTs at malicious peers! */
2536       if (malicious_bloom != NULL)
2537         {
2538           count = 0;
2539           hash_from_uid (temp_peer, &uid_hash);
2540           while ((GNUNET_YES
2541               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2542               && (count < num_peers))
2543             {
2544               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2545                                                     num_peers);
2546               hash_from_uid (temp_peer, &uid_hash);
2547               test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2548               count++;
2549             }
2550           if (count == num_peers)
2551             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2552                         "Couldn't find peer not in malicious bloom to select!\n");
2553         }
2554
2555       test_put->next = all_puts;
2556       all_puts = test_put;
2557     }
2558
2559   for (i = 0; i < num_gets; i++)
2560     {
2561       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
2562       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2563                                                 num_puts);
2564 #if REMEMBER
2565       while (remember[test_get->uid][temp_daemon] == 1)
2566       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2567       remember[test_get->uid][temp_daemon] = 1;
2568 #endif
2569       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2570                                             num_peers);
2571       test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2572       /* Don't start GETs at malicious peers! */
2573       if (malicious_bloom != NULL)
2574         {
2575           hash_from_uid (temp_peer, &uid_hash);
2576           count = 0;
2577           while ((GNUNET_YES
2578               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2579               && (count < num_peers))
2580             {
2581               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2582                                                     num_peers);
2583               hash_from_uid (temp_peer, &uid_hash);
2584               test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2585               count++;
2586             }
2587           if (count == num_peers)
2588             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2589                         "Couldn't find peer not in malicious bloom to select!\n");
2590         }
2591       test_get->next = all_gets;
2592       all_gets = test_get;
2593     }
2594
2595   /*GNUNET_SCHEDULER_cancel (die_task);*/
2596   die_task
2597       = GNUNET_SCHEDULER_add_delayed (
2598                                       GNUNET_TIME_relative_multiply (
2599                                                                      GNUNET_TIME_UNIT_SECONDS,
2600                                                                      num_puts
2601                                                                          * 2),
2602                                       &end_badly, "from do puts");
2603   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
2604
2605 }
2606
2607 /**
2608  * Set up some all of the put and get operations we want
2609  * to do.  Allocate data structure for each, add to list,
2610  * then call actual insert functions.
2611  */
2612 static void
2613 continue_puts_and_gets(void *cls,
2614     const struct GNUNET_SCHEDULER_TaskContext * tc)
2615 {
2616   int i;
2617   int max;
2618   struct TopologyIteratorContext *topo_ctx;
2619   struct FindPeerContext *find_peer_context;
2620   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "In continue_puts_and_gets\n");
2621   if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2622     {
2623       if (settle_time >= 180 * 2)
2624         max = (settle_time / 180) - 2;
2625       else
2626         max = 1;
2627       for (i = 1; i < max; i++)
2628         {
2629           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2630           topo_ctx->current_iteration = i;
2631           topo_ctx->total_iterations = max;
2632           topo_ctx->peers_seen
2633               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2634           //fprintf(stderr, "scheduled topology iteration in %d minutes\n", i);
2635           GNUNET_SCHEDULER_add_delayed (
2636                                         GNUNET_TIME_relative_multiply (
2637                                                                        GNUNET_TIME_UNIT_MINUTES,
2638                                                                        i * 3),
2639                                         &capture_current_topology, topo_ctx);
2640         }
2641       topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2642       topo_ctx->cont = &setup_puts_and_gets;
2643       topo_ctx->peers_seen = GNUNET_CONTAINER_multihashmap_create (num_peers);
2644       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2645                   "setting setup_puts_and_gets for %d seconds in the future\n",
2646                   settle_time + 10);
2647       GNUNET_SCHEDULER_add_delayed (
2648                                     GNUNET_TIME_relative_multiply (
2649                                                                    GNUNET_TIME_UNIT_SECONDS,
2650                                                                    (settle_time
2651                                                                        + 10)),
2652                                     &capture_current_topology, topo_ctx);
2653     }
2654   else
2655     GNUNET_SCHEDULER_add_delayed (
2656                                   GNUNET_TIME_relative_multiply (
2657                                                                  GNUNET_TIME_UNIT_SECONDS,
2658                                                                  (settle_time
2659                                                                      + 10)),
2660                                   &setup_puts_and_gets, NULL);
2661
2662   if (dhtlog_handle != NULL)
2663     dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2664
2665 #if HAVE_MALICIOUS
2666   if ((GNUNET_YES != malicious_after_settle) || (settle_time == 0))
2667     {
2668       GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2669     }
2670 #endif
2671
2672   if ((GNUNET_YES == do_find_peer) && (settle_time > 0))
2673     {
2674       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2675                   "Scheduling find peer requests during \"settle\" time.\n");
2676       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
2677       find_peer_context->count_peers_cb = &count_peers_cb;
2678       find_peer_context->endtime
2679           = GNUNET_TIME_relative_to_absolute (
2680                                               GNUNET_TIME_relative_multiply (
2681                                                                              GNUNET_TIME_UNIT_SECONDS,
2682                                                                              settle_time));
2683       GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests, find_peer_context);
2684     }
2685   else
2686     {
2687       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2688                   "Assuming automatic DHT find peer requests.\n");
2689     }
2690 }
2691
2692 /**
2693  * Task to release DHT handles
2694  */
2695 static void
2696 malicious_disconnect_task(void *cls,
2697                           const struct GNUNET_SCHEDULER_TaskContext * tc)
2698 {
2699   struct MaliciousContext *ctx = cls;
2700   outstanding_malicious--;
2701   malicious_completed++;
2702   ctx->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2703   GNUNET_DHT_disconnect (ctx->dht_handle);
2704   ctx->dht_handle = NULL;
2705   GNUNET_free(ctx);
2706
2707   if (malicious_completed == malicious_getters + malicious_putters
2708       + malicious_droppers)
2709     {
2710       fprintf (stderr, "Finished setting all malicious peers up!\n");
2711     }
2712 }
2713
2714 /**
2715  * Task to release DHT handles
2716  */
2717 static void
2718 malicious_done_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2719 {
2720   struct MaliciousContext *ctx = cls;
2721   GNUNET_SCHEDULER_cancel (ctx->disconnect_task);
2722   GNUNET_SCHEDULER_add_now (&malicious_disconnect_task, ctx);
2723 }
2724
2725 /**
2726  * Set up some data, and call API PUT function
2727  */
2728 static void
2729 set_malicious(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2730 {
2731   struct MaliciousContext *ctx = cls;
2732
2733   if (outstanding_malicious > DEFAULT_MAX_OUTSTANDING_GETS)
2734     {
2735       GNUNET_SCHEDULER_add_delayed (
2736                                     GNUNET_TIME_relative_multiply (
2737                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2738                                                                    100),
2739                                     &set_malicious, ctx);
2740       return;
2741     }
2742
2743   if (ctx->dht_handle == NULL)
2744     {
2745       ctx->dht_handle = GNUNET_DHT_connect (ctx->daemon->cfg, 1);
2746       outstanding_malicious++;
2747     }
2748
2749   GNUNET_assert(ctx->dht_handle != NULL);
2750
2751 #if VERBOSE > 1
2752   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Setting peer %s malicious type %d\n",
2753       ctx->daemon->shortname, ctx->malicious_type);
2754 #endif
2755
2756   switch (ctx->malicious_type)
2757     {
2758   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET:
2759     GNUNET_DHT_set_malicious_getter (ctx->dht_handle, malicious_get_frequency, &malicious_done_task, ctx);
2760     break;
2761   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT:
2762     GNUNET_DHT_set_malicious_putter (ctx->dht_handle, malicious_put_frequency, &malicious_done_task, ctx);
2763     break;
2764   case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP:
2765     GNUNET_DHT_set_malicious_dropper (ctx->dht_handle, &malicious_done_task, ctx);
2766     break;
2767   default:
2768     break;
2769     }
2770
2771   ctx->disconnect_task
2772       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
2773                                       &malicious_disconnect_task, ctx);
2774 }
2775
2776 #if HAVE_MALICIOUS
2777 /**
2778  * Choose the next peer from the peer group to set as malicious.
2779  * If we are doing a sybil attack, find the nearest peer to the
2780  * sybil location that has not already been set malicious.  Otherwise
2781  * just choose a random not already chosen peer.
2782  *
2783  * @param pg the peer group
2784  * @param bloom the bloomfilter which contains all peer already
2785  *        chosen to be malicious
2786  */
2787 static uint32_t
2788 choose_next_malicious (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_CONTAINER_BloomFilter *bloom)
2789   {
2790     int i;
2791     int nearest;
2792     int bits_match;
2793     int curr_distance;
2794     int count;
2795     struct GNUNET_TESTING_Daemon *temp_daemon;
2796     GNUNET_HashCode uid_hash;
2797
2798     curr_distance = 0;
2799     nearest = 0;
2800     GNUNET_assert (bloom != NULL);
2801
2802     if (GNUNET_YES == malicious_sybil)
2803       {
2804         for (i = 0; i < num_peers; i++)
2805           {
2806             temp_daemon = GNUNET_TESTING_daemon_get(pg, i);
2807             hash_from_uid(i, &uid_hash);
2808             /* Check if this peer matches the bloomfilter */
2809             if ((GNUNET_NO == GNUNET_TESTING_daemon_running(temp_daemon)) || (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)))
2810             continue;
2811
2812             bits_match = GNUNET_CRYPTO_hash_matching_bits (&temp_daemon->id.hashPubKey, &sybil_target);
2813             if (bits_match >= curr_distance)
2814               {
2815                 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found nearer peer %s to %s, old matching bits %d, new %d\n", GNUNET_i2s(&temp_daemon->id), GNUNET_h2s(&sybil_target), curr_distance, bits_match);
2816                 nearest = i;
2817                 curr_distance = bits_match;
2818               }
2819           }
2820       }
2821     else
2822       {
2823         nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2824         hash_from_uid(nearest, &uid_hash);
2825         count = 0;
2826         while ((GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)) && (count < num_peers))
2827           {
2828             GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Peer %d already in bloom (tried %d times)\n", nearest, count);
2829             nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2830             hash_from_uid(nearest, &uid_hash);
2831             count++;
2832           }
2833         if (count == num_peers)
2834         GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Tried %d times to find a peer, selecting %d at random!!\n", count, nearest);
2835       }
2836
2837     return nearest;
2838   }
2839
2840 /**
2841  * Select randomly from set of known peers,
2842  * set the desired number of peers to the
2843  * proper malicious types.
2844  */
2845 static void
2846 setup_malicious_peers (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2847   {
2848     struct MaliciousContext *ctx;
2849     int i;
2850     uint32_t temp_daemon;
2851     GNUNET_HashCode uid_hash;
2852
2853     for (i = 0; i < malicious_getters; i++)
2854       {
2855         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2856         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2857         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2858         hash_from_uid(temp_daemon, &uid_hash);
2859         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2860         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET;
2861         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2862
2863       }
2864
2865     for (i = 0; i < malicious_putters; i++)
2866       {
2867         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2868         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2869         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2870         hash_from_uid(temp_daemon, &uid_hash);
2871         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2872         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT;
2873         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2874
2875       }
2876
2877     for (i = 0; i < malicious_droppers; i++)
2878       {
2879         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2880         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2881         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2882         hash_from_uid(temp_daemon, &uid_hash);
2883         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2884         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP;
2885         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2886       }
2887   }
2888 #endif
2889
2890 #if ONLY_TESTING
2891 /* Forward declaration */
2892 static void
2893 topology_callback (void *cls,
2894     const struct GNUNET_PeerIdentity *first,
2895     const struct GNUNET_PeerIdentity *second,
2896     uint32_t distance,
2897     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2898     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2899     struct GNUNET_TESTING_Daemon *first_daemon,
2900     struct GNUNET_TESTING_Daemon *second_daemon,
2901     const char *emsg);
2902
2903 /**
2904  * Retry connecting two specific peers until they connect,
2905  * at a specific interval.  These two peers previously failed
2906  * to connect, and we hope they continue to so that we can
2907  * debug the reason they are having issues.
2908  */
2909 static void
2910 repeat_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2911   {
2912
2913     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Repeating connect attempt between %s and %s.\n", repeat_connect_peer1->shortname, repeat_connect_peer2->shortname);
2914     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 1 configuration `%s'\n", repeat_connect_peer1->cfgfile);
2915     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 2 configuration `%s'\n", repeat_connect_peer2->cfgfile);
2916
2917     repeat_connect_task = GNUNET_SCHEDULER_NO_TASK;
2918     GNUNET_TESTING_daemons_connect(repeat_connect_peer1,
2919         repeat_connect_peer2,
2920         GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60),
2921         2, &topology_callback, NULL);
2922   }
2923 #endif
2924
2925 /**
2926  * This function is called whenever a connection attempt is finished between two of
2927  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
2928  * number of times this function is called should equal the number returned
2929  * from the GNUNET_TESTING_connect_topology call.
2930  *
2931  * The emsg variable is NULL on success (peers connected), and non-NULL on
2932  * failure (peers failed to connect).
2933  */
2934 static void
2935 topology_callback(void *cls, const struct GNUNET_PeerIdentity *first,
2936                   const struct GNUNET_PeerIdentity *second, uint32_t distance,
2937                   const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2938                   const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2939                   struct GNUNET_TESTING_Daemon *first_daemon,
2940                   struct GNUNET_TESTING_Daemon *second_daemon, const char *emsg)
2941 {
2942   struct TopologyIteratorContext *topo_ctx;
2943   unsigned long long duration;
2944   unsigned long long total_duration;
2945   unsigned int new_connections;
2946   unsigned int new_failed_connections;
2947   double conns_per_sec_recent;
2948   double conns_per_sec_total;
2949   double failed_conns_per_sec_recent;
2950   double failed_conns_per_sec_total;
2951   char *temp_conn_string;
2952   char *temp_conn_failed_string;
2953   char *revision_str;
2954
2955 #if ONLY_TESTING
2956   if (repeat_connect_mode == GNUNET_YES)
2957     {
2958       if ((first_daemon == repeat_connect_peer1) &&
2959           (second_daemon == repeat_connect_peer2))
2960         {
2961           if (emsg != NULL) /* Peers failed to connect again! */
2962             {
2963               GNUNET_assert(repeat_connect_task == GNUNET_SCHEDULER_NO_TASK);
2964               repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
2965               return;
2966             }
2967           else /* Repeat peers actually connected! */
2968             {
2969               if (repeat_connect_task != GNUNET_SCHEDULER_NO_TASK)
2970               GNUNET_SCHEDULER_cancel(repeat_connect_task);
2971               repeat_connect_peer1 = NULL;
2972               repeat_connect_peer2 = NULL;
2973               repeat_connect_mode = GNUNET_NO;
2974               GNUNET_TESTING_resume_connections(pg);
2975               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Resuming normal connection mode, debug connection was successful!\n");
2976             }
2977         }
2978     }
2979 #endif
2980
2981   if (GNUNET_TIME_absolute_get_difference (connect_last_time,
2982                                            GNUNET_TIME_absolute_get ()).rel_value
2983       > GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
2984                                        CONN_UPDATE_DURATION).rel_value)
2985     {
2986       /* Get number of new connections */
2987       new_connections = total_connections - previous_connections;
2988
2989       /* Get number of new FAILED connections */
2990       new_failed_connections = failed_connections - previous_failed_connections;
2991
2992       /* Get duration in seconds */
2993       duration
2994           = GNUNET_TIME_absolute_get_difference (connect_last_time,
2995                                                  GNUNET_TIME_absolute_get ()).rel_value
2996               / 1000;
2997       total_duration
2998           = GNUNET_TIME_absolute_get_difference (connect_start_time,
2999                                                  GNUNET_TIME_absolute_get ()).rel_value
3000               / 1000;
3001
3002       failed_conns_per_sec_recent = (double) new_failed_connections / duration;
3003       failed_conns_per_sec_total = (double) failed_connections / total_duration;
3004       conns_per_sec_recent = (double) new_connections / duration;
3005       conns_per_sec_total = (double) total_connections / total_duration;
3006       GNUNET_log (
3007                   GNUNET_ERROR_TYPE_WARNING,
3008                   "Recent: %.2f/s, Total: %.2f/s, Recent failed: %.2f/s, total failed %.2f/s\n",
3009                   conns_per_sec_recent, CONN_UPDATE_DURATION,
3010                   conns_per_sec_total, failed_conns_per_sec_recent,
3011                   failed_conns_per_sec_total);
3012       connect_last_time = GNUNET_TIME_absolute_get ();
3013       previous_connections = total_connections;
3014       previous_failed_connections = failed_connections;
3015       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3016                   "have %u total_connections, %u failed\n", total_connections,
3017                   failed_connections);
3018 #if ONLY_TESTING
3019       /* These conditions likely mean we've entered the death spiral of doom */
3020       if ((total_connections > 20000) &&
3021           (conns_per_sec_recent < 5.0) &&
3022           (conns_per_sec_total > 10.0) &&
3023           (emsg != NULL) &&
3024           (repeat_connect_mode == GNUNET_NO))
3025         {
3026           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Entering repeat connection attempt mode!\n");
3027           repeat_connect_peer1 = first_daemon;
3028           repeat_connect_peer2 = second_daemon;
3029           repeat_connect_mode = GNUNET_YES;
3030           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping NEW connections from being scheduled!\n");
3031           GNUNET_TESTING_stop_connections(pg);
3032           repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
3033         }
3034 #endif
3035     }
3036
3037   if (emsg == NULL)
3038     {
3039       total_connections++;
3040 #if VERBOSE > 1
3041       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
3042           first_daemon->shortname,
3043           second_daemon->shortname,
3044           distance);
3045 #endif
3046     }
3047   else
3048     {
3049       failed_connections++;
3050 #if VERBOSE
3051       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peer %s to peer %s with error :\n%s\n",
3052           first_daemon->shortname,
3053           second_daemon->shortname, emsg);
3054
3055       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
3056           first_daemon->shortname,
3057           second_daemon->shortname, emsg);
3058 #endif
3059     }
3060
3061 #if ONLY_TESTING
3062   if ((repeat_connect_mode == GNUNET_YES) )
3063   return;
3064 #endif
3065
3066   GNUNET_assert(peer_connect_meter != NULL);
3067   if (GNUNET_YES == update_meter (peer_connect_meter))
3068     {
3069 #if VERBOSE
3070       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3071           "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
3072           total_connections);
3073 #endif
3074       if (failed_connections > 0)
3075         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3076                     "While connecting, had %u failed connections.\n",
3077                     failed_connections);
3078       if (dhtlog_handle != NULL)
3079         {
3080           dhtlog_handle->update_connections (total_connections);
3081           dhtlog_handle->insert_topology (expected_connections);
3082         }
3083
3084       total_duration
3085           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3086                                                  GNUNET_TIME_absolute_get ()).rel_value
3087               / 1000;
3088       failed_conns_per_sec_total = (double) failed_connections / total_duration;
3089       conns_per_sec_total = (double) total_connections / total_duration;
3090       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3091                   "Overall connection info --- Total: %u, Total Failed %u/s\n",
3092                   total_connections, failed_connections);
3093       GNUNET_log (
3094                   GNUNET_ERROR_TYPE_WARNING,
3095                   "Overall connection info --- Total: %.2f/s, Total Failed %.2f/s\n",
3096                   conns_per_sec_total, failed_conns_per_sec_total);
3097
3098       GNUNET_asprintf (&temp_conn_string,
3099                        "DHT Profiler Connection (trial %d)",
3100                        trial_to_run);
3101       GNUNET_asprintf (&temp_conn_failed_string,
3102                        "DHT Profiler Connection failed (trial %d)",
3103                        trial_to_run);
3104       GNUNET_asprintf (&revision_str, "%llu", revision);
3105
3106       if (GNUNET_YES == insert_gauger_data)
3107         GAUGER_ID("DHT_TESTING", temp_conn_string, conns_per_sec_total, "conns/s", revision_str);
3108       if (GNUNET_YES == insert_gauger_data)
3109         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_conns_per_sec_total, "failed_conns", revision_str);
3110
3111       GNUNET_free(temp_conn_string);
3112       GNUNET_free(temp_conn_failed_string);
3113       GNUNET_asprintf (&temp_conn_string,
3114                        "DHT Profiler Total Connections (trial %d)",
3115                        trial_to_run);
3116       GNUNET_asprintf (
3117                        &temp_conn_failed_string,
3118                        "DHT Profiler Total Connections failed (trial %d)",
3119                        trial_to_run);
3120       if (GNUNET_YES == insert_gauger_data)
3121         GAUGER_ID("DHT_TESTING", temp_conn_string, total_connections, "conns", revision_str);
3122       if (GNUNET_YES == insert_gauger_data)
3123         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, failed_connections, "failed conns", revision_str);
3124       GNUNET_free(temp_conn_string);
3125       GNUNET_free(temp_conn_failed_string);
3126       GNUNET_free(revision_str);
3127
3128       GNUNET_SCHEDULER_cancel (die_task);
3129
3130       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
3131         {
3132           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
3133           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3134                       "Setting continue gets and puts as topo_cont\n");
3135           topo_ctx->cont = &continue_puts_and_gets;
3136           topo_ctx->peers_seen
3137               = GNUNET_CONTAINER_multihashmap_create (num_peers);
3138           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
3139         }
3140       else
3141         {
3142           GNUNET_log (
3143                       GNUNET_ERROR_TYPE_WARNING,
3144                       "For some reason, NOT scheduling final topology capture (settle_time %d, dhtlog_handle %s)!\n",
3145                       settle_time, dhtlog_handle);
3146           GNUNET_SCHEDULER_add_now (&continue_puts_and_gets, NULL);
3147         }
3148     }
3149   else if (total_connections + failed_connections == expected_connections)
3150     {
3151       GNUNET_SCHEDULER_cancel (die_task);
3152       die_task
3153           = GNUNET_SCHEDULER_add_now (&end_badly,
3154                                       "from topology_callback (too many failed connections)");
3155     }
3156 }
3157
3158 static void
3159 peers_started_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3160     const struct GNUNET_CONFIGURATION_Handle *cfg,
3161     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3162 {
3163   if (emsg != NULL)
3164     {
3165       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3166                   "Failed to start daemon with error: `%s'\n", emsg);
3167       return;
3168     }
3169   GNUNET_assert (id != NULL);
3170
3171 #if VERBOSE > 1
3172   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
3173       (num_peers - peers_left) + 1, num_peers);
3174 #endif
3175
3176   peers_left--;
3177
3178   if (GNUNET_YES == update_meter (peer_start_meter))
3179     {
3180 #if VERBOSE
3181       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3182           "All %d daemons started, now connecting peers!\n",
3183           num_peers);
3184 #endif
3185       GNUNET_SCHEDULER_cancel (die_task);
3186
3187       expected_connections = UINT_MAX;
3188       if ((pg != NULL) && (peers_left == 0))
3189         {
3190           connect_start_time = GNUNET_TIME_absolute_get ();
3191           expected_connections
3192               = GNUNET_TESTING_connect_topology (
3193                                                  pg,
3194                                                  connect_topology,
3195                                                  connect_topology_option,
3196                                                  connect_topology_option_modifier,
3197                                                  connect_timeout,
3198                                                  connect_attempts, NULL, NULL);
3199
3200           peer_connect_meter = create_meter (expected_connections,
3201                                              "Peer connection ", GNUNET_YES);
3202           fprintf (stderr, "Have %d expected connections\n",
3203                    expected_connections);
3204         }
3205
3206       if (expected_connections == GNUNET_SYSERR)
3207         {
3208           die_task
3209               = GNUNET_SCHEDULER_add_now (&end_badly,
3210                                           "from connect topology (bad return)");
3211         }
3212
3213       die_task
3214           = GNUNET_SCHEDULER_add_delayed (
3215                                           GNUNET_TIME_relative_multiply (
3216                                                                          GNUNET_TIME_UNIT_SECONDS,
3217                                                                          DEFAULT_CONNECT_TIMEOUT
3218                                                                              * expected_connections),
3219                                           &end_badly,
3220                                           "from connect topology (timeout)");
3221
3222       ok = 0;
3223     }
3224 }
3225
3226 static void
3227 create_topology()
3228 {
3229   unsigned int create_expected_connections;
3230   peers_left = num_peers; /* Reset counter */
3231   create_expected_connections = GNUNET_TESTING_create_topology (pg, topology, blacklist_topology,
3232                                                                 blacklist_transports);
3233   if (create_expected_connections > 0)
3234     {
3235       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3236                   "Topology set up, have %u expected connections, now starting peers!\n", create_expected_connections);
3237       GNUNET_TESTING_daemons_continue_startup (pg);
3238     }
3239   else
3240     {
3241       GNUNET_SCHEDULER_cancel (die_task);
3242       die_task = GNUNET_SCHEDULER_add_now (&end_badly,
3243                                            "from create topology (bad return)");
3244     }
3245   GNUNET_free_non_null(blacklist_transports);
3246   GNUNET_SCHEDULER_cancel (die_task);
3247   die_task
3248       = GNUNET_SCHEDULER_add_delayed (
3249                                       GNUNET_TIME_relative_multiply (
3250                                                                      seconds_per_peer_start,
3251                                                                      num_peers),
3252                                       &end_badly,
3253                                       "from continue startup (timeout)");
3254 }
3255
3256 /**
3257  * Callback indicating that the hostkey was created for a peer.
3258  *
3259  * @param cls NULL
3260  * @param id the peer identity
3261  * @param d the daemon handle (pretty useless at this point, remove?)
3262  * @param emsg non-null on failure
3263  */
3264 static void
3265 hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3266     struct GNUNET_TESTING_Daemon *d, const char *emsg)
3267 {
3268   if (emsg != NULL)
3269     {
3270       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3271                   "Hostkey callback received error: %s\n", emsg);
3272     }
3273
3274 #if VERBOSE > 1
3275   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3276       "Hostkey (%d/%d) created for peer `%s'\n",
3277       num_peers - peers_left, num_peers, GNUNET_i2s(id));
3278 #endif
3279
3280   peers_left--;
3281   if (GNUNET_YES == update_meter (hostkey_meter))
3282     {
3283       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3284                   "All %d hostkeys created, now creating topology!\n",
3285                   num_peers);
3286       GNUNET_SCHEDULER_cancel (die_task);
3287       /* Set up task in case topology creation doesn't finish
3288        * within a reasonable amount of time */
3289       die_task = GNUNET_SCHEDULER_add_delayed (DEFAULT_TOPOLOGY_TIMEOUT,
3290                                                &end_badly,
3291                                                "from create_topology");
3292       GNUNET_SCHEDULER_add_now (&create_topology, NULL);
3293       ok = 0;
3294     }
3295 }
3296
3297 static void
3298 run(void *cls, char * const *args, const char *cfgfile,
3299     const struct GNUNET_CONFIGURATION_Handle *cfg)
3300 {
3301   struct stat frstat;
3302   struct GNUNET_DHTLOG_TrialInfo trial_info;
3303   struct GNUNET_TESTING_Host *hosts;
3304   struct GNUNET_TESTING_Host *temphost;
3305   struct GNUNET_TESTING_Host *tempnext;
3306   char *topology_str;
3307   char *connect_topology_str;
3308   char *blacklist_topology_str;
3309   char *connect_topology_option_str;
3310   char *connect_topology_option_modifier_string;
3311   char *trialmessage;
3312   char *topology_percentage_str;
3313   float topology_percentage;
3314   char *topology_probability_str;
3315   char *hostfile;
3316   float topology_probability;
3317   unsigned long long temp_config_number;
3318   int stop_closest;
3319   int stop_found;
3320   int strict_kademlia;
3321   char *buf;
3322   char *data;
3323   char *churn_data;
3324   char *churn_filename;
3325   int count;
3326   int ret;
3327   int line_number;
3328   int k;
3329
3330   config = cfg;
3331   rounds_finished = 0;
3332   memset (&trial_info, 0, sizeof(struct GNUNET_DHTLOG_TrialInfo));
3333   /* Get path from configuration file */
3334   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "paths",
3335                                                            "servicehome",
3336                                                            &test_directory))
3337     {
3338       ok = 404;
3339       return;
3340     }
3341
3342   /* Get number of peers to start from configuration */
3343   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3344                                                               "num_peers",
3345                                                               &num_peers))
3346     {
3347       GNUNET_log (
3348                   GNUNET_ERROR_TYPE_WARNING,
3349                   "Number of peers must be specified in section %s option %s\n",
3350                   "TESTING", "NUM_PEERS");
3351     }
3352   GNUNET_assert(num_peers > 0 && num_peers < ULONG_MAX);
3353
3354   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3355                                                           "connect_timeout",
3356                                                           &temp_config_number))
3357     connect_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3358                                                      temp_config_number);
3359   else
3360     {
3361       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3362                   "testing", "connect_timeout");
3363       return;
3364     }
3365
3366   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3367                                                           "connect_attempts",
3368                                                           &connect_attempts))
3369     {
3370       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3371                   "testing", "connect_attempts");
3372       return;
3373     }
3374
3375   if (GNUNET_OK
3376       != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3377                                                 "max_outstanding_connections",
3378                                                 &max_outstanding_connections))
3379     {
3380       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3381                   "testing", "max_outstanding_connections");
3382       return;
3383     }
3384
3385   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3386                                                           "max_concurrent_ssh",
3387                                                           &max_concurrent_ssh))
3388     {
3389       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3390                   "testing", "max_concurrent_ssh");
3391       return;
3392     }
3393
3394   /**
3395    * Get DHT specific testing options.
3396    */
3397   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3398                                                            "mysql_logging"))
3399       || (GNUNET_YES
3400           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3401                                                    "mysql_logging_extended"))
3402       || (GNUNET_YES
3403           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3404                                                    "mysql_logging_minimal")))
3405     {
3406       if (GNUNET_YES
3407           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3408                                                    "mysql_logging_minimal"))
3409         dhtlog_minimal = GNUNET_YES;
3410
3411       dhtlog_handle = GNUNET_DHTLOG_connect (cfg);
3412       if (dhtlog_handle == NULL)
3413         {
3414           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3415                       "Could not connect to mysql server for logging, will NOT log dht operations!");
3416           ok = 3306;
3417           return;
3418         }
3419     }
3420
3421   stop_closest = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3422                                                        "stop_on_closest");
3423   if (stop_closest == GNUNET_SYSERR)
3424     stop_closest = GNUNET_NO;
3425
3426   stop_found = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht", "stop_found");
3427   if (stop_found == GNUNET_SYSERR)
3428     stop_found = GNUNET_NO;
3429
3430   strict_kademlia = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3431                                                           "strict_kademlia");
3432   if (strict_kademlia == GNUNET_SYSERR)
3433     strict_kademlia = GNUNET_NO;
3434
3435   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3436                                                           "comment",
3437                                                           &trialmessage))
3438     trialmessage = NULL;
3439
3440   churn_data = NULL;
3441   /** Check for a churn file to do churny simulation */
3442   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3443                                                           "churn_file",
3444                                                           &churn_filename))
3445     {
3446       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Reading churn data from %s\n",
3447                   churn_filename);
3448       if (GNUNET_OK != GNUNET_DISK_file_test (churn_filename))
3449         {
3450           GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Error reading churn file!\n");
3451           GNUNET_free_non_null(trialmessage);
3452           GNUNET_free(churn_filename);
3453           return;
3454         }
3455       if ((0 != STAT (churn_filename, &frstat)) || (frstat.st_size == 0))
3456         {
3457           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3458                       "Could not open file specified for churn data, ending test!");
3459           ok = 1119;
3460           GNUNET_free_non_null(trialmessage);
3461           GNUNET_free(churn_filename);
3462           return;
3463         }
3464
3465       churn_data = GNUNET_malloc_large (frstat.st_size);
3466       GNUNET_assert(churn_data != NULL);
3467       if (frstat.st_size != GNUNET_DISK_fn_read (churn_filename, churn_data,
3468                                                  frstat.st_size))
3469         {
3470           GNUNET_log (
3471                       GNUNET_ERROR_TYPE_ERROR,
3472                       "Could not read file %s specified for churn, ending test!",
3473                       churn_filename);
3474           GNUNET_free (churn_filename);
3475           GNUNET_free (churn_data);
3476           GNUNET_free_non_null(trialmessage);
3477           return;
3478         }
3479
3480       GNUNET_free_non_null(churn_filename);
3481
3482       buf = churn_data;
3483       count = 0;
3484       /* Read the first line */
3485       while (count < frstat.st_size)
3486         {
3487           count++;
3488           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3489             {
3490               churn_data[count] = '\0';
3491               if (1 != sscanf (buf, "%u", &churn_rounds))
3492                 {
3493                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3494                               "Failed to read number of rounds from churn file, ending test!\n");
3495                   ret = 4200;
3496                   GNUNET_free_non_null(trialmessage);
3497                   GNUNET_free_non_null(churn_data);
3498                   return;
3499                 }
3500               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3501                           "Read %u rounds from churn file\n", churn_rounds);
3502               buf = &churn_data[count + 1];
3503               churn_array = GNUNET_malloc(sizeof(unsigned int) * churn_rounds);
3504               break; /* Done with this part */
3505             }
3506         }
3507
3508       if (GNUNET_OK
3509           != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3510                                                     "churns_per_round",
3511                                                     &churns_per_round))
3512         {
3513           churns_per_round = (unsigned long long) churn_rounds;
3514         }
3515
3516       line_number = 0;
3517       while ((count < frstat.st_size) && (line_number < churn_rounds))
3518         {
3519           count++;
3520           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3521             {
3522               churn_data[count] = '\0';
3523
3524               ret = sscanf (buf, "%u", &churn_array[line_number]);
3525               if (1 == ret)
3526                 {
3527                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3528                               "Read %u peers in round %u\n",
3529                               churn_array[line_number], line_number);
3530                   line_number++;
3531                 }
3532               else
3533                 {
3534                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3535                               "Error reading line `%s' in hostfile\n", buf);
3536                   buf = &churn_data[count + 1];
3537                   continue;
3538                 }
3539               buf = &churn_data[count + 1];
3540             }
3541           else if (churn_data[count] == '\n') /* Blank line */
3542             buf = &churn_data[count + 1];
3543         }
3544     }
3545   GNUNET_free_non_null(churn_data);
3546
3547   /* Check for a hostfile containing user@host:port triples */
3548   if (GNUNET_OK
3549       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "hostfile",
3550                                                 &hostfile))
3551     hostfile = NULL;
3552
3553   hosts = NULL;
3554   temphost = NULL;
3555   data = NULL;
3556   if (hostfile != NULL)
3557     {
3558       if (GNUNET_OK != GNUNET_DISK_file_test (hostfile))
3559         GNUNET_DISK_fn_write (hostfile, NULL, 0, GNUNET_DISK_PERM_USER_READ
3560             | GNUNET_DISK_PERM_USER_WRITE);
3561       if ((0 != STAT (hostfile, &frstat)) || (frstat.st_size == 0))
3562         {
3563           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3564                       "Could not open file specified for host list, ending test!");
3565           ok = 1119;
3566           GNUNET_free_non_null(trialmessage);
3567           GNUNET_free(hostfile);
3568           return;
3569         }
3570
3571       data = GNUNET_malloc_large (frstat.st_size);
3572       GNUNET_assert(data != NULL);
3573       if (frstat.st_size
3574           != GNUNET_DISK_fn_read (hostfile, data, frstat.st_size))
3575         {
3576           GNUNET_log (
3577                       GNUNET_ERROR_TYPE_ERROR,
3578                       "Could not read file %s specified for host list, ending test!",
3579                       hostfile);
3580           GNUNET_free (hostfile);
3581           GNUNET_free (data);
3582           GNUNET_free_non_null(trialmessage);
3583           return;
3584         }
3585
3586       GNUNET_free_non_null(hostfile);
3587
3588       buf = data;
3589       count = 0;
3590       while (count < frstat.st_size - 1)
3591         {
3592           count++;
3593           if (((data[count] == '\n')) && (buf != &data[count]))
3594             {
3595               data[count] = '\0';
3596               temphost = GNUNET_malloc(sizeof(struct GNUNET_TESTING_Host));
3597               ret = sscanf (buf, "%a[a-zA-Z0-9_]@%a[a-zA-Z0-9.]:%hd",
3598                             &temphost->username, &temphost->hostname,
3599                             &temphost->port);
3600               if (3 == ret)
3601                 {
3602                   GNUNET_log (
3603                               GNUNET_ERROR_TYPE_DEBUG,
3604                               "Successfully read host %s, port %d and user %s from file\n",
3605                               temphost->hostname, temphost->port,
3606                               temphost->username);
3607                 }
3608               else
3609                 {
3610                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3611                               "Error reading line `%s' in hostfile\n", buf);
3612                   GNUNET_free(temphost);
3613                   buf = &data[count + 1];
3614                   continue;
3615                 }
3616               temphost->next = hosts;
3617               hosts = temphost;
3618               buf = &data[count + 1];
3619             }
3620           else if ((data[count] == '\n') || (data[count] == '\0'))
3621             buf = &data[count + 1];
3622         }
3623     }
3624   GNUNET_free_non_null(data);
3625   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3626                                                           "malicious_getters",
3627                                                           &malicious_getters))
3628     malicious_getters = 0;
3629
3630   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3631                                                           "malicious_putters",
3632                                                           &malicious_putters))
3633     malicious_putters = 0;
3634
3635   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3636                                                           "malicious_droppers",
3637                                                           &malicious_droppers))
3638     malicious_droppers = 0;
3639
3640   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3641                                                           "settle_time",
3642                                                           &settle_time))
3643     settle_time = 0;
3644
3645   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3646                                                               "dht_testing",
3647                                                               "num_puts",
3648                                                               &num_puts))
3649     num_puts = num_peers;
3650
3651   if (GNUNET_SYSERR
3652       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3653                                                 "put_replication",
3654                                                 &put_replication))
3655     put_replication = DEFAULT_PUT_REPLICATION;
3656
3657   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3658                                                               "dht_testing",
3659                                                               "num_gets",
3660                                                               &num_gets))
3661     num_gets = num_peers;
3662
3663   if (GNUNET_SYSERR
3664       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3665                                                 "get_replication",
3666                                                 &get_replication))
3667     get_replication = DEFAULT_GET_REPLICATION;
3668
3669   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3670                                                           "find_peer_delay",
3671                                                           &temp_config_number))
3672     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3673                                                      temp_config_number);
3674   else
3675     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3676
3677   if (GNUNET_OK
3678       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3679                                                 "concurrent_find_peers",
3680                                                 &temp_config_number))
3681     max_outstanding_find_peers = temp_config_number;
3682   else
3683     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3684
3685   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3686                                                           "get_timeout",
3687                                                           &temp_config_number))
3688     get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3689                                                  temp_config_number);
3690   else
3691     get_timeout = DEFAULT_GET_TIMEOUT;
3692
3693   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3694                                                           "concurrent_puts",
3695                                                           &temp_config_number))
3696     max_outstanding_puts = temp_config_number;
3697   else
3698     max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
3699
3700   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3701                                                           "concurrent_gets",
3702                                                           &temp_config_number))
3703     max_outstanding_gets = temp_config_number;
3704   else
3705     max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
3706
3707   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3708                                                           "timeout",
3709                                                           &temp_config_number))
3710     all_get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3711                                                      temp_config_number);
3712   else
3713     all_get_timeout.rel_value = get_timeout.rel_value * num_gets;
3714
3715   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3716                                                           "get_delay",
3717                                                           &temp_config_number))
3718     get_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3719                                                temp_config_number);
3720   else
3721     get_delay = DEFAULT_GET_DELAY;
3722
3723   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3724                                                           "put_delay",
3725                                                           &temp_config_number))
3726     put_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3727                                                temp_config_number);
3728   else
3729     put_delay = DEFAULT_PUT_DELAY;
3730
3731   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3732                                                           "peer_start_timeout",
3733                                                           &temp_config_number))
3734     seconds_per_peer_start
3735         = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3736                                          temp_config_number);
3737   else
3738     seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START;
3739
3740   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3741                                                           "data_size",
3742                                                           &temp_config_number))
3743     test_data_size = temp_config_number;
3744   else
3745     test_data_size = DEFAULT_TEST_DATA_SIZE;
3746
3747   /**
3748    * Get testing related options.
3749    */
3750   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3751                                                           "REPLICATE_SAME"))
3752     replicate_same = GNUNET_YES;
3753
3754   /**
3755    * Get testing related options.
3756    */
3757   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3758                                                           "GET_FROM_SAME"))
3759     get_from_same = GNUNET_YES;
3760
3761   if (GNUNET_NO
3762       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3763                                               "MALICIOUS_GET_FREQUENCY",
3764                                               &malicious_get_frequency))
3765     malicious_get_frequency = DEFAULT_MALICIOUS_GET_FREQUENCY;
3766
3767   if (GNUNET_NO
3768       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3769                                               "MALICIOUS_PUT_FREQUENCY",
3770                                               &malicious_put_frequency))
3771     malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY;
3772
3773   if (GNUNET_YES
3774       == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3775                                                "MALICIOUS_AFTER_SETTLE"))
3776     malicious_after_settle = GNUNET_YES;
3777
3778   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3779                                                           "MALICIOUS_SYBIL"))
3780     {
3781       /* Set up the malicious target at random for this round */
3782       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
3783                                         &sybil_target);
3784       malicious_sybil = GNUNET_YES;
3785     }
3786
3787   /* Create the bloomfilter for choosing which peers to set malicious */
3788
3789   /* Bloomfilter size must be 2^k for some integer k */
3790   k = 1;
3791   while (1 << k < malicious_droppers)
3792     k++;
3793   if (malicious_droppers > 0)
3794     malicious_bloom = GNUNET_CONTAINER_bloomfilter_init (NULL, 1 << k,
3795                                                          DHT_BLOOM_K);
3796
3797   /* The normal behavior of the DHT is to do find peer requests
3798    * on its own.  Only if this is explicitly turned off should
3799    * the testing driver issue find peer requests (even though
3800    * this is likely the default when testing).
3801    */
3802   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3803                                                          "do_find_peer"))
3804     do_find_peer = GNUNET_YES;
3805
3806   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3807                                                            "insert_gauger_data"))
3808     insert_gauger_data = GNUNET_YES;
3809
3810   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3811                                                           "republish"))
3812     in_dht_replication = GNUNET_YES;
3813
3814   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3815                                                            "TRIAL_TO_RUN",
3816                                                            &trial_to_run))
3817     trial_to_run = 0;
3818
3819   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3820                                                            "REVISION",
3821                                                            &revision))
3822     revision = 0;
3823
3824   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3825                                                            "FIND_PEER_DELAY",
3826                                                            &temp_config_number))
3827     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3828                                                      temp_config_number);
3829   else
3830     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3831
3832   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3833                                                            "ROUND_DELAY",
3834                                                            &round_delay))
3835     round_delay = 0;
3836
3837   if (GNUNET_NO
3838       == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3839                                                 "OUTSTANDING_FIND_PEERS",
3840                                                 &max_outstanding_find_peers))
3841     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3842
3843   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3844                                                           "strict_kademlia"))
3845     max_outstanding_find_peers = max_outstanding_find_peers * 1;
3846
3847   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
3848                                                   max_outstanding_find_peers);
3849
3850   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3851                                                               "dht_testing",
3852                                                               "num_rounds",
3853                                                               &total_rounds))
3854     total_rounds = 1;
3855
3856   if ((GNUNET_SYSERR
3857       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3858                                                 "target_total_connections",
3859                                                 &target_total_connections))
3860       || (target_total_connections == 0))
3861     target_total_connections = connection_estimate (num_peers,
3862                                                     DEFAULT_BUCKET_SIZE);
3863
3864   topology_str = NULL;
3865   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3866                                                             "topology",
3867                                                             &topology_str))
3868       && (GNUNET_NO == GNUNET_TESTING_topology_get (&topology, topology_str)))
3869     {
3870       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3871                   "Invalid topology `%s' given for section %s option %s\n",
3872                   topology_str, "TESTING", "TOPOLOGY");
3873       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
3874     }
3875
3876   if (GNUNET_OK
3877       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage",
3878                                                 &topology_percentage_str))
3879     topology_percentage = 0.5;
3880   else
3881     {
3882       topology_percentage = atof (topology_percentage_str);
3883       GNUNET_free(topology_percentage_str);
3884     }
3885
3886   if (GNUNET_OK
3887       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability",
3888                                                 &topology_probability_str))
3889     topology_probability = 0.5;
3890   else
3891     {
3892       topology_probability = atof (topology_probability_str);
3893       GNUNET_free(topology_probability_str);
3894     }
3895
3896   if ((GNUNET_YES
3897       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3898                                                 "connect_topology",
3899                                                 &connect_topology_str))
3900       && (GNUNET_NO == GNUNET_TESTING_topology_get (&connect_topology,
3901                                                     connect_topology_str)))
3902     {
3903       GNUNET_log (
3904                   GNUNET_ERROR_TYPE_WARNING,
3905                   "Invalid connect topology `%s' given for section %s option %s\n",
3906                   connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
3907     }
3908   GNUNET_free_non_null(connect_topology_str);
3909
3910   if ((GNUNET_YES
3911       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3912                                                 "connect_topology_option",
3913                                                 &connect_topology_option_str))
3914       && (GNUNET_NO
3915           == GNUNET_TESTING_topology_option_get (&connect_topology_option,
3916                                                  connect_topology_option_str)))
3917     {
3918       GNUNET_log (
3919                   GNUNET_ERROR_TYPE_WARNING,
3920                   "Invalid connect topology option `%s' given for section %s option %s\n",
3921                   connect_topology_option_str, "TESTING",
3922                   "CONNECT_TOPOLOGY_OPTION");
3923       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
3924     }
3925   GNUNET_free_non_null(connect_topology_option_str);
3926
3927   if (GNUNET_YES
3928       == GNUNET_CONFIGURATION_get_value_string (
3929                                                 cfg,
3930                                                 "testing",
3931                                                 "connect_topology_option_modifier",
3932                                                 &connect_topology_option_modifier_string))
3933     {
3934       if (sscanf (connect_topology_option_modifier_string, "%lf",
3935                   &connect_topology_option_modifier) != 1)
3936         {
3937           GNUNET_log (
3938                       GNUNET_ERROR_TYPE_WARNING,
3939                       _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
3940                       connect_topology_option_modifier_string,
3941                       "connect_topology_option_modifier", "TESTING");
3942         }
3943       GNUNET_free (connect_topology_option_modifier_string);
3944     }
3945
3946   if (GNUNET_YES
3947       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3948                                                 "blacklist_transports",
3949                                                 &blacklist_transports))
3950     blacklist_transports = NULL;
3951
3952   if ((GNUNET_YES
3953       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3954                                                 "blacklist_topology",
3955                                                 &blacklist_topology_str))
3956       && (GNUNET_NO == GNUNET_TESTING_topology_get (&blacklist_topology,
3957                                                     blacklist_topology_str)))
3958     {
3959       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3960                   "Invalid topology `%s' given for section %s option %s\n",
3961                   topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
3962     }
3963   GNUNET_free_non_null(topology_str);
3964   GNUNET_free_non_null(blacklist_topology_str);
3965
3966   /* Set peers_left so we know when all peers started */
3967   peers_left = num_peers;
3968
3969   /* Set up a task to end testing if peer start fails */
3970   die_task
3971       = GNUNET_SCHEDULER_add_delayed (
3972                                       GNUNET_TIME_relative_multiply (
3973                                                                      seconds_per_peer_start,
3974                                                                      num_peers),
3975                                       &end_badly,
3976                                       "didn't generate all hostkeys within allowed startup time!");
3977
3978   if (dhtlog_handle == NULL)
3979     GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "dhtlog_handle is NULL!");
3980
3981   trial_info.other_identifier = (unsigned int) trial_to_run;
3982   trial_info.num_nodes = peers_left;
3983   trial_info.topology = topology;
3984   trial_info.blacklist_topology = blacklist_topology;
3985   trial_info.connect_topology = connect_topology;
3986   trial_info.connect_topology_option = connect_topology_option;
3987   trial_info.connect_topology_option_modifier
3988       = connect_topology_option_modifier;
3989   trial_info.topology_percentage = topology_percentage;
3990   trial_info.topology_probability = topology_probability;
3991   trial_info.puts = num_puts;
3992   trial_info.gets = num_gets;
3993   trial_info.concurrent = max_outstanding_gets;
3994   trial_info.settle_time = settle_time;
3995   trial_info.num_rounds = total_rounds;
3996   trial_info.malicious_getters = malicious_getters;
3997   trial_info.malicious_putters = malicious_putters;
3998   trial_info.malicious_droppers = malicious_droppers;
3999   trial_info.malicious_get_frequency = malicious_get_frequency.rel_value;
4000   trial_info.malicious_put_frequency = malicious_put_frequency.rel_value;
4001   trial_info.stop_closest = stop_closest;
4002   trial_info.stop_found = stop_found;
4003   trial_info.strict_kademlia = strict_kademlia;
4004
4005   if (trialmessage != NULL)
4006     trial_info.message = trialmessage;
4007   else
4008     trial_info.message = "";
4009
4010   if (dhtlog_handle != NULL)
4011     dhtlog_handle->insert_trial (&trial_info);
4012
4013   GNUNET_free_non_null(trialmessage);
4014
4015   hostkey_meter = create_meter (peers_left, "Hostkeys created ", GNUNET_YES);
4016   peer_start_meter = create_meter (peers_left, "Peers started ", GNUNET_YES);
4017
4018   put_meter = create_meter (num_puts, "Puts completed ", GNUNET_YES);
4019   get_meter = create_meter (num_gets, "Gets completed ", GNUNET_YES);
4020   pg
4021       = GNUNET_TESTING_daemons_start (
4022                                       cfg,
4023                                       peers_left,
4024                                       max_outstanding_connections,
4025                                       max_concurrent_ssh,
4026                                       GNUNET_TIME_relative_multiply (
4027                                                                      seconds_per_peer_start,
4028                                                                      num_peers),
4029                                       &hostkey_callback, NULL,
4030                                       &peers_started_callback, NULL,
4031                                       &topology_callback, NULL, hosts);
4032   temphost = hosts;
4033   while (temphost != NULL)
4034     {
4035       tempnext = temphost->next;
4036       GNUNET_free (temphost->username);
4037       GNUNET_free (temphost->hostname);
4038       GNUNET_free (temphost);
4039       temphost = tempnext;
4040     }
4041 }
4042
4043 int
4044 main(int argc, char *argv[])
4045 {
4046   int ret;
4047   struct GNUNET_GETOPT_CommandLineOption options[] =
4048     {
4049     GNUNET_GETOPT_OPTION_END };
4050
4051   ret = GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-driver", "nohelp", options,
4052                             &run, &ok);
4053
4054   if (malicious_bloom != NULL)
4055     GNUNET_CONTAINER_bloomfilter_free (malicious_bloom);
4056
4057   if (ret != GNUNET_OK)
4058     {
4059       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4060                   "`gnunet-dht-driver': Failed with error code %d\n", ret);
4061     }
4062
4063   /**
4064    * Need to remove base directory, subdirectories taken care
4065    * of by the testing framework.
4066    */
4067   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
4068     {
4069       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4070                   "Failed to remove testing directory %s\n", test_directory);
4071     }
4072   return ret;
4073 }
4074
4075 /* end of gnunet-dht-driver.c */