src/dht/gnunet-dht-driver.c

   1 /*
   2  This file is part of GNUnet.
   3  (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5  GNUnet is free software; you can redistribute it and/or modify
   6  it under the terms of the GNU General Public License as published
   7  by the Free Software Foundation; either version 3, or (at your
   8  option) any later version.
   9
  10  GNUnet is distributed in the hope that it will be useful, but
  11  WITHOUT ANY WARRANTY; without even the implied warranty of
  12  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  General Public License for more details.
  14
  15  You should have received a copy of the GNU General Public License
  16  along with GNUnet; see the file COPYING.  If not, write to the
  17  Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18  Boston, MA 02111-1307, USA.
  19  */
  20 /**
  21  * @file dht/gnunet-dht-driver.c
  22  * @brief Driver for setting up a group of gnunet peers and
  23  *        then issuing GETS and PUTS on the DHT.  Coarse results
  24  *        are reported, fine grained results (if requested) are
  25  *        logged to a (mysql) database, or to file.
  26  * @author Nathan Evans (who to blame)
  27  */
  28 #include "platform.h"
  29 #ifndef HAVE_MALICIOUS
  30 #error foo
  31 #endif
  32 #include "gnunet_testing_lib.h"
  33 #include "gnunet_core_service.h"
  34 #include "gnunet_dht_service.h"
  35 #include "dhtlog.h"
  36 #include "dht.h"
  37 #include "gauger.h"
  38
  39 /* Specific DEBUG hack, do not use normally (may leak memory, segfault, or eat children.) */
  40 #define ONLY_TESTING GNUNET_NO
  41
  42 /* DEFINES */
  43 #define VERBOSE GNUNET_NO
  44
  45 /* Timeout for entire driver to run */
  46 #define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  47
  48 /* Timeout for waiting for (individual) replies to get requests */
  49 #define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  50
  51 #define DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  52
  53 /* Timeout for waiting for gets to be sent to the service */
  54 #define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  55
  56 /* Timeout for waiting for puts to be sent to the service */
  57 #define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10)
  58
  59 /* Time to allow a find peer request to take */
  60 #define DEFAULT_FIND_PEER_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 40)
  61
  62 /* Time to wait for all peers disconnected due to to churn to actually be removed from system */
  63 #define DEFAULT_PEER_DISCONNECT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  64
  65 #define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45)
  66
  67 #define DEFAULT_TEST_DATA_SIZE 8
  68
  69 #define DEFAULT_BUCKET_SIZE 4
  70
  71 /* If more than this many peers are added, slow down sending */
  72 #define MAX_FIND_PEER_CUTOFF 2000
  73
  74 /* If less than this many peers are added, speed up sending */
  75 #define MIN_FIND_PEER_CUTOFF 500
  76
  77 /* How often (in seconds) to print out connection information */
  78 #define CONN_UPDATE_DURATION 10
  79
  80 #define DEFAULT_MAX_OUTSTANDING_PUTS 10
  81
  82 #define DEFAULT_MAX_OUTSTANDING_FIND_PEERS 64
  83
  84 #define DEFAULT_FIND_PEER_OFFSET GNUNET_TIME_relative_divide (DEFAULT_FIND_PEER_DELAY, DEFAULT_MAX_OUTSTANDING_FIND_PEERS)
  85
  86 #define DEFAULT_MAX_OUTSTANDING_GETS 10
  87
  88 #define DEFAULT_CONNECT_TIMEOUT 60
  89
  90 #define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8)
  91
  92 #define DEFAULT_RECONNECT_ATTEMPTS 8
  93
  94 /*
  95  * Default frequency for sending malicious get messages
  96  */
  97 #define DEFAULT_MALICIOUS_GET_FREQUENCY GNUNET_TIME_UNIT_SECONDS
  98
  99 /*
 100  * Default frequency for sending malicious put messages
 101  */
 102 #define DEFAULT_MALICIOUS_PUT_FREQUENCY GNUNET_TIME_UNIT_SECONDS
 103
 104 /* Structs */
 105
 106 struct MaliciousContext
 107 {
 108   /**
 109    * Handle to DHT service (via the API)
 110    */
 111   struct GNUNET_DHT_Handle *dht_handle;
 112
 113   /**
 114    *  Handle to the peer daemon
 115    */
 116   struct GNUNET_TESTING_Daemon *daemon;
 117
 118   /**
 119    * Task for disconnecting DHT handles
 120    */
 121   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 122
 123   /**
 124    * What type of malicious to set this peer to.
 125    */
 126   int malicious_type;
 127 };
 128
 129 struct TestFindPeer
 130 {
 131   /* This is a linked list */
 132   struct TestFindPeer *next;
 133
 134   /* Handle to the bigger context */
 135   struct FindPeerContext *find_peer_context;
 136
 137   /**
 138    * Handle to the peer's DHT service (via the API)
 139    */
 140   struct GNUNET_DHT_Handle *dht_handle;
 141
 142   /**
 143    *  Handle to the peer daemon
 144    */
 145   struct GNUNET_TESTING_Daemon *daemon;
 146
 147   /**
 148    * Task for disconnecting DHT handles
 149    */
 150   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 151 };
 152
 153 struct TestPutContext
 154 {
 155   /* This is a linked list */
 156   struct TestPutContext *next;
 157
 158   /**
 159    * Handle to the first peers DHT service (via the API)
 160    */
 161   struct GNUNET_DHT_Handle *dht_handle;
 162
 163   /**
 164    *  Handle to the PUT peer daemon
 165    */
 166   struct GNUNET_TESTING_Daemon *daemon;
 167
 168   /**
 169    *  Identifier for this PUT
 170    */
 171   uint32_t uid;
 172
 173   /**
 174    * Task for disconnecting DHT handles
 175    */
 176   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 177 };
 178
 179 struct TestGetContext
 180 {
 181   /* This is a linked list */
 182   struct TestGetContext *next;
 183
 184   /**
 185    * Handle to the first peers DHT service (via the API)
 186    */
 187   struct GNUNET_DHT_Handle *dht_handle;
 188
 189   /**
 190    * Handle for the DHT get request
 191    */
 192   struct GNUNET_DHT_GetHandle *get_handle;
 193
 194   /**
 195    *  Handle to the GET peer daemon
 196    */
 197   struct GNUNET_TESTING_Daemon *daemon;
 198
 199   /**
 200    *  Identifier for this GET
 201    */
 202   uint32_t uid;
 203
 204   /**
 205    * Task for disconnecting DHT handles (and stopping GET)
 206    */
 207   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 208
 209   /**
 210    * Whether or not this request has been fulfilled already.
 211    */
 212   int succeeded;
 213 };
 214
 215 /**
 216  * Simple struct to keep track of progress, and print a
 217  * nice little percentage meter for long running tasks.
 218  */
 219 struct ProgressMeter
 220 {
 221   unsigned int total;
 222
 223   unsigned int modnum;
 224
 225   unsigned int dotnum;
 226
 227   unsigned int completed;
 228
 229   int print;
 230
 231   char *startup_string;
 232 };
 233
 234 /**
 235  * Linked list of information for populating statistics
 236  * before ending trial.
 237  */
 238 struct StatisticsIteratorContext
 239 {
 240   const struct GNUNET_PeerIdentity *peer;
 241   unsigned int stat_routes;
 242   unsigned int stat_route_forwards;
 243   unsigned int stat_results;
 244   unsigned int stat_results_to_client;
 245   unsigned int stat_result_forwards;
 246   unsigned int stat_gets;
 247   unsigned int stat_puts;
 248   unsigned int stat_puts_inserted;
 249   unsigned int stat_find_peer;
 250   unsigned int stat_find_peer_start;
 251   unsigned int stat_get_start;
 252   unsigned int stat_put_start;
 253   unsigned int stat_find_peer_reply;
 254   unsigned int stat_get_reply;
 255   unsigned int stat_find_peer_answer;
 256   unsigned int stat_get_response_start;
 257 };
 258
 259 /**
 260  * Context for getting a topology, logging it, and continuing
 261  * on with some next operation.
 262  */
 263 struct TopologyIteratorContext
 264 {
 265   unsigned int total_iterations;
 266   unsigned int current_iteration;
 267   unsigned int total_connections;
 268   unsigned int total_peers;
 269   struct GNUNET_CONTAINER_MultiHashMap *peers_seen;
 270   struct GNUNET_PeerIdentity *peer;
 271   GNUNET_SCHEDULER_Task cont;
 272   void *cls;
 273   struct GNUNET_TIME_Relative timeout;
 274 };
 275
 276 struct PeerCount
 277 {
 278   /** Node in the heap */
 279   struct GNUNET_CONTAINER_HeapNode *heap_node;
 280
 281   /** Peer the count refers to */
 282   struct GNUNET_PeerIdentity peer_id;
 283
 284   /** Count of connections this peer has */
 285   unsigned int count;
 286 };
 287
 288 /**
 289  * Context for sending out find peer requests.
 290  */
 291 struct FindPeerContext
 292 {
 293   /**
 294    * How long to send find peer requests, once the settle time
 295    * is over don't send any more out!
 296    */
 297   struct GNUNET_TIME_Absolute endtime;
 298
 299   /**
 300    * Number of connections in the current topology
 301    * (after this round of find peer requests has ended).
 302    */
 303   unsigned int current_peers;
 304
 305   /**
 306    * Number of connections in the current topology
 307    * (before this round of find peer requests started).
 308    */
 309   unsigned int previous_peers;
 310
 311   /**
 312    * Number of find peer requests we have currently
 313    * outstanding.
 314    */
 315   unsigned int outstanding;
 316
 317   /**
 318    * Number of find peer requests to send in this round.
 319    */
 320   unsigned int total;
 321
 322   /**
 323    * Number of find peer requests sent last time around.
 324    */
 325   unsigned int last_sent;
 326
 327   /**
 328    * Hashmap of peers in the current topology, value
 329    * is a PeerCount, with the number of connections
 330    * this peer has.
 331    */
 332   struct GNUNET_CONTAINER_MultiHashMap *peer_hash;
 333
 334   /**
 335    * Min heap which orders values in the peer_hash for
 336    * easy lookup.
 337    */
 338   struct GNUNET_CONTAINER_Heap *peer_min_heap;
 339
 340   /**
 341    * Callback for counting the peers in the current topology.
 342    */
 343   GNUNET_TESTING_NotifyTopology count_peers_cb;
 344 };
 345
 346 enum DHT_ROUND_TYPES
 347 {
 348   /**
 349    * Next full round (puts + gets).
 350    */
 351   DHT_ROUND_NORMAL,
 352
 353   /**
 354    * Next round of gets.
 355    */
 356   DHT_ROUND_GET,
 357
 358   /**
 359    * Next round of puts.
 360    */
 361   DHT_ROUND_PUT,
 362
 363   /**
 364    * Next round of churn.
 365    */
 366   DHT_ROUND_CHURN
 367 };
 368
 369 /* Globals */
 370
 371 /**
 372  * How long to try to connect two peers.
 373  */
 374 struct GNUNET_TIME_Relative connect_timeout;
 375
 376 /**
 377  * How many times to re-attempt connecting two peers.
 378  */
 379 static unsigned long long connect_attempts;
 380
 381 /**
 382  * Timeout to let all GET requests happen.
 383  */
 384 static struct GNUNET_TIME_Relative all_get_timeout;
 385
 386 /**
 387  * Per get timeout
 388  */
 389 static struct GNUNET_TIME_Relative get_timeout;
 390
 391 /**
 392  * Time to allow for GET requests to be sent to service.
 393  */
 394 static struct GNUNET_TIME_Relative get_delay;
 395
 396 /**
 397  * Time to allow for PUT requests to be sent to service.
 398  */
 399 static struct GNUNET_TIME_Relative put_delay;
 400
 401 /**
 402  * Delay between sending find peer requests (if
 403  * handled by the driver, no effect if sent by service).
 404  */
 405 static struct GNUNET_TIME_Relative find_peer_delay;
 406
 407 /**
 408  * Time between find peer requests
 409  * (find_peer_delay / max_outstanding_find_peer)
 410  */
 411 static struct GNUNET_TIME_Relative find_peer_offset;
 412
 413 /**
 414  * How many seconds to allow each peer to start.
 415  */
 416 static struct GNUNET_TIME_Relative seconds_per_peer_start;
 417
 418 /**
 419  * At what time did we start the connection process.
 420  */
 421 static struct GNUNET_TIME_Absolute connect_start_time;
 422
 423 /**
 424  * What was the last time we updated connection/second information.
 425  */
 426 static struct GNUNET_TIME_Absolute connect_last_time;
 427
 428 /**
 429  * At what time did we start the hostkey creation process.
 430  */
 431 static struct GNUNET_TIME_Absolute hostkey_start_time;
 432
 433 /**
 434  * At what time did we start the peer startup process.
 435  */
 436 static struct GNUNET_TIME_Absolute peer_start_time;
 437
 438 /**
 439  * Boolean value, should the driver issue find peer requests
 440  * (GNUNET_YES) or should it be left to the service (GNUNET_NO)
 441  */
 442 static unsigned int do_find_peer;
 443
 444 /**
 445  * Whether or not to insert gauger data.
 446  */
 447 static unsigned int insert_gauger_data;
 448
 449 #if ONLY_TESTING
 450 /**
 451  * Are we currently trying to connect two peers repeatedly?
 452  */
 453 static unsigned int repeat_connect_mode;
 454
 455 /**
 456  * Task for repeating connects.
 457  */
 458 GNUNET_SCHEDULER_TaskIdentifier repeat_connect_task;
 459
 460 struct GNUNET_TESTING_Daemon *repeat_connect_peer1;
 461 struct GNUNET_TESTING_Daemon *repeat_connect_peer2;
 462 #endif
 463
 464 /**
 465  * Boolean value, should replication be done by the dht
 466  * service (GNUNET_YES) or by the driver (GNUNET_NO)
 467  */
 468 static unsigned int in_dht_replication;
 469
 470 /**
 471  * Size of test data to insert/retrieve during testing.
 472  */
 473 static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE;
 474
 475 /**
 476  * Maximum number of concurrent connections to peers.
 477  */
 478 static unsigned long long max_outstanding_connections;
 479
 480 /**
 481  * Maximum number of concurrent ssh instances to peers.
 482  */
 483 static unsigned long long max_concurrent_ssh;
 484
 485 /**
 486  * Maximum number of concurrent PUT requests.
 487  */
 488 static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
 489
 490 /**
 491  * Maximum number of concurrent GET requests.
 492  */
 493 static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
 494
 495 /**
 496  * Number of nodes issuing malicious GET messages.
 497  */
 498 static unsigned long long malicious_getters;
 499
 500 /**
 501  * Maximum number of concurrent find peer messages being sent.
 502  */
 503 static unsigned long long max_outstanding_find_peers;
 504
 505 /**
 506  * Number of nodes issuing malicious PUT messages.
 507  */
 508 static unsigned long long malicious_putters;
 509
 510 /**
 511  * Time (in seconds) to delay between rounds.
 512  */
 513 static unsigned long long round_delay;
 514
 515 /**
 516  * The identifier for this trial (if we have one)
 517  * for external data collection.
 518  */
 519 static unsigned long long trial_to_run;
 520
 521 /**
 522  * How many malicious droppers to seed in the network.
 523  */
 524 static unsigned long long malicious_droppers;
 525
 526 /**
 527  * Bloom filter to restrict malicious nodes chosen.
 528  */
 529 struct GNUNET_CONTAINER_BloomFilter *malicious_bloom;
 530
 531 /**
 532  * Whether malicious droppers should be chosen based on proximity to a key.
 533  */
 534 static int malicious_sybil;
 535
 536 /**
 537  * Target for the malicious sybil nodes (choose the closest to this key).
 538  */
 539 static GNUNET_HashCode sybil_target;
 540
 541 /**
 542  * How often to send malicious GET messages.
 543  */
 544 static struct GNUNET_TIME_Relative malicious_get_frequency;
 545
 546 /**
 547  * How often to send malicious PUT messages.
 548  */
 549 static struct GNUNET_TIME_Relative malicious_put_frequency;
 550
 551 /**
 552  * How long to send find peer requests.
 553  */
 554 static unsigned long long settle_time;
 555
 556 /**
 557  * Handle to the dhtlog service.
 558  */
 559 static struct GNUNET_DHTLOG_Handle *dhtlog_handle;
 560
 561 /**
 562  * Replication value for GET requests.
 563  */
 564 static unsigned long long get_replication;
 565
 566 /**
 567  * Replication value for PUT requests.
 568  */
 569 static unsigned long long put_replication;
 570
 571 /**
 572  * If GNUNET_YES, insert data at the same peers every time.
 573  * Otherwise, choose a new random peer to insert at each time.
 574  */
 575 static unsigned int replicate_same;
 576
 577 /**
 578  * If GNUNET_YES, issue GET requests at the same peers every time.
 579  * Otherwise, choose a new random peer/data combination to search
 580  * each time.
 581  */
 582 static unsigned int get_from_same;
 583
 584 /**
 585  * Should malicious peers be set after allowing for settle time?
 586  * Default is to set them malicious after initial connection setup.
 587  */
 588 static unsigned int malicious_after_settle;
 589
 590 /**
 591  * Number of rounds for testing (PUTS + GETS)
 592  */
 593 static unsigned long long total_rounds;
 594
 595 /**
 596  * Target number of connections (will stop sending find peer
 597  * messages when this number is exceeded)
 598  */
 599 static unsigned long long target_total_connections;
 600
 601 /**
 602  * Number of rounds already run
 603  */
 604 static unsigned int rounds_finished;
 605
 606 /**
 607  * Number of rounds of churn to read from the file (first line, should be a single number).
 608  */
 609 static unsigned int churn_rounds;
 610
 611 /**
 612  * Current round we are in for churn, tells us how many peers to connect/disconnect.
 613  */
 614 static unsigned int current_churn_round;
 615
 616 /**
 617  * Number of times to churn per round
 618  */
 619 static unsigned long long churns_per_round;
 620
 621 /**
 622  * Array of churn values.
 623  */
 624 static unsigned int *churn_array;
 625
 626 /**
 627  * Hash map of stats contexts.
 628  */
 629 static struct GNUNET_CONTAINER_MultiHashMap *stats_map;
 630
 631 /**
 632  * LL of malicious settings.
 633  */
 634 struct MaliciousContext *all_malicious;
 635
 636 /**
 637  * List of GETS to perform
 638  */
 639 struct TestGetContext *all_gets;
 640
 641 /**
 642  * List of PUTS to perform
 643  */
 644 struct TestPutContext *all_puts;
 645
 646 /**
 647  * Directory to store temporary data in, defined in config file
 648  */
 649 static char *test_directory;
 650
 651 /**
 652  * Variable used to store the number of connections we should wait for.
 653  */
 654 static unsigned int expected_connections;
 655
 656 /**
 657  * Variable used to keep track of how many peers aren't yet started.
 658  */
 659 static unsigned long long peers_left;
 660
 661 /**
 662  * Handle to the set of all peers run for this test.
 663  */
 664 static struct GNUNET_TESTING_PeerGroup *pg;
 665
 666 /**
 667  * Global config handle.
 668  */
 669 static const struct GNUNET_CONFIGURATION_Handle *config;
 670
 671 /**
 672  * Total number of peers to run, set based on config file.
 673  */
 674 static unsigned long long num_peers;
 675
 676 /**
 677  * Total number of items to insert.
 678  */
 679 static unsigned long long num_puts;
 680
 681 /**
 682  * How many puts do we currently have in flight?
 683  */
 684 static unsigned long long outstanding_puts;
 685
 686 /**
 687  * How many puts are done?
 688  */
 689 static unsigned long long puts_completed;
 690
 691 /**
 692  * Total number of items to attempt to get.
 693  */
 694 static unsigned long long num_gets;
 695
 696 /**
 697  * How many puts do we currently have in flight?
 698  */
 699 static unsigned long long outstanding_gets;
 700
 701 /**
 702  * How many gets are done?
 703  */
 704 static unsigned long long gets_completed;
 705
 706 /**
 707  * If non-zero, end testing if this many GETs
 708  * complete in a single round.
 709  */
 710 static unsigned long long target_completions;
 711
 712 /**
 713  * Total number of items to attempt to get.
 714  */
 715 static unsigned long long cumulative_num_gets;
 716
 717 /**
 718  * How many gets are done?
 719  */
 720 static unsigned long long cumulative_successful_gets;
 721
 722 /**
 723  * How many gets failed?
 724  */
 725 static unsigned long long gets_failed;
 726
 727 /**
 728  * How many malicious control messages do
 729  * we currently have in flight?
 730  */
 731 static unsigned long long outstanding_malicious;
 732
 733 /**
 734  * How many set malicious peers are done?
 735  */
 736 static unsigned int malicious_completed;
 737
 738 /**
 739  * For gauger logging, what specific identifier (svn revision)
 740  * should be used?
 741  */
 742 static unsigned long long revision;
 743
 744 /**
 745  * Global used to count how many connections we have currently
 746  * been notified about (how many times has topology_callback been called
 747  * with success?)
 748  */
 749 static uint64_t total_connections;
 750
 751 /**
 752  * Previous connections, for counting new connections during some duration.
 753  */
 754 static uint64_t previous_connections;
 755
 756 /**
 757  * For counting failed connections during some duration.
 758  */
 759 static uint64_t previous_failed_connections;
 760
 761 /**
 762  * Global used to count how many failed connections we have
 763  * been notified about (how many times has topology_callback
 764  * been called with failure?)
 765  */
 766 static uint64_t failed_connections;
 767
 768 /**
 769  * If GNUNET_YES, only log PUT/GET round data to mysql, otherwise
 770  * log everything (including each dht service logging).
 771  */
 772 static unsigned int dhtlog_minimal;
 773
 774 /* Task handle to use to schedule shutdown if something goes wrong */
 775 GNUNET_SCHEDULER_TaskIdentifier die_task;
 776
 777 static char *blacklist_transports;
 778
 779 static enum GNUNET_TESTING_Topology topology;
 780
 781 static enum GNUNET_TESTING_Topology blacklist_topology =
 782     GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 783
 784 static enum GNUNET_TESTING_Topology connect_topology =
 785     GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 786
 787 static enum GNUNET_TESTING_TopologyOption connect_topology_option =
 788     GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 789
 790 static double connect_topology_option_modifier = 0.0;
 791
 792 static struct ProgressMeter *hostkey_meter;
 793
 794 static struct ProgressMeter *peer_start_meter;
 795
 796 static struct ProgressMeter *peer_connect_meter;
 797
 798 static struct ProgressMeter *put_meter;
 799
 800 static struct ProgressMeter *get_meter;
 801
 802 static GNUNET_HashCode *known_keys;
 803
 804 /* Global return value (0 for success, anything else for failure) */
 805 static int ok;
 806
 807 /**
 808  * Create a meter to keep track of the progress of some task.
 809  *
 810  * @param total the total number of items to complete
 811  * @param start_string a string to prefix the meter with (if printing)
 812  * @param print GNUNET_YES to print the meter, GNUNET_NO to count
 813  *              internally only
 814  *
 815  * @return the progress meter
 816  */
 817 static struct ProgressMeter *
 818 create_meter(unsigned int total, char * start_string, int print)
 819 {
 820   struct ProgressMeter *ret;
 821   ret = GNUNET_malloc(sizeof(struct ProgressMeter));
 822   ret->print = print;
 823   ret->total = total;
 824   ret->modnum = total / 4;
 825   ret->dotnum = (total / 50) + 1;
 826   if (start_string != NULL)
 827     ret->startup_string = GNUNET_strdup(start_string);
 828   else
 829     ret->startup_string = GNUNET_strdup("");
 830
 831   return ret;
 832 }
 833
 834 /**
 835  * Update progress meter (increment by one).
 836  *
 837  * @param meter the meter to update and print info for
 838  *
 839  * @return GNUNET_YES if called the total requested,
 840  *         GNUNET_NO if more items expected
 841  */
 842 static int
 843 update_meter(struct ProgressMeter *meter)
 844 {
 845   if (meter->print == GNUNET_YES)
 846     {
 847       if (meter->completed % meter->modnum == 0)
 848         {
 849           if (meter->completed == 0)
 850             {
 851               fprintf (stdout, "%sProgress: [0%%", meter->startup_string);
 852             }
 853           else
 854             fprintf (stdout, "%d%%", (int) (((float) meter->completed
 855                 / meter->total) * 100));
 856         }
 857       else if (meter->completed % meter->dotnum == 0)
 858         fprintf (stdout, ".");
 859
 860       if (meter->completed + 1 == meter->total)
 861         fprintf (stdout, "%d%%]\n", 100);
 862       fflush (stdout);
 863     }
 864   meter->completed++;
 865
 866   if (meter->completed == meter->total)
 867     return GNUNET_YES;
 868   return GNUNET_NO;
 869 }
 870
 871 /**
 872  * Reset progress meter.
 873  *
 874  * @param meter the meter to reset
 875  *
 876  * @return GNUNET_YES if meter reset,
 877  *         GNUNET_SYSERR on error
 878  */
 879 static int
 880 reset_meter(struct ProgressMeter *meter)
 881 {
 882   if (meter == NULL)
 883     return GNUNET_SYSERR;
 884
 885   meter->completed = 0;
 886   return GNUNET_YES;
 887 }
 888
 889 /**
 890  * Release resources for meter
 891  *
 892  * @param meter the meter to free
 893  */
 894 static void
 895 free_meter(struct ProgressMeter *meter)
 896 {
 897   GNUNET_free_non_null (meter->startup_string);
 898   GNUNET_free (meter);
 899 }
 900
 901 /**
 902  * Check whether peers successfully shut down.
 903  */
 904 static void
 905 shutdown_callback(void *cls, const char *emsg)
 906 {
 907   if (emsg != NULL)
 908     {
 909       if (ok == 0)
 910         ok = 2;
 911     }
 912 }
 913
 914 /**
 915  * Task to release DHT handles for PUT
 916  */
 917 static void
 918 put_disconnect_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 919 {
 920   struct TestPutContext *test_put = cls;
 921   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 922   GNUNET_DHT_disconnect (test_put->dht_handle);
 923   test_put->dht_handle = NULL;
 924   if (replicate_same == GNUNET_NO)
 925     test_put->daemon
 926         = GNUNET_TESTING_daemon_get (
 927                                      pg,
 928                                      GNUNET_CRYPTO_random_u32 (
 929                                                                GNUNET_CRYPTO_QUALITY_WEAK,
 930                                                                num_peers));
 931 }
 932
 933 /**
 934  * Function scheduled to be run on the successful completion of this
 935  * testcase.
 936  */
 937 static void
 938 finish_testing(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 939 {
 940   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Ending test normally!\n",
 941               (char *) cls);
 942   GNUNET_assert (pg != NULL);
 943   struct TestPutContext *test_put = all_puts;
 944   struct TestGetContext *test_get = all_gets;
 945   char *temp_get_string;
 946   char *revision_str;
 947
 948   while (test_put != NULL)
 949     {
 950       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 951         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
 952       if (test_put->dht_handle != NULL)
 953         GNUNET_DHT_disconnect (test_put->dht_handle);
 954       test_put = test_put->next;
 955     }
 956
 957   while (test_get != NULL)
 958     {
 959       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 960         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
 961       if (test_get->get_handle != NULL)
 962         GNUNET_DHT_get_stop (test_get->get_handle);
 963       if (test_get->dht_handle != NULL)
 964         GNUNET_DHT_disconnect (test_get->dht_handle);
 965       test_get = test_get->next;
 966     }
 967
 968   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
 969
 970   if (dhtlog_handle != NULL)
 971     {
 972       fprintf (stderr, "Update trial endtime\n");
 973       dhtlog_handle->update_trial (cumulative_successful_gets);
 974       GNUNET_DHTLOG_disconnect (dhtlog_handle);
 975       dhtlog_handle = NULL;
 976     }
 977
 978   if (hostkey_meter != NULL)
 979     free_meter (hostkey_meter);
 980   if (peer_start_meter != NULL)
 981     free_meter (peer_start_meter);
 982   if (peer_connect_meter != NULL)
 983     free_meter (peer_connect_meter);
 984   if (put_meter != NULL)
 985     free_meter (put_meter);
 986   if (get_meter != NULL)
 987     free_meter (get_meter);
 988
 989   GNUNET_asprintf (&temp_get_string, "DHT Successful GETs", trial_to_run);
 990   GNUNET_asprintf (&revision_str, "%llu", revision);
 991   if (GNUNET_YES == insert_gauger_data)
 992     GAUGER_ID("DHT_TESTING", temp_get_string, cumulative_successful_gets / (double)cumulative_num_gets, "percent successful", revision_str);
 993   fprintf (
 994            stderr,
 995            "Finished trial, had %llu successful gets out of %llu total, %.2f percent succeeded\n",
 996            cumulative_successful_gets, cumulative_num_gets,
 997            cumulative_successful_gets / (double) cumulative_num_gets);
 998   GNUNET_free(temp_get_string);
 999
1000   ok = 0;
1001 }
1002
1003 /**
1004  * Callback for iterating over all the peer connections of a peer group.
1005  */
1006 static void
1007 log_topology_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1008                 const struct GNUNET_PeerIdentity *second, const char *emsg)
1009 {
1010   struct TopologyIteratorContext *topo_ctx = cls;
1011   if ((first != NULL) && (second != NULL))
1012     {
1013       if ((topo_ctx->peers_seen != NULL) && (GNUNET_NO
1014           == GNUNET_CONTAINER_multihashmap_contains (topo_ctx->peers_seen,
1015                                                      &first->hashPubKey)))
1016         {
1017           GNUNET_CONTAINER_multihashmap_put (topo_ctx->peers_seen,
1018                                              &first->hashPubKey, NULL,
1019                                              GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1020           topo_ctx->total_peers++;
1021         }
1022       topo_ctx->total_connections++;
1023       if ((GNUNET_NO == dhtlog_minimal) && (dhtlog_handle != NULL))
1024         dhtlog_handle->insert_extended_topology (first, second);
1025     }
1026   else
1027     {
1028       GNUNET_assert(dhtlog_handle != NULL);
1029       GNUNET_log (
1030                   GNUNET_ERROR_TYPE_WARNING,
1031                   "Topology iteration (%u/%u) finished (%u connections, %u peers)\n",
1032                   topo_ctx->current_iteration, topo_ctx->total_iterations,
1033                   topo_ctx->total_connections, topo_ctx->total_peers);
1034       dhtlog_handle->update_topology (topo_ctx->total_connections);
1035       if (topo_ctx->cont != NULL)
1036         GNUNET_SCHEDULER_add_now (topo_ctx->cont, topo_ctx->cls);
1037       if (topo_ctx->peers_seen != NULL)
1038         GNUNET_CONTAINER_multihashmap_destroy (topo_ctx->peers_seen);
1039       GNUNET_free(topo_ctx);
1040     }
1041 }
1042
1043 /**
1044  * Iterator over hash map entries.
1045  *
1046  * @param cls closure - always NULL
1047  * @param key current key code
1048  * @param value value in the hash map, a stats context
1049  * @return GNUNET_YES if we should continue to
1050  *         iterate,
1051  *         GNUNET_NO if not.
1052  */
1053 static int
1054 stats_iterate(void *cls, const GNUNET_HashCode * key, void *value)
1055 {
1056   struct StatisticsIteratorContext *stats_ctx;
1057   if (value == NULL)
1058     return GNUNET_NO;
1059   stats_ctx = value;
1060   dhtlog_handle->insert_stat (stats_ctx->peer, stats_ctx->stat_routes,
1061                               stats_ctx->stat_route_forwards,
1062                               stats_ctx->stat_results,
1063                               stats_ctx->stat_results_to_client,
1064                               stats_ctx->stat_result_forwards,
1065                               stats_ctx->stat_gets, stats_ctx->stat_puts,
1066                               stats_ctx->stat_puts_inserted,
1067                               stats_ctx->stat_find_peer,
1068                               stats_ctx->stat_find_peer_start,
1069                               stats_ctx->stat_get_start,
1070                               stats_ctx->stat_put_start,
1071                               stats_ctx->stat_find_peer_reply,
1072                               stats_ctx->stat_get_reply,
1073                               stats_ctx->stat_find_peer_answer,
1074                               stats_ctx->stat_get_response_start);
1075   GNUNET_free(stats_ctx);
1076   return GNUNET_YES;
1077 }
1078
1079 static void
1080 stats_finished(void *cls, int result)
1081 {
1082   fprintf (stderr, "Finished getting all peers statistics, iterating!\n");
1083   GNUNET_CONTAINER_multihashmap_iterate (stats_map, &stats_iterate, NULL);
1084   GNUNET_CONTAINER_multihashmap_destroy (stats_map);
1085   GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1086 }
1087
1088 /**
1089  * Callback function to process statistic values.
1090  *
1091  * @param cls closure
1092  * @param peer the peer the statistics belong to
1093  * @param subsystem name of subsystem that created the statistic
1094  * @param name the name of the datum
1095  * @param value the current value
1096  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
1097  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
1098  */
1099 static int
1100 stats_handle(void *cls, const struct GNUNET_PeerIdentity *peer,
1101              const char *subsystem, const char *name, uint64_t value,
1102              int is_persistent)
1103 {
1104   struct StatisticsIteratorContext *stats_ctx;
1105
1106   if (dhtlog_handle != NULL)
1107     dhtlog_handle->add_generic_stat (peer, name, subsystem, value);
1108   if (GNUNET_CONTAINER_multihashmap_contains (stats_map, &peer->hashPubKey))
1109     {
1110       stats_ctx = GNUNET_CONTAINER_multihashmap_get (stats_map,
1111                                                      &peer->hashPubKey);
1112     }
1113   else
1114     {
1115       stats_ctx = GNUNET_malloc(sizeof(struct StatisticsIteratorContext));
1116       stats_ctx->peer = peer;
1117       GNUNET_CONTAINER_multihashmap_put (stats_map, &peer->hashPubKey,
1118                                          stats_ctx,
1119                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1120     }
1121   GNUNET_assert(stats_ctx != NULL);
1122
1123   if (strcmp (name, STAT_ROUTES) == 0)
1124     stats_ctx->stat_routes = value;
1125   else if (strcmp (name, STAT_ROUTE_FORWARDS) == 0)
1126     stats_ctx->stat_route_forwards = value;
1127   else if (strcmp (name, STAT_RESULTS) == 0)
1128     stats_ctx->stat_results = value;
1129   else if (strcmp (name, STAT_RESULTS_TO_CLIENT) == 0)
1130     stats_ctx->stat_results_to_client = value;
1131   else if (strcmp (name, STAT_RESULT_FORWARDS) == 0)
1132     stats_ctx->stat_result_forwards = value;
1133   else if (strcmp (name, STAT_GETS) == 0)
1134     stats_ctx->stat_gets = value;
1135   else if (strcmp (name, STAT_PUTS) == 0)
1136     stats_ctx->stat_puts = value;
1137   else if (strcmp (name, STAT_PUTS_INSERTED) == 0)
1138     stats_ctx->stat_puts_inserted = value;
1139   else if (strcmp (name, STAT_FIND_PEER) == 0)
1140     stats_ctx->stat_find_peer = value;
1141   else if (strcmp (name, STAT_FIND_PEER_START) == 0)
1142     stats_ctx->stat_find_peer_start = value;
1143   else if (strcmp (name, STAT_GET_START) == 0)
1144     stats_ctx->stat_get_start = value;
1145   else if (strcmp (name, STAT_PUT_START) == 0)
1146     stats_ctx->stat_put_start = value;
1147   else if (strcmp (name, STAT_FIND_PEER_REPLY) == 0)
1148     stats_ctx->stat_find_peer_reply = value;
1149   else if (strcmp (name, STAT_GET_REPLY) == 0)
1150     stats_ctx->stat_get_reply = value;
1151   else if (strcmp (name, STAT_FIND_PEER_ANSWER) == 0)
1152     stats_ctx->stat_find_peer_answer = value;
1153   else if (strcmp (name, STAT_GET_RESPONSE_START) == 0)
1154     stats_ctx->stat_get_response_start = value;
1155
1156   return GNUNET_OK;
1157 }
1158
1159 /**
1160  * Connect to statistics service for each peer and get the appropriate
1161  * dht statistics for safe keeping.
1162  */
1163 static void
1164 log_dht_statistics(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1165 {
1166   stats_map = GNUNET_CONTAINER_multihashmap_create (num_peers);
1167   fprintf (stderr, "Starting statistics logging\n");
1168   GNUNET_TESTING_get_statistics (pg, &stats_finished, &stats_handle, NULL);
1169 }
1170
1171 /**
1172  * Connect to all peers in the peer group and iterate over their
1173  * connections.
1174  */
1175 static void
1176 capture_current_topology(void *cls,
1177                          const struct GNUNET_SCHEDULER_TaskContext * tc)
1178 {
1179   struct TopologyIteratorContext *topo_ctx = cls;
1180   dhtlog_handle->insert_topology (0);
1181   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Called capture_current_topology\n");
1182   GNUNET_TESTING_get_topology (pg, &log_topology_cb, topo_ctx);
1183 }
1184
1185 /**
1186  * Check if the get_handle is being used, if so stop the request.  Either
1187  * way, schedule the end_badly_cont function which actually shuts down the
1188  * test.
1189  */
1190 static void
1191 end_badly(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1192 {
1193   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failing test with error: `%s'!\n",
1194               (char *) cls);
1195
1196   struct TestPutContext *test_put = all_puts;
1197   struct TestGetContext *test_get = all_gets;
1198
1199   while (test_put != NULL)
1200     {
1201       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1202         GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
1203       if (test_put->dht_handle != NULL)
1204         GNUNET_DHT_disconnect (test_put->dht_handle);
1205       test_put = test_put->next;
1206     }
1207
1208   while (test_get != NULL)
1209     {
1210       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
1211         GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
1212       if (test_get->get_handle != NULL)
1213         GNUNET_DHT_get_stop (test_get->get_handle);
1214       if (test_get->dht_handle != NULL)
1215         GNUNET_DHT_disconnect (test_get->dht_handle);
1216       test_get = test_get->next;
1217     }
1218
1219   GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL);
1220
1221   if (dhtlog_handle != NULL)
1222     {
1223       fprintf (stderr, "Update trial endtime\n");
1224       dhtlog_handle->update_trial (gets_completed);
1225       GNUNET_DHTLOG_disconnect (dhtlog_handle);
1226       dhtlog_handle = NULL;
1227     }
1228
1229   if (hostkey_meter != NULL)
1230     free_meter (hostkey_meter);
1231   if (peer_start_meter != NULL)
1232     free_meter (peer_start_meter);
1233   if (peer_connect_meter != NULL)
1234     free_meter (peer_connect_meter);
1235   if (put_meter != NULL)
1236     free_meter (put_meter);
1237   if (get_meter != NULL)
1238     free_meter (get_meter);
1239
1240   ok = 1;
1241 }
1242
1243 /**
1244  * Forward declaration.
1245  */
1246 static void
1247 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1248
1249 /**
1250  * Forward declaration.
1251  */
1252 static void
1253 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc);
1254
1255 /**
1256  * Iterator over hash map entries.
1257  *
1258  * @param cls closure
1259  * @param key current key code
1260  * @param value value in the hash map
1261  * @return GNUNET_YES if we should continue to
1262  *         iterate,
1263  *         GNUNET_NO if not.
1264  */
1265 static int
1266 remove_peer_count(void *cls, const GNUNET_HashCode * key, void *value)
1267 {
1268   struct PeerCount *peer_count = value;
1269   GNUNET_CONTAINER_heap_remove_node (peer_count->heap_node);
1270   GNUNET_free(peer_count);
1271
1272   return GNUNET_YES;
1273 }
1274
1275 /**
1276  * Connect to all peers in the peer group and iterate over their
1277  * connections.
1278  */
1279 static void
1280 count_new_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1281 {
1282   struct FindPeerContext *find_peer_context = cls;
1283   find_peer_context->previous_peers = find_peer_context->current_peers;
1284   find_peer_context->current_peers = 0;
1285   GNUNET_TESTING_get_topology (pg, find_peer_context->count_peers_cb,
1286                                find_peer_context);
1287 }
1288
1289 static void
1290 decrement_find_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1291 {
1292   struct TestFindPeer *test_find_peer = cls;
1293   GNUNET_assert(test_find_peer->find_peer_context->outstanding > 0);
1294   test_find_peer->find_peer_context->outstanding--;
1295   test_find_peer->find_peer_context->total--;
1296   if (0 == test_find_peer->find_peer_context->total)
1297     {
1298       GNUNET_SCHEDULER_add_now (&count_new_peers,
1299                                 test_find_peer->find_peer_context);
1300     }
1301   GNUNET_free(test_find_peer);
1302 }
1303
1304 /**
1305  * A find peer request has been sent to the server, now we will schedule a task
1306  * to wait the appropriate time to allow the request to go out and back.
1307  *
1308  * @param cls closure - a TestFindPeer struct
1309  * @param tc context the task is being called with
1310  */
1311 static void
1312 handle_find_peer_sent(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1313 {
1314   struct TestFindPeer *test_find_peer = cls;
1315
1316   GNUNET_DHT_disconnect (test_find_peer->dht_handle);
1317   GNUNET_SCHEDULER_add_delayed (
1318                                 GNUNET_TIME_relative_divide (find_peer_delay, 2),
1319                                 &decrement_find_peers, test_find_peer);
1320 }
1321
1322 static void
1323 send_find_peer_request(void *cls,
1324                        const struct GNUNET_SCHEDULER_TaskContext * tc)
1325 {
1326   struct TestFindPeer *test_find_peer = cls;
1327
1328   if (test_find_peer->find_peer_context->outstanding
1329       > max_outstanding_find_peers)
1330     {
1331       GNUNET_SCHEDULER_add_delayed (find_peer_offset, &send_find_peer_request,
1332                                     test_find_peer);
1333       return;
1334     }
1335
1336   test_find_peer->find_peer_context->outstanding++;
1337   if (GNUNET_TIME_absolute_get_remaining (
1338                                           test_find_peer->find_peer_context->endtime).rel_value
1339       == 0)
1340     {
1341       GNUNET_SCHEDULER_add_now (&decrement_find_peers, test_find_peer);
1342       return;
1343     }
1344
1345   test_find_peer->dht_handle = GNUNET_DHT_connect (test_find_peer->daemon->cfg,
1346                                                    1);
1347   GNUNET_assert(test_find_peer->dht_handle != NULL);
1348   GNUNET_DHT_find_peers (test_find_peer->dht_handle, &handle_find_peer_sent,
1349                          test_find_peer);
1350 }
1351
1352 /**
1353  * Add a connection to the find_peer_context given.  This may
1354  * be complete overkill, but allows us to choose the peers with
1355  * the least connections to initiate find peer requests from.
1356  */
1357 static void
1358 add_new_connection(struct FindPeerContext *find_peer_context,
1359                    const struct GNUNET_PeerIdentity *first,
1360                    const struct GNUNET_PeerIdentity *second)
1361 {
1362   struct PeerCount *first_count;
1363   struct PeerCount *second_count;
1364
1365   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1366                                               &first->hashPubKey))
1367     {
1368       first_count
1369           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1370                                                &first->hashPubKey);
1371       GNUNET_assert(first_count != NULL);
1372       first_count->count++;
1373       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1374                                          first_count->heap_node,
1375                                          first_count->count);
1376     }
1377   else
1378     {
1379       first_count = GNUNET_malloc(sizeof(struct PeerCount));
1380       first_count->count = 1;
1381       memcpy (&first_count->peer_id, first, sizeof(struct GNUNET_PeerIdentity));
1382       first_count->heap_node
1383           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1384                                           first_count, first_count->count);
1385       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1386                                          &first->hashPubKey, first_count,
1387                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1388     }
1389
1390   if (GNUNET_CONTAINER_multihashmap_contains (find_peer_context->peer_hash,
1391                                               &second->hashPubKey))
1392     {
1393       second_count
1394           = GNUNET_CONTAINER_multihashmap_get (find_peer_context->peer_hash,
1395                                                &second->hashPubKey);
1396       GNUNET_assert(second_count != NULL);
1397       second_count->count++;
1398       GNUNET_CONTAINER_heap_update_cost (find_peer_context->peer_min_heap,
1399                                          second_count->heap_node,
1400                                          second_count->count);
1401     }
1402   else
1403     {
1404       second_count = GNUNET_malloc(sizeof(struct PeerCount));
1405       second_count->count = 1;
1406       memcpy (&second_count->peer_id, second,
1407               sizeof(struct GNUNET_PeerIdentity));
1408       second_count->heap_node
1409           = GNUNET_CONTAINER_heap_insert (find_peer_context->peer_min_heap,
1410                                           second_count, second_count->count);
1411       GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1412                                          &second->hashPubKey, second_count,
1413                                          GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1414     }
1415 }
1416
1417 /**
1418  * Iterate over min heap of connections per peer.  For any
1419  * peer that has 0 connections, attempt to connect them to
1420  * some random peer.
1421  *
1422  * @param cls closure a struct FindPeerContext
1423  * @param node internal node of the heap
1424  * @param element value stored, a struct PeerCount
1425  * @param cost cost associated with the node
1426  * @return GNUNET_YES if we should continue to iterate,
1427  *         GNUNET_NO if not.
1428  */
1429 static int
1430 iterate_min_heap_peers(void *cls, struct GNUNET_CONTAINER_HeapNode *node,
1431                        void *element, GNUNET_CONTAINER_HeapCostType cost)
1432 {
1433   struct FindPeerContext *find_peer_context = cls;
1434   struct PeerCount *peer_count = element;
1435   struct GNUNET_TESTING_Daemon *d1;
1436   struct GNUNET_TESTING_Daemon *d2;
1437   struct GNUNET_TIME_Relative timeout;
1438   if (cost == 0)
1439     {
1440       d1 = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1441       GNUNET_assert(d1 != NULL);
1442       d2 = d1;
1443       while ((d2 == d1) || (GNUNET_YES != GNUNET_TESTING_daemon_running (d2)))
1444         {
1445           d2
1446               = GNUNET_TESTING_daemon_get (
1447                                            pg,
1448                                            GNUNET_CRYPTO_random_u32 (
1449                                                                      GNUNET_CRYPTO_QUALITY_WEAK,
1450                                                                      num_peers));
1451           GNUNET_assert(d2 != NULL);
1452         }
1453
1454       /** Just try to connect the peers, don't worry about callbacks, etc. **/
1455       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1456                   "Peer %s has 0 connections.  Trying to connect to %s...\n",
1457                   GNUNET_i2s (&peer_count->peer_id), d2->shortname);
1458       timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
1459                                                DEFAULT_CONNECT_TIMEOUT);
1460       if (GNUNET_TIME_relative_to_absolute (timeout).abs_value
1461           > find_peer_context->endtime.abs_value)
1462         {
1463           timeout
1464               = GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime);
1465         }
1466       GNUNET_TESTING_daemons_connect (d1, d2, timeout,
1467                                       DEFAULT_RECONNECT_ATTEMPTS, GNUNET_YES,
1468                                       NULL, NULL);
1469     }
1470   if (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1471       > 0)
1472     return GNUNET_YES;
1473   else
1474     return GNUNET_NO;
1475 }
1476
1477 /**
1478  * Forward declaration.
1479  */
1480 static void
1481     schedule_churn_find_peer_requests(
1482                                       void *cls,
1483                                       const struct GNUNET_SCHEDULER_TaskContext * tc);
1484
1485 /**
1486  * Callback for iterating over all the peer connections of a peer group.
1487  * Used after we have churned on some peers to find which ones have zero
1488  * connections so we can make them issue find peer requests.
1489  */
1490 static void
1491 count_peers_churn_cb(void *cls, const struct GNUNET_PeerIdentity *first,
1492                      const struct GNUNET_PeerIdentity *second, const char *emsg)
1493 {
1494   struct FindPeerContext *find_peer_context = cls;
1495   struct TopologyIteratorContext *topo_ctx;
1496   struct PeerCount *peer_count;
1497
1498   if ((first != NULL) && (second != NULL))
1499     {
1500       add_new_connection (find_peer_context, first, second);
1501       find_peer_context->current_peers++;
1502     }
1503   else
1504     {
1505       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1506                   "Peer count finished (%u connections)\n",
1507                   find_peer_context->current_peers);
1508       peer_count
1509           = GNUNET_CONTAINER_heap_peek (find_peer_context->peer_min_heap);
1510       GNUNET_assert(peer_count != NULL);
1511       /* WAIT. When peers are churned they will come back with their peers (at least in peerinfo), because the HOSTS file doesn't likely get removed. CRAP. */
1512       /* NO they won't, because we have disabled peerinfo writing to disk (remember?) so we WILL have to give them new connections */
1513       /* Best course of action: have DHT automatically try to add peers from peerinfo on startup. This way IF peerinfo writes to file
1514        * then some peers will end up connected.
1515        *
1516        * Also, find any peers that have zero connections here and set up a task to choose at random another peer in the network to
1517        * connect to.  Of course, if they are blacklisted from that peer they won't be able to connect, so we will have to keep trying
1518        * until they get a peer.
1519        */
1520       /* However, they won't automatically be connected to any of their previous peers... How can we handle that? */
1521       /* So now we have choices: do we want them to come back with all their connections?  Probably not, but it solves this mess. */
1522
1523       /* Second problem, which is still a problem, is that a FIND_PEER request won't work when a peer has no connections */
1524
1525       /**
1526        * Okay, so here's how this *should* work now.
1527        *
1528        * 1. We check the min heap for any peers that have 0 connections.
1529        *    a. If any are found, we iterate over the heap and just randomly
1530        *       choose another peer and ask testing to please connect the two.
1531        *       This takes care of the case that a peer just randomly joins the
1532        *       network.  However, if there are strict topology restrictions
1533        *       (imagine a ring) choosing randomly most likely won't help.
1534        *       We make sure the connection attempt doesn't take longer than
1535        *       the total timeout, but don't care too much about the result.
1536        *    b. After that, we still schedule the find peer requests (concurrently
1537        *       with the connect attempts most likely).  This handles the case
1538        *       that the DHT iterates over peerinfo and just needs to try to send
1539        *       a message to get connected.  This should handle the case that the
1540        *       topology is very strict.
1541        *
1542        * 2. If all peers have > 0 connections, we still send find peer requests
1543        *    as long as possible (until timeout is reached) to help out those
1544        *    peers that were newly churned and need more connections.  This is because
1545        *    once all new peers have established a single connection, they won't be
1546        *    well connected.
1547        *
1548        * 3. Once we reach the timeout, we can do no more.  We must schedule the
1549        *    next iteration of get requests regardless of connections that peers
1550        *    may or may not have.
1551        *
1552        * Caveat: it would be nice to get peers to take data offline with them and
1553        *         come back with it (or not) based on the testing framework.  The
1554        *         same goes for remembering previous connections, but putting either
1555        *         into the general testing churn options seems like overkill because
1556        *         these are very specialized cases.
1557        */
1558       GNUNET_log (
1559                   GNUNET_ERROR_TYPE_WARNING,
1560                   "Out of %u peers, fewest connections is %d\n",
1561                   GNUNET_CONTAINER_heap_get_size (
1562                                                   find_peer_context->peer_min_heap),
1563                   peer_count->count);
1564       if ((peer_count->count == 0)
1565           && (GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1566               > 0))
1567         {
1568           GNUNET_log (
1569                       GNUNET_ERROR_TYPE_WARNING,
1570                       "Found peer with no connections, will choose some peer(s) at random to connect to!\n");
1571           GNUNET_CONTAINER_heap_iterate (find_peer_context->peer_min_heap,
1572                                          &iterate_min_heap_peers,
1573                                          find_peer_context);
1574           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1575                                     find_peer_context);
1576         }
1577       else if ((GNUNET_TIME_absolute_get_remaining (find_peer_context->endtime).rel_value
1578           > 0) && (find_peer_context->last_sent != 0))
1579         {
1580           GNUNET_SCHEDULER_add_now (&schedule_churn_find_peer_requests,
1581                                     find_peer_context);
1582         }
1583       else
1584         {
1585           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
1586                                                  &remove_peer_count,
1587                                                  find_peer_context);
1588           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
1589           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
1590           GNUNET_free(find_peer_context);
1591           GNUNET_log (
1592                       GNUNET_ERROR_TYPE_WARNING,
1593                       "Churn round %u of %llu finished, scheduling next GET round.\n",
1594                       current_churn_round, churn_rounds);
1595           if (dhtlog_handle != NULL)
1596             {
1597               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1598               topo_ctx->cont = &do_get;
1599               topo_ctx->cls = all_gets;
1600               topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1601               topo_ctx->peers_seen
1602                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1603               die_task
1604                   = GNUNET_SCHEDULER_add_delayed (
1605                                                   GNUNET_TIME_relative_add (
1606                                                                             GNUNET_TIME_relative_add (
1607                                                                                                       DEFAULT_GET_TIMEOUT,
1608                                                                                                       all_get_timeout),
1609                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1610                                                   &end_badly,
1611                                                   "from do gets (count_peers_churn_cb)");
1612               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1613             }
1614           else
1615             {
1616               die_task
1617                   = GNUNET_SCHEDULER_add_delayed (
1618                                                   GNUNET_TIME_relative_add (
1619                                                                             GNUNET_TIME_relative_add (
1620                                                                                                       DEFAULT_GET_TIMEOUT,
1621                                                                                                       all_get_timeout),
1622                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
1623                                                   &end_badly,
1624                                                   "from do gets (count_peers_churn_cb)");
1625               GNUNET_SCHEDULER_add_now (&do_get, all_gets);
1626             }
1627         }
1628     }
1629 }
1630
1631 /**
1632  * Set up a single find peer request for each peer in the topology.  Do this
1633  * until the settle time is over, limited by the number of outstanding requests
1634  * and the time allowed for each one!
1635  */
1636 static void
1637 schedule_churn_find_peer_requests(
1638                                   void *cls,
1639                                   const struct GNUNET_SCHEDULER_TaskContext * tc)
1640 {
1641   struct FindPeerContext *find_peer_ctx = cls;
1642   struct TestFindPeer *test_find_peer;
1643   struct PeerCount *peer_count;
1644   uint32_t i;
1645
1646   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
1647     find_peer_ctx->total = 1;
1648   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1649       < MIN_FIND_PEER_CUTOFF)
1650     find_peer_ctx->total = find_peer_ctx->total / 2;
1651   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
1652       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
1653     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
1654         / 4);
1655   else
1656     find_peer_ctx->total = find_peer_ctx->last_sent * 4;
1657
1658   if (find_peer_ctx->total > max_outstanding_find_peers)
1659     find_peer_ctx->total = max_outstanding_find_peers;
1660
1661   find_peer_ctx->last_sent = find_peer_ctx->total;
1662   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1663               "Sending %u find peer messages (after churn)\n",
1664               find_peer_ctx->total);
1665
1666   if (find_peer_ctx->total > 0)
1667     find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
1668                                                     find_peer_ctx->total);
1669   else
1670     {
1671       find_peer_ctx->previous_peers = find_peer_ctx->current_peers;
1672       find_peer_ctx->current_peers = 0;
1673       GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_ctx);
1674     }
1675
1676   for (i = 0; i < find_peer_ctx->total; i++)
1677     {
1678       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
1679       /* If we have sent requests, choose peers with a low number of connections to send requests from */
1680       peer_count
1681           = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
1682       GNUNET_assert(peer_count != NULL);
1683       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1684                   "Sending find peer request from peer with %u connections\n",
1685                   peer_count->count);
1686       GNUNET_CONTAINER_multihashmap_remove (find_peer_ctx->peer_hash,
1687                                             &peer_count->peer_id.hashPubKey,
1688                                             peer_count);
1689       test_find_peer->daemon
1690           = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
1691       GNUNET_assert(test_find_peer->daemon != NULL);
1692       test_find_peer->find_peer_context = find_peer_ctx;
1693       GNUNET_SCHEDULER_add_delayed (
1694                                     GNUNET_TIME_relative_multiply (
1695                                                                    find_peer_offset,
1696                                                                    i),
1697                                     &send_find_peer_request, test_find_peer);
1698     }
1699
1700   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
1701       == NULL))
1702     {
1703       find_peer_ctx->peer_hash
1704           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1705       find_peer_ctx->peer_min_heap
1706           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1707     }
1708   else
1709     {
1710       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
1711                                              &remove_peer_count, find_peer_ctx);
1712       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
1713       find_peer_ctx->peer_hash
1714           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1715     }
1716
1717   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
1718   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
1719 }
1720
1721 static void
1722 schedule_churn_get_topology(void *cls,
1723                             const struct GNUNET_SCHEDULER_TaskContext * tc)
1724 {
1725   struct FindPeerContext *find_peer_context = cls;
1726   GNUNET_TESTING_get_topology (pg, &count_peers_churn_cb, find_peer_context);
1727 }
1728
1729 /**
1730  * Called when churning of the topology has finished.
1731  *
1732  * @param cls closure unused
1733  * @param emsg NULL on success, or a printable error on failure
1734  */
1735 static void
1736 churn_complete(void *cls, const char *emsg)
1737 {
1738   struct FindPeerContext *find_peer_context = cls;
1739   struct PeerCount *peer_count;
1740   unsigned int i;
1741   struct GNUNET_TESTING_Daemon *temp_daemon;
1742   struct TopologyIteratorContext *topo_ctx;
1743   struct GNUNET_TIME_Relative calc_timeout;
1744   int count_added;
1745
1746   if (emsg != NULL)
1747     {
1748       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1749                   "Ending test, churning of peers failed with error `%s'", emsg);
1750       GNUNET_SCHEDULER_add_now (&end_badly, (void *) emsg);
1751       return;
1752     }
1753
1754   /**
1755    * If we switched any peers on, we have to somehow force connect the new peer to
1756    * SOME bootstrap peer in the network.  First schedule a task to find all peers
1757    * with no connections, then choose a random peer for each and connect them.
1758    */
1759   if (find_peer_context != NULL)
1760     {
1761       GNUNET_log (
1762                   GNUNET_ERROR_TYPE_WARNING,
1763                   "We have churned on some peers, so we must schedule find peer requests for them!\n");
1764       count_added = 0;
1765       for (i = 0; i < num_peers; i++)
1766         {
1767           temp_daemon = GNUNET_TESTING_daemon_get (pg, i);
1768           if (GNUNET_YES == GNUNET_TESTING_daemon_running (temp_daemon))
1769             {
1770               peer_count = GNUNET_malloc (sizeof(struct PeerCount));
1771               memcpy (&peer_count->peer_id, &temp_daemon->id,
1772                       sizeof(struct GNUNET_PeerIdentity));
1773               GNUNET_assert(peer_count->count == 0);
1774               peer_count->heap_node
1775                   = GNUNET_CONTAINER_heap_insert (
1776                                                   find_peer_context->peer_min_heap,
1777                                                   peer_count, peer_count->count);
1778               GNUNET_CONTAINER_multihashmap_put (find_peer_context->peer_hash,
1779                                                  &temp_daemon->id.hashPubKey,
1780                                                  peer_count,
1781                                                  GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY);
1782               count_added++;
1783             }
1784         }
1785       GNUNET_log (
1786                   GNUNET_ERROR_TYPE_WARNING,
1787                   "Added %d peers to heap, total size %d\n",
1788                   count_added,
1789                   GNUNET_CONTAINER_heap_get_size (
1790                                                   find_peer_context->peer_min_heap));
1791       GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1792                                     &schedule_churn_get_topology,
1793                                     find_peer_context);
1794     }
1795   else
1796     {
1797       GNUNET_log (
1798                   GNUNET_ERROR_TYPE_WARNING,
1799                   "Only churned off peers, no find peer requests, scheduling more gets (after allowing time for peers to disconnect properly!)...\n");
1800       if (dhtlog_handle != NULL)
1801         {
1802           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1803           topo_ctx->cont = &do_get;
1804           topo_ctx->cls = all_gets;
1805           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
1806           topo_ctx->peers_seen
1807               = GNUNET_CONTAINER_multihashmap_create (num_peers);
1808           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1809                                                    all_get_timeout);
1810           calc_timeout
1811               = GNUNET_TIME_relative_add (calc_timeout,
1812                                           DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT);
1813           calc_timeout
1814               = GNUNET_TIME_relative_add (calc_timeout,
1815                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1816           die_task
1817               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1818                                               "from do gets (churn_complete)");
1819           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1820                                         &capture_current_topology, topo_ctx);
1821           dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
1822         }
1823       else
1824         {
1825           calc_timeout = GNUNET_TIME_relative_add (DEFAULT_GET_TIMEOUT,
1826                                                    all_get_timeout);
1827           calc_timeout
1828               = GNUNET_TIME_relative_add (calc_timeout,
1829                                           DEFAULT_PEER_DISCONNECT_TIMEOUT);
1830           die_task
1831               = GNUNET_SCHEDULER_add_delayed (calc_timeout, &end_badly,
1832                                               "from do gets (churn_complete)");
1833           GNUNET_SCHEDULER_add_delayed (DEFAULT_PEER_DISCONNECT_TIMEOUT,
1834                                         &do_get, all_gets);
1835         }
1836     }
1837 }
1838
1839 /**
1840  * Decide how many peers to turn on or off in this round, make sure the
1841  * numbers actually make sense, then do so.  This function sets in motion
1842  * churn, find peer requests for newly joined peers, and issuing get
1843  * requests once the new peers have done so.
1844  *
1845  * @param cls closure (unused)
1846  * @param tc task context (unused)
1847  */
1848 static void
1849 churn_peers(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1850 {
1851   unsigned int count_running;
1852   unsigned int churn_up;
1853   unsigned int churn_down;
1854   struct GNUNET_TIME_Relative timeout;
1855   struct FindPeerContext *find_peer_context;
1856
1857   churn_up = churn_down = 0;
1858   count_running = GNUNET_TESTING_daemons_running (pg);
1859   if (count_running > churn_array[current_churn_round])
1860     churn_down = count_running - churn_array[current_churn_round];
1861   else if (count_running < churn_array[current_churn_round])
1862     churn_up = churn_array[current_churn_round] - count_running;
1863   else
1864     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1865                 "Not churning any peers, topology unchanged.\n");
1866
1867   if (churn_up > num_peers - count_running)
1868     {
1869       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1870                   "Churn file specified %u peers (up); only have %u!",
1871                   churn_array[current_churn_round], num_peers);
1872       churn_up = num_peers - count_running;
1873     }
1874   else if (churn_down > count_running)
1875     {
1876       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1877                   "Churn file specified %u peers (down); only have %u!",
1878                   churn_array[current_churn_round], count_running);
1879       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1880                   "This will leave NO peers running (mistake in churn configuration?)!");
1881       churn_down = count_running;
1882     }
1883   //timeout = GNUNET_TIME_relative_multiply(seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1884   //timeout = GNUNET_TIME_relative_multiply (seconds_per_peer_start, churn_up > 0 ? churn_up : churn_down);
1885   timeout = GNUNET_TIME_relative_multiply (DEFAULT_TIMEOUT, 2); /* FIXME: Lack of intelligent choice here */
1886   find_peer_context = NULL;
1887   if (churn_up > 0) /* Only need to do find peer requests if we turned new peers on */
1888     {
1889       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
1890       find_peer_context->count_peers_cb = &count_peers_churn_cb;
1891       find_peer_context->previous_peers = 0;
1892       find_peer_context->current_peers = 0;
1893       find_peer_context->endtime = GNUNET_TIME_relative_to_absolute (timeout);
1894       find_peer_context->peer_hash
1895           = GNUNET_CONTAINER_multihashmap_create (num_peers);
1896       find_peer_context->peer_min_heap
1897           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1898     }
1899   GNUNET_log (
1900               GNUNET_ERROR_TYPE_WARNING,
1901               "churn_peers: want %u total, %u running, starting %u, stopping %u\n",
1902               churn_array[current_churn_round], count_running, churn_up,
1903               churn_down);
1904   GNUNET_TESTING_daemons_churn (pg, NULL, churn_down, churn_up, timeout,
1905                                 &churn_complete, find_peer_context);
1906   current_churn_round++;
1907 }
1908
1909 /**
1910  * Task to release DHT handle associated with GET request.
1911  */
1912 static void
1913 get_stop_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
1914 {
1915   struct TestGetContext *test_get = cls;
1916   struct TopologyIteratorContext *topo_ctx;
1917
1918   /* The dht_handle may be null if this get was scheduled from a down peer */
1919   if (test_get->dht_handle != NULL)
1920     {
1921       GNUNET_DHT_disconnect (test_get->dht_handle);
1922       outstanding_gets--; /* GET is really finished */
1923       test_get->dht_handle = NULL;
1924     }
1925
1926   /* Reset the uid (which item to search for) and the daemon (which peer to search from) for later get request iterations */
1927   if (get_from_same == GNUNET_NO)
1928     {
1929       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
1930                                                 num_puts);
1931       test_get->daemon
1932           = GNUNET_TESTING_daemon_get (
1933                                        pg,
1934                                        GNUNET_CRYPTO_random_u32 (
1935                                                                  GNUNET_CRYPTO_QUALITY_WEAK,
1936                                                                  num_peers));
1937     }
1938
1939 #if VERBOSE > 1
1940   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
1941 #endif
1942   update_meter (get_meter);
1943   if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0))
1944     {
1945       fprintf (
1946                stderr,
1947                "Canceling die task (get_stop_finished) %llu gets completed, %llu gets failed\n",
1948                gets_completed, gets_failed);
1949       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
1950         dhtlog_handle->insert_round_details (DHT_ROUND_GET, rounds_finished,
1951                                              num_gets, gets_completed);
1952       GNUNET_SCHEDULER_cancel (die_task);
1953       reset_meter (put_meter);
1954       reset_meter (get_meter);
1955       if ((target_completions > 0)
1956           && (gets_completed > target_completions))
1957         fprintf(stderr, "Ending test early due to GET success!\n");
1958       /**
1959        *  Handle all cases:
1960        *    1) Testing is completely finished, call the topology iteration dealy and die
1961        *    2) Testing is not finished, churn the network and do gets again (current_churn_round < churn_rounds)
1962        *    3) Testing is not finished, reschedule all the PUTS *and* GETS again (num_rounds > 1)
1963        */
1964       if ((rounds_finished == total_rounds - 1) || ((target_completions > 0)
1965           && (gets_completed > target_completions))) /* Everything is finished, end testing */
1966         {
1967           if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
1968             {
1969               topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
1970               topo_ctx->cont = &log_dht_statistics;
1971               topo_ctx->peers_seen
1972                   = GNUNET_CONTAINER_multihashmap_create (num_peers);
1973               GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
1974             }
1975           else
1976             GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
1977         }
1978       else if (current_churn_round < churns_per_round * (rounds_finished + 1)) /* Do next round of churn */
1979         {
1980           GNUNET_log (
1981                       GNUNET_ERROR_TYPE_WARNING,
1982                       "Current churn round %u, real round %u, scheduling next round of churn.\n",
1983                       current_churn_round, rounds_finished + 1);
1984           gets_completed = 0;
1985           gets_failed = 0;
1986
1987           if (dhtlog_handle != NULL)
1988             dhtlog_handle->insert_round (DHT_ROUND_CHURN, rounds_finished);
1989
1990           GNUNET_SCHEDULER_add_now (&churn_peers, NULL);
1991         }
1992       else if (rounds_finished < total_rounds - 1) /* Start a new complete round */
1993         {
1994           rounds_finished++;
1995           gets_completed = 0;
1996           gets_failed = 0;
1997           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1998                       "Round %u of %llu finished, scheduling next round.\n",
1999                       rounds_finished, total_rounds);
2000
2001           /** We reset the peer daemon for puts and gets on each disconnect, so all we need to do is start another round! */
2002           if (GNUNET_YES == in_dht_replication) /* Replication done in DHT, don't redo puts! */
2003             {
2004               if (dhtlog_handle != NULL)
2005                 dhtlog_handle->insert_round (DHT_ROUND_GET, rounds_finished);
2006
2007               die_task
2008                   = GNUNET_SCHEDULER_add_delayed (
2009                                                   GNUNET_TIME_relative_add (
2010                                                                             GNUNET_TIME_relative_add (
2011                                                                                                       GNUNET_TIME_relative_multiply (
2012                                                                                                                                      GNUNET_TIME_UNIT_SECONDS,
2013                                                                                                                                      round_delay),
2014                                                                                                       all_get_timeout),
2015                                                                             DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2016                                                   &end_badly,
2017                                                   "from do gets (next round)");
2018               GNUNET_SCHEDULER_add_delayed (
2019                                             GNUNET_TIME_relative_multiply (
2020                                                                            GNUNET_TIME_UNIT_SECONDS,
2021                                                                            round_delay),
2022                                             &do_get, all_gets);
2023             }
2024           else
2025             {
2026               if (dhtlog_handle != NULL)
2027                 dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2028               die_task
2029                   = GNUNET_SCHEDULER_add_delayed (
2030                                                   GNUNET_TIME_relative_add (
2031                                                                             GNUNET_TIME_relative_multiply (
2032                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2033                                                                                                            round_delay),
2034                                                                             GNUNET_TIME_relative_multiply (
2035                                                                                                            GNUNET_TIME_UNIT_SECONDS,
2036                                                                                                            num_puts
2037                                                                                                                * 2)),
2038                                                   &end_badly, "from do puts");
2039               GNUNET_SCHEDULER_add_delayed (
2040                                             GNUNET_TIME_relative_multiply (
2041                                                                            GNUNET_TIME_UNIT_SECONDS,
2042                                                                            round_delay),
2043                                             &do_put, all_puts);
2044             }
2045         }
2046     }
2047 }
2048
2049 /**
2050  * Task to release get handle.
2051  */
2052 static void
2053 get_stop_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2054 {
2055   struct TestGetContext *test_get = cls;
2056
2057   if ((tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT) != 0)
2058     gets_failed++;
2059   else
2060     cumulative_successful_gets++;
2061
2062   GNUNET_assert(test_get->get_handle != NULL);
2063   GNUNET_DHT_get_stop (test_get->get_handle);
2064   test_get->get_handle = NULL;
2065   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2066   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2067 }
2068
2069 /**
2070  * Iterator called if the GET request initiated returns a response.
2071  *
2072  * @param cls closure
2073  * @param exp when will this value expire
2074  * @param key key of the result
2075  * @param get_path NULL-terminated array of pointers
2076  *                 to the peers on reverse GET path (or NULL if not recorded)
2077  * @param put_path NULL-terminated array of pointers
2078  *                 to the peers on the PUT path (or NULL if not recorded)
2079  * @param type type of the result
2080  * @param size number of bytes in data
2081  * @param data pointer to the result data
2082  */
2083 static void
2084 get_result_iterator(void *cls, struct GNUNET_TIME_Absolute exp,
2085                     const GNUNET_HashCode * key,
2086                     const struct GNUNET_PeerIdentity * const *get_path,
2087                     const struct GNUNET_PeerIdentity * const *put_path,
2088                     enum GNUNET_BLOCK_Type type, size_t size, const void *data)
2089 {
2090   struct TestGetContext *test_get = cls;
2091
2092   if (test_get->succeeded == GNUNET_YES)
2093     return; /* Get has already been successful, probably ending now */
2094
2095   if (0 != memcmp (&known_keys[test_get->uid], key, sizeof(GNUNET_HashCode))) /* || (0 != memcmp(original_data, data, sizeof(original_data))))*/
2096     {
2097       gets_completed++;
2098       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2099                   "Key or data is not the same as was inserted!\n");
2100     }
2101   else
2102     {
2103       gets_completed++;
2104       test_get->succeeded = GNUNET_YES;
2105     }
2106 #if VERBOSE > 1
2107   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
2108 #endif
2109   GNUNET_SCHEDULER_cancel (test_get->disconnect_task);
2110   GNUNET_SCHEDULER_add_continuation (&get_stop_task, test_get,
2111                                      GNUNET_SCHEDULER_REASON_PREREQ_DONE);
2112 }
2113
2114 /**
2115  * Set up some data, and call API PUT function
2116  */
2117 static void
2118 do_get(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2119 {
2120   struct TestGetContext *test_get = cls;
2121
2122   if (num_gets == 0)
2123     {
2124       GNUNET_SCHEDULER_cancel (die_task);
2125       GNUNET_SCHEDULER_add_now (&finish_testing, NULL);
2126     }
2127
2128   if (test_get == NULL)
2129     return; /* End of the list */
2130
2131   /* Set this here in case we are re-running gets */
2132   test_get->succeeded = GNUNET_NO;
2133
2134   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_get->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2135     {
2136       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2137                   "Peer we should issue get request from is down, skipping.\n");
2138       gets_failed++;
2139       GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
2140       GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2141       return;
2142     }
2143
2144   /* Check if more gets are outstanding than should be */
2145   if (outstanding_gets > max_outstanding_gets)
2146     {
2147       GNUNET_SCHEDULER_add_delayed (
2148                                     GNUNET_TIME_relative_multiply (
2149                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2150                                                                    200),
2151                                     &do_get, test_get);
2152       return;
2153     }
2154
2155   /* Connect to the first peer's DHT */
2156   test_get->dht_handle = GNUNET_DHT_connect (test_get->daemon->cfg, 10);
2157   GNUNET_assert(test_get->dht_handle != NULL);
2158   outstanding_gets++;
2159
2160   cumulative_num_gets++;
2161   /* Insert the data at the first peer */
2162   test_get->get_handle = GNUNET_DHT_get_start (test_get->dht_handle, get_delay,
2163                                                GNUNET_BLOCK_TYPE_TEST,
2164                                                &known_keys[test_get->uid],
2165                                                get_replication,
2166                                                GNUNET_DHT_RO_NONE, NULL, 0,
2167                                                NULL, 0, &get_result_iterator,
2168                                                test_get);
2169
2170 #if VERBOSE > 1
2171   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
2172       test_get->uid,
2173       test_get->daemon->shortname);
2174 #endif
2175   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed (get_timeout,
2176                                                             &get_stop_task,
2177                                                             test_get);
2178
2179   /* Schedule the next request in the linked list of get requests */
2180   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
2181 }
2182
2183 /**
2184  * Called when the PUT request has been transmitted to the DHT service.
2185  * Schedule the GET request for some time in the future.
2186  */
2187 static void
2188 put_finished(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2189 {
2190   struct TestPutContext *test_put = cls;
2191   struct TopologyIteratorContext *topo_ctx;
2192   outstanding_puts--;
2193   puts_completed++;
2194
2195   if ((tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT) != 0)
2196     fprintf (stderr, "PUT Request failed!\n");
2197
2198   /* Reset the daemon (which peer to insert at) for later put request iterations */
2199   if (replicate_same == GNUNET_NO)
2200     test_put->daemon
2201         = GNUNET_TESTING_daemon_get (
2202                                      pg,
2203                                      GNUNET_CRYPTO_random_u32 (
2204                                                                GNUNET_CRYPTO_QUALITY_WEAK,
2205                                                                num_peers));
2206
2207   GNUNET_SCHEDULER_cancel (test_put->disconnect_task);
2208   test_put->disconnect_task = GNUNET_SCHEDULER_add_now (&put_disconnect_task,
2209                                                         test_put);
2210   if (GNUNET_YES == update_meter (put_meter))
2211     {
2212       GNUNET_assert(outstanding_puts == 0);
2213       GNUNET_SCHEDULER_cancel (die_task);
2214       if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2215         {
2216           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2217           topo_ctx->cont = &do_get;
2218           topo_ctx->cls = all_gets;
2219           topo_ctx->timeout = DEFAULT_GET_TIMEOUT;
2220           topo_ctx->peers_seen
2221               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2222           die_task
2223               = GNUNET_SCHEDULER_add_delayed (
2224                                               GNUNET_TIME_relative_add (
2225                                                                         GNUNET_TIME_relative_add (
2226                                                                                                   DEFAULT_GET_TIMEOUT,
2227                                                                                                   all_get_timeout),
2228                                                                         DEFAULT_TOPOLOGY_CAPTURE_TIMEOUT),
2229                                               &end_badly,
2230                                               "from do gets (put finished)");
2231           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
2232         }
2233       else
2234         {
2235           fprintf (stderr, "Scheduling die task (put finished)\n");
2236           die_task
2237               = GNUNET_SCHEDULER_add_delayed (
2238                                               GNUNET_TIME_relative_add (
2239                                                                         DEFAULT_GET_TIMEOUT,
2240                                                                         all_get_timeout),
2241                                               &end_badly,
2242                                               "from do gets (put finished)");
2243           GNUNET_SCHEDULER_add_delayed (DEFAULT_GET_TIMEOUT, &do_get, all_gets);
2244         }
2245       return;
2246     }
2247 }
2248
2249 /**
2250  * Set up some data, and call API PUT function
2251  */
2252 static void
2253 do_put(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2254 {
2255   struct TestPutContext *test_put = cls;
2256   char data[test_data_size]; /* Made up data to store */
2257   uint32_t rand;
2258   int i;
2259
2260   if (test_put == NULL)
2261     return; /* End of list */
2262
2263   if (GNUNET_YES != GNUNET_TESTING_daemon_running (test_put->daemon)) /* If the peer has been churned off, don't try issuing request from it! */
2264     {
2265       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2266                   "Peer we should issue put request at is down, skipping.\n");
2267       update_meter (put_meter);
2268       GNUNET_SCHEDULER_add_now (&do_put, test_put->next);
2269       return;
2270     }
2271
2272   for (i = 0; i < sizeof(data); i++)
2273     {
2274       memset (&data[i], GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2275                                                   UINT32_MAX), 1);
2276     }
2277
2278   if (outstanding_puts > max_outstanding_puts)
2279     {
2280       GNUNET_SCHEDULER_add_delayed (
2281                                     GNUNET_TIME_relative_multiply (
2282                                                                    GNUNET_TIME_UNIT_MILLISECONDS,
2283                                                                    200),
2284                                     &do_put, test_put);
2285       return;
2286     }
2287
2288 #if VERBOSE > 1
2289   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
2290       test_put->uid,
2291       test_put->daemon->shortname);
2292 #endif
2293   test_put->dht_handle = GNUNET_DHT_connect (test_put->daemon->cfg, 10);
2294
2295   GNUNET_assert(test_put->dht_handle != NULL);
2296   outstanding_puts++;
2297   GNUNET_DHT_put (test_put->dht_handle, &known_keys[test_put->uid],
2298                   put_replication, GNUNET_DHT_RO_NONE, GNUNET_BLOCK_TYPE_TEST,
2299                   sizeof(data), data, GNUNET_TIME_UNIT_FOREVER_ABS, put_delay,
2300                   &put_finished, test_put);
2301   test_put->disconnect_task
2302       = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever (),
2303                                       &put_disconnect_task, test_put);
2304   rand = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 2);
2305   GNUNET_SCHEDULER_add_delayed (
2306                                 GNUNET_TIME_relative_multiply (
2307                                                                GNUNET_TIME_UNIT_SECONDS,
2308                                                                rand), &do_put,
2309                                 test_put->next);
2310 }
2311
2312 static void
2313 schedule_find_peer_requests(void *cls,
2314                             const struct GNUNET_SCHEDULER_TaskContext * tc);
2315
2316 #if HAVE_MALICIOUS
2317 static void
2318 setup_malicious_peers(void *cls,
2319     const struct GNUNET_SCHEDULER_TaskContext * tc);
2320 #endif
2321
2322 /**
2323  * Given a number of total peers and a bucket size, estimate the number of
2324  * connections in a perfect kademlia topology.
2325  */
2326 static unsigned int
2327 connection_estimate(unsigned int peer_count, unsigned int bucket_size)
2328 {
2329   unsigned int i;
2330   unsigned int filled;
2331   i = num_peers;
2332
2333   filled = 0;
2334   while (i >= bucket_size)
2335     {
2336       filled++;
2337       i = i / 2;
2338     }
2339   filled++; /* Add one filled bucket to account for one "half full" and some miscellaneous */
2340   return filled * bucket_size * peer_count;
2341
2342 }
2343
2344 /**
2345  * Callback for iterating over all the peer connections of a peer group.
2346  */
2347 static void
2348 count_peers_cb(void *cls, const struct GNUNET_PeerIdentity *first,
2349                const struct GNUNET_PeerIdentity *second, const char *emsg)
2350 {
2351   struct FindPeerContext *find_peer_context = cls;
2352   if ((first != NULL) && (second != NULL))
2353     {
2354       add_new_connection (find_peer_context, first, second);
2355       find_peer_context->current_peers++;
2356     }
2357   else
2358     {
2359       GNUNET_log (
2360                   GNUNET_ERROR_TYPE_WARNING,
2361                   "Peer count finished (%u connections), %u new peers, connection estimate %u (target %u)\n",
2362                   find_peer_context->current_peers,
2363                   find_peer_context->current_peers
2364                       - find_peer_context->previous_peers,
2365                   connection_estimate (num_peers, DEFAULT_BUCKET_SIZE),
2366                   target_total_connections);
2367
2368       if ((find_peer_context->last_sent < 8)
2369           || ((find_peer_context->current_peers < 2
2370               * connection_estimate (num_peers, DEFAULT_BUCKET_SIZE))
2371               && (GNUNET_TIME_absolute_get_remaining (
2372                                                       find_peer_context->endtime).rel_value
2373                   > 0) && (find_peer_context->current_peers
2374               < target_total_connections)))
2375         {
2376           GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests,
2377                                     find_peer_context);
2378         }
2379       else
2380         {
2381           GNUNET_CONTAINER_multihashmap_iterate (find_peer_context->peer_hash,
2382                                                  &remove_peer_count,
2383                                                  find_peer_context);
2384           GNUNET_CONTAINER_multihashmap_destroy (find_peer_context->peer_hash);
2385           GNUNET_CONTAINER_heap_destroy (find_peer_context->peer_min_heap);
2386           GNUNET_free(find_peer_context);
2387           fprintf (stderr, "Not sending any more find peer requests.\n");
2388
2389 #if HAVE_MALICIOUS
2390           if (GNUNET_YES == malicious_after_settle)
2391             {
2392               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "calling setup_malicious_peers\n");
2393               GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2394             }
2395 #endif
2396         }
2397     }
2398 }
2399
2400 /**
2401  * Set up a single find peer request for each peer in the topology.  Do this
2402  * until the settle time is over, limited by the number of outstanding requests
2403  * and the time allowed for each one!
2404  */
2405 static void
2406 schedule_find_peer_requests(void *cls,
2407                             const struct GNUNET_SCHEDULER_TaskContext * tc)
2408 {
2409   struct FindPeerContext *find_peer_ctx = cls;
2410   struct TestFindPeer *test_find_peer;
2411   struct PeerCount *peer_count;
2412   uint32_t i;
2413   uint32_t random;
2414
2415   if (find_peer_ctx->previous_peers == 0) /* First time, go slowly */
2416     find_peer_ctx->total = 1;
2417   else if (find_peer_ctx->current_peers - find_peer_ctx->previous_peers
2418       > MAX_FIND_PEER_CUTOFF) /* Found LOTS of peers, still go slowly */
2419     find_peer_ctx->total = find_peer_ctx->last_sent - (find_peer_ctx->last_sent
2420         / 8);
2421   else
2422     find_peer_ctx->total = find_peer_ctx->last_sent * 2;
2423
2424   if (find_peer_ctx->total > max_outstanding_find_peers)
2425     find_peer_ctx->total = max_outstanding_find_peers;
2426
2427   if (find_peer_ctx->total > num_peers) /* Don't try to send more messages than we have peers! */
2428     find_peer_ctx->total = num_peers;
2429
2430   find_peer_ctx->last_sent = find_peer_ctx->total;
2431   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2432               "Sending %u find peer messages (goal at least %u connections)\n",
2433               find_peer_ctx->total, target_total_connections);
2434
2435   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
2436                                                   find_peer_ctx->total);
2437   for (i = 0; i < find_peer_ctx->total; i++)
2438     {
2439       test_find_peer = GNUNET_malloc(sizeof(struct TestFindPeer));
2440       /* If we haven't sent any requests yet, choose random peers */
2441       /* Also choose random in _half_ of all cases, so we don't
2442        * get stuck choosing topologically restricted peers with
2443        * few connections that will never be able to find any new
2444        * peers! */
2445       if ((find_peer_ctx->previous_peers == 0) || (i % 2 == 0))
2446         {
2447           /**
2448            * Attempt to spread find peer requests across even sections of the peer address
2449            * space.  Choose basically 1 peer in every num_peers / max_outstanding_requests
2450            * each time, then offset it by a randomish value.
2451            *
2452            * For instance, if num_peers is 100 and max_outstanding is 10, first chosen peer
2453            * will be between 0 - 10, second between 10 - 20, etc.
2454            */
2455           random = (num_peers / find_peer_ctx->total) * i;
2456           random = random
2457               + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2458                                           (num_peers / find_peer_ctx->total));
2459           if (random >= num_peers)
2460             {
2461               random = random - num_peers;
2462             }
2463 #if REAL_RANDOM
2464           random = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2465 #endif
2466           test_find_peer->daemon = GNUNET_TESTING_daemon_get (pg, random);
2467         }
2468       else /* If we have sent requests, choose peers with a low number of connections to send requests from */
2469         {
2470           peer_count
2471               = GNUNET_CONTAINER_heap_remove_root (find_peer_ctx->peer_min_heap);
2472           GNUNET_assert(GNUNET_YES == GNUNET_CONTAINER_multihashmap_remove(find_peer_ctx->peer_hash, &peer_count->peer_id.hashPubKey, peer_count));
2473           test_find_peer->daemon
2474               = GNUNET_TESTING_daemon_get_by_id (pg, &peer_count->peer_id);
2475           GNUNET_assert(test_find_peer->daemon != NULL);
2476         }
2477
2478       test_find_peer->find_peer_context = find_peer_ctx;
2479       GNUNET_SCHEDULER_add_delayed (
2480                                     GNUNET_TIME_relative_multiply (
2481                                                                    find_peer_offset,
2482                                                                    i),
2483                                     &send_find_peer_request, test_find_peer);
2484     }
2485
2486   if ((find_peer_ctx->peer_hash == NULL) && (find_peer_ctx->peer_min_heap
2487       == NULL))
2488     {
2489       find_peer_ctx->peer_hash
2490           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2491       find_peer_ctx->peer_min_heap
2492           = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
2493     }
2494   else
2495     {
2496       GNUNET_CONTAINER_multihashmap_iterate (find_peer_ctx->peer_hash,
2497                                              &remove_peer_count, find_peer_ctx);
2498       GNUNET_CONTAINER_multihashmap_destroy (find_peer_ctx->peer_hash);
2499       find_peer_ctx->peer_hash
2500           = GNUNET_CONTAINER_multihashmap_create (num_peers);
2501     }
2502
2503   GNUNET_assert(0 == GNUNET_CONTAINER_multihashmap_size(find_peer_ctx->peer_hash));
2504   GNUNET_assert(0 == GNUNET_CONTAINER_heap_get_size(find_peer_ctx->peer_min_heap));
2505
2506 }
2507
2508 /**
2509  * Convert unique ID to hash code.
2510  *
2511  * @param uid unique ID to convert
2512  * @param hash set to uid (extended with zeros)
2513  */
2514 static void
2515 hash_from_uid(uint32_t uid, GNUNET_HashCode *hash)
2516 {
2517   memset (hash, 0, sizeof(GNUNET_HashCode));
2518   *((uint32_t *) hash) = uid;
2519 }
2520
2521 /**
2522  * Set up all of the put and get operations we want to do
2523  * in the current round.  Allocate data structure for each,
2524  * add to list, then schedule the actual PUT operations.
2525  */
2526 static void
2527 setup_puts_and_gets(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2528 {
2529   int i;
2530   struct TestPutContext *test_put;
2531   struct TestGetContext *test_get;
2532   uint32_t temp_peer;
2533   GNUNET_HashCode uid_hash;
2534   int count;
2535 #if REMEMBER
2536   int remember[num_puts][num_peers];
2537   memset(&remember, 0, sizeof(int) * num_puts * num_peers);
2538 #endif
2539   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "in setup_puts_and_gets\n");
2540   known_keys = GNUNET_malloc(sizeof(GNUNET_HashCode) * num_puts);
2541   for (i = 0; i < num_puts; i++)
2542     {
2543       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
2544       test_put->uid = i;
2545       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
2546                                         &known_keys[i]);
2547       /* Set first X bits to match the chosen sybil location if we want to do the sybil attack! */
2548       if (GNUNET_YES == malicious_sybil)
2549         {
2550           memcpy (&known_keys[i], &sybil_target, sizeof(GNUNET_HashCode) / 2);
2551           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2552                       "Distance between sybil location and key is %d\n",
2553                       GNUNET_CRYPTO_hash_matching_bits (&known_keys[i],
2554                                                         &sybil_target));
2555         }
2556       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2557                                             num_peers);
2558       test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2559       /* Don't start PUTs at malicious peers! */
2560       if (malicious_bloom != NULL)
2561         {
2562           count = 0;
2563           hash_from_uid (temp_peer, &uid_hash);
2564           while ((GNUNET_YES
2565               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2566               && (count < num_peers))
2567             {
2568               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2569                                                     num_peers);
2570               hash_from_uid (temp_peer, &uid_hash);
2571               test_put->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2572               count++;
2573             }
2574           if (count == num_peers)
2575             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2576                         "Couldn't find peer not in malicious bloom to select!\n");
2577         }
2578
2579       test_put->next = all_puts;
2580       all_puts = test_put;
2581     }
2582
2583   for (i = 0; i < num_gets; i++)
2584     {
2585       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
2586       test_get->uid = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2587                                                 num_puts);
2588 #if REMEMBER
2589       while (remember[test_get->uid][temp_daemon] == 1)
2590       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2591       remember[test_get->uid][temp_daemon] = 1;
2592 #endif
2593       temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2594                                             num_peers);
2595       test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2596       /* Don't start GETs at malicious peers! */
2597       if (malicious_bloom != NULL)
2598         {
2599           hash_from_uid (temp_peer, &uid_hash);
2600           count = 0;
2601           while ((GNUNET_YES
2602               == GNUNET_CONTAINER_bloomfilter_test (malicious_bloom, &uid_hash))
2603               && (count < num_peers))
2604             {
2605               temp_peer = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK,
2606                                                     num_peers);
2607               hash_from_uid (temp_peer, &uid_hash);
2608               test_get->daemon = GNUNET_TESTING_daemon_get (pg, temp_peer);
2609               count++;
2610             }
2611           if (count == num_peers)
2612             GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2613                         "Couldn't find peer not in malicious bloom to select!\n");
2614         }
2615       test_get->next = all_gets;
2616       all_gets = test_get;
2617     }
2618
2619   /*GNUNET_SCHEDULER_cancel (die_task);*/
2620   die_task
2621       = GNUNET_SCHEDULER_add_delayed (
2622                                       GNUNET_TIME_relative_multiply (
2623                                                                      GNUNET_TIME_UNIT_SECONDS,
2624                                                                      num_puts
2625                                                                          * 2),
2626                                       &end_badly, "from do puts");
2627   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
2628
2629 }
2630
2631 /**
2632  * Set up some all of the put and get operations we want
2633  * to do.  Allocate data structure for each, add to list,
2634  * then call actual insert functions.
2635  */
2636 static void
2637 continue_puts_and_gets(void *cls,
2638                        const struct GNUNET_SCHEDULER_TaskContext * tc)
2639 {
2640   int i;
2641   int max;
2642   struct TopologyIteratorContext *topo_ctx;
2643   struct FindPeerContext *find_peer_context;
2644   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "In continue_puts_and_gets\n");
2645   if ((dhtlog_handle != NULL) && (GNUNET_NO == dhtlog_minimal))
2646     {
2647       if (settle_time >= 180 * 2)
2648         max = (settle_time / 180) - 2;
2649       else
2650         max = 1;
2651       for (i = 1; i < max; i++)
2652         {
2653           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2654           topo_ctx->current_iteration = i;
2655           topo_ctx->total_iterations = max;
2656           topo_ctx->peers_seen
2657               = GNUNET_CONTAINER_multihashmap_create (num_peers);
2658           //fprintf(stderr, "scheduled topology iteration in %d minutes\n", i);
2659           GNUNET_SCHEDULER_add_delayed (
2660                                         GNUNET_TIME_relative_multiply (
2661                                                                        GNUNET_TIME_UNIT_MINUTES,
2662                                                                        i * 3),
2663                                         &capture_current_topology, topo_ctx);
2664         }
2665       topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
2666       topo_ctx->cont = &setup_puts_and_gets;
2667       topo_ctx->peers_seen = GNUNET_CONTAINER_multihashmap_create (num_peers);
2668       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
2669                   "setting setup_puts_and_gets for %d seconds in the future\n",
2670                   settle_time + 10);
2671       GNUNET_SCHEDULER_add_delayed (
2672                                     GNUNET_TIME_relative_multiply (
2673                                                                    GNUNET_TIME_UNIT_SECONDS,
2674                                                                    (settle_time
2675                                                                        + 10)),
2676                                     &capture_current_topology, topo_ctx);
2677     }
2678   else
2679     GNUNET_SCHEDULER_add_delayed (
2680                                   GNUNET_TIME_relative_multiply (
2681                                                                  GNUNET_TIME_UNIT_SECONDS,
2682                                                                  (settle_time
2683                                                                      + 10)),
2684                                   &setup_puts_and_gets, NULL);
2685
2686   if (dhtlog_handle != NULL)
2687     dhtlog_handle->insert_round (DHT_ROUND_NORMAL, rounds_finished);
2688
2689 #if HAVE_MALICIOUS
2690   if ((GNUNET_YES != malicious_after_settle) || (settle_time == 0))
2691     {
2692       GNUNET_SCHEDULER_add_now(&setup_malicious_peers, NULL);
2693     }
2694 #endif
2695
2696   if ((GNUNET_YES == do_find_peer) && (settle_time > 0))
2697     {
2698       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2699                   "Scheduling find peer requests during \"settle\" time.\n");
2700       find_peer_context = GNUNET_malloc(sizeof(struct FindPeerContext));
2701       find_peer_context->count_peers_cb = &count_peers_cb;
2702       find_peer_context->endtime
2703           = GNUNET_TIME_relative_to_absolute (
2704                                               GNUNET_TIME_relative_multiply (
2705                                                                              GNUNET_TIME_UNIT_SECONDS,
2706                                                                              settle_time));
2707       GNUNET_SCHEDULER_add_now (&schedule_find_peer_requests, find_peer_context);
2708     }
2709   else
2710     {
2711       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
2712                   "Assuming automatic DHT find peer requests.\n");
2713     }
2714 }
2715
2716 /**
2717  * Task to release DHT handles
2718  */
2719 static void
2720 malicious_disconnect_task(void *cls,
2721                           const struct GNUNET_SCHEDULER_TaskContext * tc)
2722 {
2723   struct MaliciousContext *ctx = cls;
2724   outstanding_malicious--;
2725   malicious_completed++;
2726   ctx->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
2727   GNUNET_DHT_disconnect (ctx->dht_handle);
2728   ctx->dht_handle = NULL;
2729   GNUNET_free(ctx);
2730
2731   if (malicious_completed == malicious_getters + malicious_putters
2732       + malicious_droppers)
2733     {
2734       fprintf (stderr, "Finished setting all malicious peers up!\n");
2735     }
2736 }
2737
2738 #if HAVE_MALICIOUS
2739 /**
2740  * Task to release DHT handles
2741  */
2742 static void
2743 malicious_done_task(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2744   {
2745     struct MaliciousContext *ctx = cls;
2746     GNUNET_SCHEDULER_cancel (ctx->disconnect_task);
2747     GNUNET_SCHEDULER_add_now (&malicious_disconnect_task, ctx);
2748   }
2749
2750 /**
2751  * Set up some data, and call API PUT function
2752  */
2753 static void
2754 set_malicious(void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2755   {
2756     struct MaliciousContext *ctx = cls;
2757
2758     if (outstanding_malicious > DEFAULT_MAX_OUTSTANDING_GETS)
2759       {
2760         GNUNET_SCHEDULER_add_delayed (
2761             GNUNET_TIME_relative_multiply (
2762                 GNUNET_TIME_UNIT_MILLISECONDS,
2763                 100),
2764             &set_malicious, ctx);
2765         return;
2766       }
2767
2768     if (ctx->dht_handle == NULL)
2769       {
2770         ctx->dht_handle = GNUNET_DHT_connect (ctx->daemon->cfg, 1);
2771         outstanding_malicious++;
2772       }
2773
2774     GNUNET_assert(ctx->dht_handle != NULL);
2775
2776 #if VERBOSE > 1
2777     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Setting peer %s malicious type %d\n",
2778         ctx->daemon->shortname, ctx->malicious_type);
2779 #endif
2780
2781     switch (ctx->malicious_type)
2782       {
2783         case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET:
2784         GNUNET_DHT_set_malicious_getter (ctx->dht_handle, malicious_get_frequency, &malicious_done_task, ctx);
2785         break;
2786         case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT:
2787         GNUNET_DHT_set_malicious_putter (ctx->dht_handle, malicious_put_frequency, &malicious_done_task, ctx);
2788         break;
2789         case GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP:
2790         GNUNET_DHT_set_malicious_dropper (ctx->dht_handle, &malicious_done_task, ctx);
2791         break;
2792         default:
2793         break;
2794       }
2795
2796     ctx->disconnect_task
2797     = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL,
2798         &malicious_disconnect_task, ctx);
2799   }
2800
2801 /**
2802  * Choose the next peer from the peer group to set as malicious.
2803  * If we are doing a sybil attack, find the nearest peer to the
2804  * sybil location that has not already been set malicious.  Otherwise
2805  * just choose a random not already chosen peer.
2806  *
2807  * @param pg the peer group
2808  * @param bloom the bloomfilter which contains all peer already
2809  *        chosen to be malicious
2810  */
2811 static uint32_t
2812 choose_next_malicious (struct GNUNET_TESTING_PeerGroup *pg, struct GNUNET_CONTAINER_BloomFilter *bloom)
2813   {
2814     int i;
2815     int nearest;
2816     int bits_match;
2817     int curr_distance;
2818     int count;
2819     struct GNUNET_TESTING_Daemon *temp_daemon;
2820     GNUNET_HashCode uid_hash;
2821
2822     curr_distance = 0;
2823     nearest = 0;
2824     GNUNET_assert (bloom != NULL);
2825
2826     if (GNUNET_YES == malicious_sybil)
2827       {
2828         for (i = 0; i < num_peers; i++)
2829           {
2830             temp_daemon = GNUNET_TESTING_daemon_get(pg, i);
2831             hash_from_uid(i, &uid_hash);
2832             /* Check if this peer matches the bloomfilter */
2833             if ((GNUNET_NO == GNUNET_TESTING_daemon_running(temp_daemon)) || (GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)))
2834             continue;
2835
2836             bits_match = GNUNET_CRYPTO_hash_matching_bits (&temp_daemon->id.hashPubKey, &sybil_target);
2837             if (bits_match >= curr_distance)
2838               {
2839                 GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Found nearer peer %s to %s, old matching bits %d, new %d\n", GNUNET_i2s(&temp_daemon->id), GNUNET_h2s(&sybil_target), curr_distance, bits_match);
2840                 nearest = i;
2841                 curr_distance = bits_match;
2842               }
2843           }
2844       }
2845     else
2846       {
2847         nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2848         hash_from_uid(nearest, &uid_hash);
2849         count = 0;
2850         while ((GNUNET_YES == GNUNET_CONTAINER_bloomfilter_test (bloom, &uid_hash)) && (count < num_peers))
2851           {
2852             GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Peer %d already in bloom (tried %d times)\n", nearest, count);
2853             nearest = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
2854             hash_from_uid(nearest, &uid_hash);
2855             count++;
2856           }
2857         if (count == num_peers)
2858         GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Tried %d times to find a peer, selecting %d at random!!\n", count, nearest);
2859       }
2860
2861     return nearest;
2862   }
2863
2864 /**
2865  * Select randomly from set of known peers,
2866  * set the desired number of peers to the
2867  * proper malicious types.
2868  */
2869 static void
2870 setup_malicious_peers (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2871   {
2872     struct MaliciousContext *ctx;
2873     int i;
2874     uint32_t temp_daemon;
2875     GNUNET_HashCode uid_hash;
2876
2877     for (i = 0; i < malicious_getters; i++)
2878       {
2879         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2880         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2881         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2882         hash_from_uid(temp_daemon, &uid_hash);
2883         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2884         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_GET;
2885         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2886
2887       }
2888
2889     for (i = 0; i < malicious_putters; i++)
2890       {
2891         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2892         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2893         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2894         hash_from_uid(temp_daemon, &uid_hash);
2895         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2896         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_PUT;
2897         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2898
2899       }
2900
2901     for (i = 0; i < malicious_droppers; i++)
2902       {
2903         ctx = GNUNET_malloc(sizeof(struct MaliciousContext));
2904         temp_daemon = choose_next_malicious(pg, malicious_bloom);
2905         ctx->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
2906         hash_from_uid(temp_daemon, &uid_hash);
2907         GNUNET_CONTAINER_bloomfilter_add(malicious_bloom, &uid_hash);
2908         ctx->malicious_type = GNUNET_MESSAGE_TYPE_DHT_MALICIOUS_DROP;
2909         GNUNET_SCHEDULER_add_now (&set_malicious, ctx);
2910       }
2911   }
2912 #endif
2913
2914 #if ONLY_TESTING
2915 /* Forward declaration */
2916 static void
2917 topology_callback (void *cls,
2918     const struct GNUNET_PeerIdentity *first,
2919     const struct GNUNET_PeerIdentity *second,
2920     uint32_t distance,
2921     const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2922     const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2923     struct GNUNET_TESTING_Daemon *first_daemon,
2924     struct GNUNET_TESTING_Daemon *second_daemon,
2925     const char *emsg);
2926
2927 /**
2928  * Retry connecting two specific peers until they connect,
2929  * at a specific interval.  These two peers previously failed
2930  * to connect, and we hope they continue to so that we can
2931  * debug the reason they are having issues.
2932  */
2933 static void
2934 repeat_connect (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
2935   {
2936
2937     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Repeating connect attempt between %s and %s.\n", repeat_connect_peer1->shortname, repeat_connect_peer2->shortname);
2938     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 1 configuration `%s'\n", repeat_connect_peer1->cfgfile);
2939     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Peer 2 configuration `%s'\n", repeat_connect_peer2->cfgfile);
2940
2941     repeat_connect_task = GNUNET_SCHEDULER_NO_TASK;
2942     GNUNET_TESTING_daemons_connect(repeat_connect_peer1,
2943         repeat_connect_peer2,
2944         GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60),
2945         2, &topology_callback, NULL);
2946   }
2947 #endif
2948
2949 /**
2950  * This function is called whenever a connection attempt is finished between two of
2951  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
2952  * number of times this function is called should equal the number returned
2953  * from the GNUNET_TESTING_connect_topology call.
2954  *
2955  * The emsg variable is NULL on success (peers connected), and non-NULL on
2956  * failure (peers failed to connect).
2957  */
2958 static void
2959 topology_callback(void *cls, const struct GNUNET_PeerIdentity *first,
2960                   const struct GNUNET_PeerIdentity *second, uint32_t distance,
2961                   const struct GNUNET_CONFIGURATION_Handle *first_cfg,
2962                   const struct GNUNET_CONFIGURATION_Handle *second_cfg,
2963                   struct GNUNET_TESTING_Daemon *first_daemon,
2964                   struct GNUNET_TESTING_Daemon *second_daemon, const char *emsg)
2965 {
2966   struct TopologyIteratorContext *topo_ctx;
2967   uint64_t duration;
2968   uint64_t total_duration;
2969   uint64_t new_connections;
2970   uint64_t new_failed_connections;
2971   double conns_per_sec_recent;
2972   double conns_per_sec_total;
2973   double failed_conns_per_sec_recent;
2974   double failed_conns_per_sec_total;
2975   char *temp_conn_string;
2976   char *temp_conn_failed_string;
2977   char *revision_str;
2978
2979 #if ONLY_TESTING
2980   if (repeat_connect_mode == GNUNET_YES)
2981     {
2982       if ((first_daemon == repeat_connect_peer1) &&
2983           (second_daemon == repeat_connect_peer2))
2984         {
2985           if (emsg != NULL) /* Peers failed to connect again! */
2986             {
2987               GNUNET_assert(repeat_connect_task == GNUNET_SCHEDULER_NO_TASK);
2988               repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
2989               return;
2990             }
2991           else /* Repeat peers actually connected! */
2992             {
2993               if (repeat_connect_task != GNUNET_SCHEDULER_NO_TASK)
2994               GNUNET_SCHEDULER_cancel(repeat_connect_task);
2995               repeat_connect_peer1 = NULL;
2996               repeat_connect_peer2 = NULL;
2997               repeat_connect_mode = GNUNET_NO;
2998               GNUNET_TESTING_resume_connections(pg);
2999               GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Resuming normal connection mode, debug connection was successful!\n");
3000             }
3001         }
3002     }
3003 #endif
3004
3005   if (GNUNET_TIME_absolute_get_difference (connect_last_time,
3006                                            GNUNET_TIME_absolute_get ()).rel_value
3007       > GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3008                                        CONN_UPDATE_DURATION).rel_value)
3009     {
3010       /* Get number of new connections */
3011       new_connections = total_connections - previous_connections;
3012
3013       /* Get number of new FAILED connections */
3014       new_failed_connections = failed_connections - previous_failed_connections;
3015
3016       /* Get duration in seconds */
3017       duration
3018           = GNUNET_TIME_absolute_get_difference (connect_last_time,
3019                                                  GNUNET_TIME_absolute_get ()).rel_value
3020               / 1000;
3021       total_duration
3022           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3023                                                  GNUNET_TIME_absolute_get ()).rel_value
3024               / 1000;
3025
3026       failed_conns_per_sec_recent = (double) new_failed_connections
3027           / (double) duration;
3028       failed_conns_per_sec_total = (double) failed_connections
3029           / (double) total_duration;
3030       conns_per_sec_recent = (double) new_connections / (double) duration;
3031       conns_per_sec_total = (double) total_connections
3032           / (double) total_duration;
3033       GNUNET_log (
3034                   GNUNET_ERROR_TYPE_WARNING,
3035                   "Recent: %.2f/s, Total: %.2f/s, Recent failed: %.2f/s, total failed %.2f/s\n",
3036                   conns_per_sec_recent, conns_per_sec_total,
3037                   failed_conns_per_sec_recent, failed_conns_per_sec_total);
3038       connect_last_time = GNUNET_TIME_absolute_get ();
3039       previous_connections = total_connections;
3040       previous_failed_connections = failed_connections;
3041       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3042                   "have %llu total_connections, %llu failed\n",
3043                   total_connections, failed_connections);
3044 #if ONLY_TESTING
3045       /* These conditions likely mean we've entered the death spiral of doom */
3046       if ((total_connections > 20000) &&
3047           (conns_per_sec_recent < 5.0) &&
3048           (conns_per_sec_total > 10.0) &&
3049           (emsg != NULL) &&
3050           (repeat_connect_mode == GNUNET_NO))
3051         {
3052           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Entering repeat connection attempt mode!\n");
3053           repeat_connect_peer1 = first_daemon;
3054           repeat_connect_peer2 = second_daemon;
3055           repeat_connect_mode = GNUNET_YES;
3056           GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Stopping NEW connections from being scheduled!\n");
3057           GNUNET_TESTING_stop_connections(pg);
3058           repeat_connect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 60), &repeat_connect, NULL);
3059         }
3060 #endif
3061     }
3062
3063   if (emsg == NULL)
3064     {
3065       total_connections++;
3066 #if VERBOSE > 1
3067       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
3068           first_daemon->shortname,
3069           second_daemon->shortname,
3070           distance);
3071 #endif
3072     }
3073   else
3074     {
3075       failed_connections++;
3076 #if VERBOSE
3077       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Failed to connect peer %s to peer %s with error :\n%s\n",
3078           first_daemon->shortname,
3079           second_daemon->shortname, emsg);
3080
3081       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
3082           first_daemon->shortname,
3083           second_daemon->shortname, emsg);
3084 #endif
3085     }
3086
3087 #if ONLY_TESTING
3088   if ((repeat_connect_mode == GNUNET_YES) )
3089   return;
3090 #endif
3091
3092   GNUNET_assert(peer_connect_meter != NULL);
3093   if (GNUNET_YES == update_meter (peer_connect_meter))
3094     {
3095 #if VERBOSE
3096       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3097           "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
3098           total_connections);
3099 #endif
3100       if (failed_connections > 0)
3101         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3102                     "While connecting, had %u failed connections.\n",
3103                     failed_connections);
3104       if (dhtlog_handle != NULL)
3105         {
3106           dhtlog_handle->update_connections (total_connections);
3107           dhtlog_handle->insert_topology (expected_connections);
3108         }
3109
3110       total_duration
3111           = GNUNET_TIME_absolute_get_difference (connect_start_time,
3112                                                  GNUNET_TIME_absolute_get ()).rel_value
3113               / 1000;
3114       failed_conns_per_sec_total = (long double) failed_connections
3115           / total_duration;
3116       conns_per_sec_total = (long double) total_connections / total_duration;
3117       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3118                   "Overall connection info --- Total: %u, Total Failed %u/s\n",
3119                   total_connections, failed_connections);
3120       GNUNET_log (
3121                   GNUNET_ERROR_TYPE_WARNING,
3122                   "Overall connection info --- Total: %.2f/s, Total Failed %.2f/s\n",
3123                   conns_per_sec_total, failed_conns_per_sec_total);
3124
3125       GNUNET_asprintf (&temp_conn_string, "DHT Profiler Connection/s",
3126                        trial_to_run);
3127       GNUNET_asprintf (&temp_conn_failed_string,
3128                        "DHT Profiler Connection/s failed", trial_to_run);
3129       GNUNET_asprintf (&revision_str, "%llu", revision);
3130
3131       if (GNUNET_YES == insert_gauger_data)
3132         GAUGER_ID("DHT_TESTING", temp_conn_string, (long double)conns_per_sec_total, "conns/s", revision_str);
3133       if (GNUNET_YES == insert_gauger_data)
3134         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, (long double)failed_conns_per_sec_total, "failed_conns", revision_str);
3135
3136       GNUNET_free(temp_conn_string);
3137       GNUNET_free(temp_conn_failed_string);
3138       GNUNET_asprintf (&temp_conn_string, "DHT Profiler Total Connections",
3139                        trial_to_run);
3140       GNUNET_asprintf (&temp_conn_failed_string,
3141                        "DHT Profiler Total Connections failed", trial_to_run);
3142       if (GNUNET_YES == insert_gauger_data)
3143         GAUGER_ID("DHT_TESTING", temp_conn_string, (double)total_connections, "conns", revision_str);
3144       if (GNUNET_YES == insert_gauger_data)
3145         GAUGER_ID("DHT_TESTING", temp_conn_failed_string, (double)failed_connections, "failed conns", revision_str);
3146       GNUNET_free(temp_conn_string);
3147       GNUNET_free(temp_conn_failed_string);
3148       GNUNET_free(revision_str);
3149
3150       GNUNET_SCHEDULER_cancel (die_task);
3151
3152       if ((GNUNET_YES == dhtlog_minimal) && (NULL != dhtlog_handle))
3153         {
3154           topo_ctx = GNUNET_malloc(sizeof(struct TopologyIteratorContext));
3155           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3156                       "Setting continue gets and puts as topo_cont\n");
3157           topo_ctx->cont = &continue_puts_and_gets;
3158           topo_ctx->peers_seen
3159               = GNUNET_CONTAINER_multihashmap_create (num_peers);
3160           GNUNET_SCHEDULER_add_now (&capture_current_topology, topo_ctx);
3161         }
3162       else
3163         {
3164           GNUNET_log (
3165                       GNUNET_ERROR_TYPE_WARNING,
3166                       "For some reason, NOT scheduling final topology capture (settle_time %d, dhtlog_handle %s)!\n",
3167                       settle_time, dhtlog_handle);
3168           GNUNET_SCHEDULER_add_now (&continue_puts_and_gets, NULL);
3169         }
3170     }
3171   else if (total_connections + failed_connections == expected_connections)
3172     {
3173       GNUNET_SCHEDULER_cancel (die_task);
3174       die_task
3175           = GNUNET_SCHEDULER_add_now (&end_badly,
3176                                       "from topology_callback (too many failed connections)");
3177     }
3178 }
3179
3180 static void
3181 peers_started_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3182                        const struct GNUNET_CONFIGURATION_Handle *cfg,
3183                        struct GNUNET_TESTING_Daemon *d, const char *emsg)
3184 {
3185   char *revision_str;
3186   if (emsg != NULL)
3187     {
3188       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3189                   "Failed to start daemon with error: `%s'\n", emsg);
3190       return;
3191     }
3192   GNUNET_assert (id != NULL);
3193
3194 #if VERBOSE > 1
3195   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
3196       (num_peers - peers_left) + 1, num_peers);
3197 #endif
3198
3199   peers_left--;
3200
3201   if (GNUNET_YES == update_meter (peer_start_meter))
3202     {
3203 #if VERBOSE
3204       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3205           "All %d daemons started, now connecting peers!\n",
3206           num_peers);
3207 #endif
3208       GNUNET_SCHEDULER_cancel (die_task);
3209
3210       GNUNET_asprintf (&revision_str, "%llu", revision);
3211       if (GNUNET_YES == insert_gauger_data)
3212         GAUGER_ID("DHT_TESTING",
3213             "peer_startup_time",
3214             GNUNET_TIME_absolute_get_duration(peer_start_time).rel_value / (double)num_peers,
3215             "ms/peer", revision_str);
3216       GNUNET_free(revision_str);
3217
3218       expected_connections = UINT_MAX;
3219       if ((pg != NULL) && (peers_left == 0))
3220         {
3221           connect_start_time = GNUNET_TIME_absolute_get ();
3222           expected_connections
3223               = GNUNET_TESTING_connect_topology (
3224                                                  pg,
3225                                                  connect_topology,
3226                                                  connect_topology_option,
3227                                                  connect_topology_option_modifier,
3228                                                  connect_timeout,
3229                                                  connect_attempts, NULL, NULL);
3230
3231           peer_connect_meter = create_meter (expected_connections,
3232                                              "Peer connection ", GNUNET_YES);
3233           fprintf (stderr, "Have %d expected connections\n",
3234                    expected_connections);
3235         }
3236
3237       if (expected_connections == 0)
3238         {
3239           die_task
3240               = GNUNET_SCHEDULER_add_now (&end_badly,
3241                                           "from connect topology (bad return)");
3242         }
3243
3244       die_task
3245           = GNUNET_SCHEDULER_add_delayed (
3246                                           GNUNET_TIME_relative_multiply (
3247                                                                          GNUNET_TIME_UNIT_SECONDS,
3248                                                                          DEFAULT_CONNECT_TIMEOUT
3249                                                                              * expected_connections),
3250                                           &end_badly,
3251                                           "from connect topology (timeout)");
3252
3253       ok = 0;
3254     }
3255 }
3256
3257 static void
3258 create_topology()
3259 {
3260   unsigned int create_expected_connections;
3261   peers_left = num_peers; /* Reset counter */
3262   create_expected_connections
3263       = GNUNET_TESTING_create_topology (pg, topology, blacklist_topology,
3264                                         blacklist_transports);
3265   if (create_expected_connections > 0)
3266     {
3267       GNUNET_log (
3268                   GNUNET_ERROR_TYPE_WARNING,
3269                   "Topology set up, have %u expected connections, now starting peers!\n",
3270                   create_expected_connections);
3271       GNUNET_TESTING_daemons_continue_startup (pg);
3272       peer_start_time = GNUNET_TIME_absolute_get ();
3273     }
3274   else
3275     {
3276       GNUNET_SCHEDULER_cancel (die_task);
3277       die_task = GNUNET_SCHEDULER_add_now (&end_badly,
3278                                            "from create topology (bad return)");
3279     }
3280   GNUNET_free_non_null(blacklist_transports);
3281   GNUNET_SCHEDULER_cancel (die_task);
3282   die_task
3283       = GNUNET_SCHEDULER_add_delayed (
3284                                       GNUNET_TIME_relative_multiply (
3285                                                                      seconds_per_peer_start,
3286                                                                      num_peers),
3287                                       &end_badly,
3288                                       "from continue startup (timeout)");
3289 }
3290
3291 /**
3292  * Callback indicating that the hostkey was created for a peer.
3293  *
3294  * @param cls NULL
3295  * @param id the peer identity
3296  * @param d the daemon handle (pretty useless at this point, remove?)
3297  * @param emsg non-null on failure
3298  */
3299 static void
3300 hostkey_callback(void *cls, const struct GNUNET_PeerIdentity *id,
3301                  struct GNUNET_TESTING_Daemon *d, const char *emsg)
3302 {
3303   char * revision_str;
3304   if (emsg != NULL)
3305     {
3306       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3307                   "Hostkey callback received error: %s\n", emsg);
3308     }
3309
3310 #if VERBOSE > 1
3311   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
3312       "Hostkey (%d/%d) created for peer `%s'\n",
3313       num_peers - peers_left, num_peers, GNUNET_i2s(id));
3314 #endif
3315
3316   peers_left--;
3317   if (GNUNET_YES == update_meter (hostkey_meter))
3318     {
3319       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3320                   "All %d hostkeys created, now creating topology!\n",
3321                   num_peers);
3322
3323       GNUNET_asprintf (&revision_str, "%llu", revision);
3324       if (GNUNET_YES == insert_gauger_data)
3325         {
3326           if (GNUNET_YES == GNUNET_CONFIGURATION_have_value (config, "TESTING",
3327                                                              "HOSTKEYSFILE"))
3328             {
3329               GAUGER_ID("DHT_TESTING",
3330                   "HOSTKEY_GENERATION",
3331                   GNUNET_TIME_absolute_get_duration(hostkey_start_time).rel_value / (double)num_peers,
3332                   "ms/hostkey", revision_str);
3333             }
3334           else
3335             {
3336               GAUGER_ID("DHT_TESTING",
3337                   "HOSTKEY_GENERATION_REAL",
3338                   GNUNET_TIME_absolute_get_duration(hostkey_start_time).rel_value / (double)num_peers,
3339                   "ms/hostkey", revision_str);
3340             }
3341         }
3342
3343       GNUNET_free(revision_str);
3344
3345       GNUNET_SCHEDULER_cancel (die_task);
3346       /* Set up task in case topology creation doesn't finish
3347        * within a reasonable amount of time */
3348       die_task = GNUNET_SCHEDULER_add_delayed (DEFAULT_TOPOLOGY_TIMEOUT,
3349                                                &end_badly,
3350                                                "from create_topology");
3351       GNUNET_SCHEDULER_add_now (&create_topology, NULL);
3352       ok = 0;
3353     }
3354 }
3355
3356 static void
3357 run(void *cls, char * const *args, const char *cfgfile,
3358     const struct GNUNET_CONFIGURATION_Handle *cfg)
3359 {
3360   struct stat frstat;
3361   struct GNUNET_DHTLOG_TrialInfo trial_info;
3362   struct GNUNET_TESTING_Host *hosts;
3363   struct GNUNET_TESTING_Host *temphost;
3364   struct GNUNET_TESTING_Host *tempnext;
3365   char *topology_str;
3366   char *connect_topology_str;
3367   char *blacklist_topology_str;
3368   char *connect_topology_option_str;
3369   char *connect_topology_option_modifier_string;
3370   char *trialmessage;
3371   char *topology_percentage_str;
3372   float topology_percentage;
3373   char *topology_probability_str;
3374   char *hostfile;
3375   float topology_probability;
3376   unsigned long long temp_config_number;
3377   int stop_closest;
3378   int stop_found;
3379   int strict_kademlia;
3380   char *buf;
3381   char *data;
3382   char *churn_data;
3383   char *churn_filename;
3384   int count;
3385   int ret;
3386   int line_number;
3387   int k;
3388
3389   config = cfg;
3390   rounds_finished = 0;
3391   memset (&trial_info, 0, sizeof(struct GNUNET_DHTLOG_TrialInfo));
3392   /* Get path from configuration file */
3393   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "paths",
3394                                                            "servicehome",
3395                                                            &test_directory))
3396     {
3397       ok = 404;
3398       return;
3399     }
3400
3401   /* Get number of peers to start from configuration */
3402   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3403                                                               "num_peers",
3404                                                               &num_peers))
3405     {
3406       GNUNET_log (
3407                   GNUNET_ERROR_TYPE_WARNING,
3408                   "Number of peers must be specified in section %s option %s\n",
3409                   "TESTING", "NUM_PEERS");
3410     }
3411   GNUNET_assert(num_peers > 0 && num_peers < ULONG_MAX);
3412
3413   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3414                                                           "connect_timeout",
3415                                                           &temp_config_number))
3416     connect_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3417                                                      temp_config_number);
3418   else
3419     {
3420       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3421                   "testing", "connect_timeout");
3422       return;
3423     }
3424
3425   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3426                                                           "connect_attempts",
3427                                                           &connect_attempts))
3428     {
3429       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3430                   "testing", "connect_attempts");
3431       return;
3432     }
3433
3434   if (GNUNET_OK
3435       != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3436                                                 "max_outstanding_connections",
3437                                                 &max_outstanding_connections))
3438     {
3439       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3440                   "testing", "max_outstanding_connections");
3441       return;
3442     }
3443
3444   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "testing",
3445                                                           "max_concurrent_ssh",
3446                                                           &max_concurrent_ssh))
3447     {
3448       GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Must provide option %s:%s!\n",
3449                   "testing", "max_concurrent_ssh");
3450       return;
3451     }
3452
3453   /**
3454    * Get DHT specific testing options.
3455    */
3456   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3457                                                            "mysql_logging"))
3458       || (GNUNET_YES
3459           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3460                                                    "mysql_logging_extended"))
3461       || (GNUNET_YES
3462           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3463                                                    "mysql_logging_minimal")))
3464     {
3465       if (GNUNET_YES
3466           == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3467                                                    "mysql_logging_minimal"))
3468         dhtlog_minimal = GNUNET_YES;
3469
3470       dhtlog_handle = GNUNET_DHTLOG_connect (cfg);
3471       if (dhtlog_handle == NULL)
3472         {
3473           GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3474                       "Could not connect to mysql server for logging, will NOT log dht operations!");
3475           ok = 3306;
3476           return;
3477         }
3478     }
3479
3480   stop_closest = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3481                                                        "stop_on_closest");
3482   if (stop_closest == GNUNET_SYSERR)
3483     stop_closest = GNUNET_NO;
3484
3485   stop_found = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht", "stop_found");
3486   if (stop_found == GNUNET_SYSERR)
3487     stop_found = GNUNET_NO;
3488
3489   strict_kademlia = GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3490                                                           "strict_kademlia");
3491   if (strict_kademlia == GNUNET_SYSERR)
3492     strict_kademlia = GNUNET_NO;
3493
3494   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3495                                                           "comment",
3496                                                           &trialmessage))
3497     trialmessage = NULL;
3498
3499   churn_data = NULL;
3500   /** Check for a churn file to do churny simulation */
3501   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing",
3502                                                           "churn_file",
3503                                                           &churn_filename))
3504     {
3505       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Reading churn data from %s\n",
3506                   churn_filename);
3507       if (GNUNET_OK != GNUNET_DISK_file_test (churn_filename))
3508         {
3509           GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Error reading churn file!\n");
3510           GNUNET_free_non_null(trialmessage);
3511           GNUNET_free(churn_filename);
3512           return;
3513         }
3514       if ((0 != STAT (churn_filename, &frstat)) || (frstat.st_size == 0))
3515         {
3516           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3517                       "Could not open file specified for churn data, ending test!");
3518           ok = 1119;
3519           GNUNET_free_non_null(trialmessage);
3520           GNUNET_free(churn_filename);
3521           return;
3522         }
3523
3524       churn_data = GNUNET_malloc_large (frstat.st_size);
3525       GNUNET_assert(churn_data != NULL);
3526       if (frstat.st_size != GNUNET_DISK_fn_read (churn_filename, churn_data,
3527                                                  frstat.st_size))
3528         {
3529           GNUNET_log (
3530                       GNUNET_ERROR_TYPE_ERROR,
3531                       "Could not read file %s specified for churn, ending test!",
3532                       churn_filename);
3533           GNUNET_free (churn_filename);
3534           GNUNET_free (churn_data);
3535           GNUNET_free_non_null(trialmessage);
3536           return;
3537         }
3538
3539       GNUNET_free_non_null(churn_filename);
3540
3541       buf = churn_data;
3542       count = 0;
3543       /* Read the first line */
3544       while (count < frstat.st_size)
3545         {
3546           count++;
3547           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3548             {
3549               churn_data[count] = '\0';
3550               if (1 != sscanf (buf, "%u", &churn_rounds))
3551                 {
3552                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3553                               "Failed to read number of rounds from churn file, ending test!\n");
3554                   ret = 4200;
3555                   GNUNET_free_non_null(trialmessage);
3556                   GNUNET_free_non_null(churn_data);
3557                   return;
3558                 }
3559               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3560                           "Read %u rounds from churn file\n", churn_rounds);
3561               buf = &churn_data[count + 1];
3562               churn_array = GNUNET_malloc(sizeof(unsigned int) * churn_rounds);
3563               break; /* Done with this part */
3564             }
3565         }
3566
3567       if (GNUNET_OK
3568           != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3569                                                     "churns_per_round",
3570                                                     &churns_per_round))
3571         {
3572           churns_per_round = (unsigned long long) churn_rounds;
3573         }
3574
3575       line_number = 0;
3576       while ((count < frstat.st_size) && (line_number < churn_rounds))
3577         {
3578           count++;
3579           if (((churn_data[count] == '\n')) && (buf != &churn_data[count]))
3580             {
3581               churn_data[count] = '\0';
3582
3583               ret = sscanf (buf, "%u", &churn_array[line_number]);
3584               if (1 == ret)
3585                 {
3586                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3587                               "Read %u peers in round %u\n",
3588                               churn_array[line_number], line_number);
3589                   line_number++;
3590                 }
3591               else
3592                 {
3593                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3594                               "Error reading line `%s' in hostfile\n", buf);
3595                   buf = &churn_data[count + 1];
3596                   continue;
3597                 }
3598               buf = &churn_data[count + 1];
3599             }
3600           else if (churn_data[count] == '\n') /* Blank line */
3601             buf = &churn_data[count + 1];
3602         }
3603     }
3604   GNUNET_free_non_null(churn_data);
3605
3606   /* Check for a hostfile containing user@host:port triples */
3607   if (GNUNET_OK
3608       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "hostfile",
3609                                                 &hostfile))
3610     hostfile = NULL;
3611
3612   hosts = NULL;
3613   temphost = NULL;
3614   data = NULL;
3615   if (hostfile != NULL)
3616     {
3617       if (GNUNET_OK != GNUNET_DISK_file_test (hostfile))
3618         GNUNET_DISK_fn_write (hostfile, NULL, 0, GNUNET_DISK_PERM_USER_READ
3619             | GNUNET_DISK_PERM_USER_WRITE);
3620       if ((0 != STAT (hostfile, &frstat)) || (frstat.st_size == 0))
3621         {
3622           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
3623                       "Could not open file specified for host list, ending test!");
3624           ok = 1119;
3625           GNUNET_free_non_null(trialmessage);
3626           GNUNET_free(hostfile);
3627           return;
3628         }
3629
3630       data = GNUNET_malloc_large (frstat.st_size);
3631       GNUNET_assert(data != NULL);
3632       if (frstat.st_size
3633           != GNUNET_DISK_fn_read (hostfile, data, frstat.st_size))
3634         {
3635           GNUNET_log (
3636                       GNUNET_ERROR_TYPE_ERROR,
3637                       "Could not read file %s specified for host list, ending test!",
3638                       hostfile);
3639           GNUNET_free (hostfile);
3640           GNUNET_free (data);
3641           GNUNET_free_non_null(trialmessage);
3642           return;
3643         }
3644
3645       GNUNET_free_non_null(hostfile);
3646
3647       buf = data;
3648       count = 0;
3649       while (count < frstat.st_size - 1)
3650         {
3651           count++;
3652           if (((data[count] == '\n')) && (buf != &data[count]))
3653             {
3654               data[count] = '\0';
3655               temphost = GNUNET_malloc(sizeof(struct GNUNET_TESTING_Host));
3656               ret = sscanf (buf, "%a[a-zA-Z0-9_]@%a[a-zA-Z0-9.]:%hd",
3657                             &temphost->username, &temphost->hostname,
3658                             &temphost->port);
3659               if (3 == ret)
3660                 {
3661                   GNUNET_log (
3662                               GNUNET_ERROR_TYPE_DEBUG,
3663                               "Successfully read host %s, port %d and user %s from file\n",
3664                               temphost->hostname, temphost->port,
3665                               temphost->username);
3666                 }
3667               else
3668                 {
3669                   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3670                               "Error reading line `%s' in hostfile\n", buf);
3671                   GNUNET_free(temphost);
3672                   buf = &data[count + 1];
3673                   continue;
3674                 }
3675               temphost->next = hosts;
3676               hosts = temphost;
3677               buf = &data[count + 1];
3678             }
3679           else if ((data[count] == '\n') || (data[count] == '\0'))
3680             buf = &data[count + 1];
3681         }
3682     }
3683   GNUNET_free_non_null(data);
3684   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3685                                                           "malicious_getters",
3686                                                           &malicious_getters))
3687     malicious_getters = 0;
3688
3689   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3690                                                           "malicious_putters",
3691                                                           &malicious_putters))
3692     malicious_putters = 0;
3693
3694   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3695                                                           "malicious_droppers",
3696                                                           &malicious_droppers))
3697     malicious_droppers = 0;
3698
3699   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3700                                                           "settle_time",
3701                                                           &settle_time))
3702     settle_time = 0;
3703
3704   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3705                                                               "dht_testing",
3706                                                               "num_puts",
3707                                                               &num_puts))
3708     num_puts = num_peers;
3709
3710   if (GNUNET_SYSERR
3711       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3712                                                 "put_replication",
3713                                                 &put_replication))
3714     put_replication = DEFAULT_PUT_REPLICATION;
3715
3716   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3717                                                               "dht_testing",
3718                                                               "num_gets",
3719                                                               &num_gets))
3720     num_gets = num_peers;
3721
3722   if (GNUNET_SYSERR
3723       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3724                                                 "get_replication",
3725                                                 &get_replication))
3726     get_replication = DEFAULT_GET_REPLICATION;
3727
3728   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3729                                                           "find_peer_delay",
3730                                                           &temp_config_number))
3731     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3732                                                      temp_config_number);
3733   else
3734     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3735
3736   if (GNUNET_OK
3737       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3738                                                 "concurrent_find_peers",
3739                                                 &temp_config_number))
3740     max_outstanding_find_peers = temp_config_number;
3741   else
3742     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3743
3744   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3745                                                           "get_timeout",
3746                                                           &temp_config_number))
3747     get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3748                                                  temp_config_number);
3749   else
3750     get_timeout = DEFAULT_GET_TIMEOUT;
3751
3752   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3753                                                           "concurrent_puts",
3754                                                           &temp_config_number))
3755     max_outstanding_puts = temp_config_number;
3756   else
3757     max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS;
3758
3759   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3760                                                           "concurrent_gets",
3761                                                           &temp_config_number))
3762     max_outstanding_gets = temp_config_number;
3763   else
3764     max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS;
3765
3766   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3767                                                           "timeout",
3768                                                           &temp_config_number))
3769     all_get_timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3770                                                      temp_config_number);
3771   else
3772     all_get_timeout.rel_value = get_timeout.rel_value * num_gets;
3773
3774   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3775                                                           "get_delay",
3776                                                           &temp_config_number))
3777     get_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3778                                                temp_config_number);
3779   else
3780     get_delay = DEFAULT_GET_DELAY;
3781
3782   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3783                                                           "put_delay",
3784                                                           &temp_config_number))
3785     put_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3786                                                temp_config_number);
3787   else
3788     put_delay = DEFAULT_PUT_DELAY;
3789
3790   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3791                                                           "peer_start_timeout",
3792                                                           &temp_config_number))
3793     seconds_per_peer_start
3794         = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3795                                          temp_config_number);
3796   else
3797     seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START;
3798
3799   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3800                                                           "data_size",
3801                                                           &temp_config_number))
3802     test_data_size = temp_config_number;
3803   else
3804     test_data_size = DEFAULT_TEST_DATA_SIZE;
3805
3806   /**
3807    * Get DHT testing related options.
3808    */
3809   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3810                                                           "REPLICATE_SAME"))
3811     replicate_same = GNUNET_YES;
3812
3813   if (GNUNET_SYSERR
3814       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3815                                                 "target_completions",
3816                                                 &target_completions))
3817     target_completions = 0; /* Not required, on stack */
3818
3819   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3820                                                           "GET_FROM_SAME"))
3821     get_from_same = GNUNET_YES;
3822
3823   if (GNUNET_NO
3824       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3825                                               "MALICIOUS_GET_FREQUENCY",
3826                                               &malicious_get_frequency))
3827     malicious_get_frequency = DEFAULT_MALICIOUS_GET_FREQUENCY;
3828
3829   if (GNUNET_NO
3830       == GNUNET_CONFIGURATION_get_value_time (cfg, "DHT_TESTING",
3831                                               "MALICIOUS_PUT_FREQUENCY",
3832                                               &malicious_put_frequency))
3833     malicious_put_frequency = DEFAULT_MALICIOUS_PUT_FREQUENCY;
3834
3835   if (GNUNET_YES
3836       == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3837                                                "MALICIOUS_AFTER_SETTLE"))
3838     malicious_after_settle = GNUNET_YES;
3839
3840   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT_TESTING",
3841                                                           "MALICIOUS_SYBIL"))
3842     {
3843       /* Set up the malicious target at random for this round */
3844       GNUNET_CRYPTO_hash_create_random (GNUNET_CRYPTO_QUALITY_WEAK,
3845                                         &sybil_target);
3846       malicious_sybil = GNUNET_YES;
3847     }
3848
3849   /* Create the bloomfilter for choosing which peers to set malicious */
3850
3851   /* Bloomfilter size must be 2^k for some integer k */
3852   k = 1;
3853   while (1 << k < malicious_droppers)
3854     k++;
3855   if (malicious_droppers > 0)
3856     malicious_bloom = GNUNET_CONTAINER_bloomfilter_init (NULL, 1 << k,
3857                                                          DHT_BLOOM_K);
3858
3859   /* The normal behavior of the DHT is to do find peer requests
3860    * on its own.  Only if this is explicitly turned off should
3861    * the testing driver issue find peer requests (even though
3862    * this is likely the default when testing).
3863    */
3864   if (GNUNET_NO == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3865                                                          "do_find_peer"))
3866     do_find_peer = GNUNET_YES;
3867
3868   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht_testing",
3869                                                           "insert_gauger_data"))
3870     insert_gauger_data = GNUNET_YES;
3871
3872   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3873                                                           "republish"))
3874     in_dht_replication = GNUNET_YES;
3875
3876   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3877                                                            "TRIAL_TO_RUN",
3878                                                            &trial_to_run))
3879     trial_to_run = 0;
3880
3881   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3882                                                            "REVISION",
3883                                                            &revision))
3884     revision = 0;
3885
3886   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3887                                                            "FIND_PEER_DELAY",
3888                                                            &temp_config_number))
3889     find_peer_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS,
3890                                                      temp_config_number);
3891   else
3892     find_peer_delay = DEFAULT_FIND_PEER_DELAY;
3893
3894   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3895                                                            "ROUND_DELAY",
3896                                                            &round_delay))
3897     round_delay = 0;
3898
3899   if (GNUNET_NO
3900       == GNUNET_CONFIGURATION_get_value_number (cfg, "DHT_TESTING",
3901                                                 "OUTSTANDING_FIND_PEERS",
3902                                                 &max_outstanding_find_peers))
3903     max_outstanding_find_peers = DEFAULT_MAX_OUTSTANDING_FIND_PEERS;
3904
3905   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (cfg, "dht",
3906                                                           "strict_kademlia"))
3907     max_outstanding_find_peers = max_outstanding_find_peers * 1;
3908
3909   find_peer_offset = GNUNET_TIME_relative_divide (find_peer_delay,
3910                                                   max_outstanding_find_peers);
3911
3912   if (GNUNET_SYSERR == GNUNET_CONFIGURATION_get_value_number (cfg,
3913                                                               "dht_testing",
3914                                                               "num_rounds",
3915                                                               &total_rounds))
3916     total_rounds = 1;
3917
3918   if ((GNUNET_SYSERR
3919       == GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing",
3920                                                 "target_total_connections",
3921                                                 &target_total_connections))
3922       || (target_total_connections == 0))
3923     target_total_connections = connection_estimate (num_peers,
3924                                                     DEFAULT_BUCKET_SIZE);
3925
3926   topology_str = NULL;
3927   if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3928                                                             "topology",
3929                                                             &topology_str))
3930       && (GNUNET_NO == GNUNET_TESTING_topology_get (&topology, topology_str)))
3931     {
3932       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
3933                   "Invalid topology `%s' given for section %s option %s\n",
3934                   topology_str, "TESTING", "TOPOLOGY");
3935       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
3936     }
3937
3938   if (GNUNET_OK
3939       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage",
3940                                                 &topology_percentage_str))
3941     topology_percentage = 0.5;
3942   else
3943     {
3944       topology_percentage = atof (topology_percentage_str);
3945       GNUNET_free(topology_percentage_str);
3946     }
3947
3948   if (GNUNET_OK
3949       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability",
3950                                                 &topology_probability_str))
3951     topology_probability = 0.5;
3952   else
3953     {
3954       topology_probability = atof (topology_probability_str);
3955       GNUNET_free(topology_probability_str);
3956     }
3957
3958   if ((GNUNET_YES
3959       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3960                                                 "connect_topology",
3961                                                 &connect_topology_str))
3962       && (GNUNET_NO == GNUNET_TESTING_topology_get (&connect_topology,
3963                                                     connect_topology_str)))
3964     {
3965       GNUNET_log (
3966                   GNUNET_ERROR_TYPE_WARNING,
3967                   "Invalid connect topology `%s' given for section %s option %s\n",
3968                   connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
3969     }
3970   GNUNET_free_non_null(connect_topology_str);
3971
3972   if ((GNUNET_YES
3973       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
3974                                                 "connect_topology_option",
3975                                                 &connect_topology_option_str))
3976       && (GNUNET_NO
3977           == GNUNET_TESTING_topology_option_get (&connect_topology_option,
3978                                                  connect_topology_option_str)))
3979     {
3980       GNUNET_log (
3981                   GNUNET_ERROR_TYPE_WARNING,
3982                   "Invalid connect topology option `%s' given for section %s option %s\n",
3983                   connect_topology_option_str, "TESTING",
3984                   "CONNECT_TOPOLOGY_OPTION");
3985       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
3986     }
3987   GNUNET_free_non_null(connect_topology_option_str);
3988
3989   if (GNUNET_YES
3990       == GNUNET_CONFIGURATION_get_value_string (
3991                                                 cfg,
3992                                                 "testing",
3993                                                 "connect_topology_option_modifier",
3994                                                 &connect_topology_option_modifier_string))
3995     {
3996       if (sscanf (connect_topology_option_modifier_string, "%lf",
3997                   &connect_topology_option_modifier) != 1)
3998         {
3999           GNUNET_log (
4000                       GNUNET_ERROR_TYPE_WARNING,
4001                       _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
4002                       connect_topology_option_modifier_string,
4003                       "connect_topology_option_modifier", "TESTING");
4004         }
4005       GNUNET_free (connect_topology_option_modifier_string);
4006     }
4007
4008   if (GNUNET_YES
4009       != GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
4010                                                 "blacklist_transports",
4011                                                 &blacklist_transports))
4012     blacklist_transports = NULL;
4013
4014   if ((GNUNET_YES
4015       == GNUNET_CONFIGURATION_get_value_string (cfg, "testing",
4016                                                 "blacklist_topology",
4017                                                 &blacklist_topology_str))
4018       && (GNUNET_NO == GNUNET_TESTING_topology_get (&blacklist_topology,
4019                                                     blacklist_topology_str)))
4020     {
4021       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4022                   "Invalid topology `%s' given for section %s option %s\n",
4023                   topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
4024     }
4025   GNUNET_free_non_null(topology_str);
4026   GNUNET_free_non_null(blacklist_topology_str);
4027
4028   /* Set peers_left so we know when all peers started */
4029   peers_left = num_peers;
4030
4031   /* Set up a task to end testing if peer start fails */
4032   die_task
4033       = GNUNET_SCHEDULER_add_delayed (
4034                                       GNUNET_TIME_relative_multiply (
4035                                                                      seconds_per_peer_start,
4036                                                                      num_peers),
4037                                       &end_badly,
4038                                       "didn't generate all hostkeys within allowed startup time!");
4039
4040   if (dhtlog_handle == NULL)
4041     GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "dhtlog_handle is NULL!");
4042
4043   trial_info.other_identifier = (unsigned int) trial_to_run;
4044   trial_info.num_nodes = peers_left;
4045   trial_info.topology = topology;
4046   trial_info.blacklist_topology = blacklist_topology;
4047   trial_info.connect_topology = connect_topology;
4048   trial_info.connect_topology_option = connect_topology_option;
4049   trial_info.connect_topology_option_modifier
4050       = connect_topology_option_modifier;
4051   trial_info.topology_percentage = topology_percentage;
4052   trial_info.topology_probability = topology_probability;
4053   trial_info.puts = num_puts;
4054   trial_info.gets = num_gets;
4055   trial_info.concurrent = max_outstanding_gets;
4056   trial_info.settle_time = settle_time;
4057   trial_info.num_rounds = total_rounds;
4058   trial_info.malicious_getters = malicious_getters;
4059   trial_info.malicious_putters = malicious_putters;
4060   trial_info.malicious_droppers = malicious_droppers;
4061   trial_info.malicious_get_frequency = malicious_get_frequency.rel_value;
4062   trial_info.malicious_put_frequency = malicious_put_frequency.rel_value;
4063   trial_info.stop_closest = stop_closest;
4064   trial_info.stop_found = stop_found;
4065   trial_info.strict_kademlia = strict_kademlia;
4066
4067   if (trialmessage != NULL)
4068     trial_info.message = trialmessage;
4069   else
4070     trial_info.message = "";
4071
4072   if (dhtlog_handle != NULL)
4073     dhtlog_handle->insert_trial (&trial_info);
4074
4075   GNUNET_free_non_null(trialmessage);
4076
4077   hostkey_meter = create_meter (peers_left, "Hostkeys created ", GNUNET_YES);
4078   peer_start_meter = create_meter (peers_left, "Peers started ", GNUNET_YES);
4079
4080   put_meter = create_meter (num_puts, "Puts completed ", GNUNET_YES);
4081   get_meter = create_meter (num_gets, "Gets completed ", GNUNET_YES);
4082   hostkey_start_time = GNUNET_TIME_absolute_get ();
4083   pg
4084       = GNUNET_TESTING_daemons_start (
4085                                       cfg,
4086                                       peers_left,
4087                                       max_outstanding_connections,
4088                                       max_concurrent_ssh,
4089                                       GNUNET_TIME_relative_multiply (
4090                                                                      seconds_per_peer_start,
4091                                                                      num_peers),
4092                                       &hostkey_callback, NULL,
4093                                       &peers_started_callback, NULL,
4094                                       &topology_callback, NULL, hosts);
4095   temphost = hosts;
4096   while (temphost != NULL)
4097     {
4098       tempnext = temphost->next;
4099       GNUNET_free (temphost->username);
4100       GNUNET_free (temphost->hostname);
4101       GNUNET_free (temphost);
4102       temphost = tempnext;
4103     }
4104 }
4105
4106 int
4107 main(int argc, char *argv[])
4108 {
4109   int ret;
4110   struct GNUNET_GETOPT_CommandLineOption options[] =
4111     {
4112     GNUNET_GETOPT_OPTION_END };
4113
4114   ret = GNUNET_PROGRAM_run (argc, argv, "gnunet-dht-driver", "nohelp", options,
4115                             &run, &ok);
4116
4117   if (malicious_bloom != NULL)
4118     GNUNET_CONTAINER_bloomfilter_free (malicious_bloom);
4119
4120   if (ret != GNUNET_OK)
4121     {
4122       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4123                   "`gnunet-dht-driver': Failed with error code %d\n", ret);
4124     }
4125
4126   /**
4127    * Need to remove base directory, subdirectories taken care
4128    * of by the testing framework.
4129    */
4130   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
4131     {
4132       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
4133                   "Failed to remove testing directory %s\n", test_directory);
4134     }
4135   return ret;
4136 }
4137
4138 /* end of gnunet-dht-driver.c */