src/dht/test_dht_multipeer.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file dht/test_dht_multipeer.c
  22  * @brief testcase for testing DHT service with
  23  *        multiple peers.
  24  */
  25 #include "platform.h"
  26 #include "gnunet_testing_lib.h"
  27 #include "gnunet_core_service.h"
  28 #include "gnunet_dht_service.h"
  29
  30 /* DEFINES */
  31 #define VERBOSE GNUNET_NO
  32
  33 /* Timeout for entire testcase */
  34 #define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  35
  36 /* Timeout for waiting for replies to get requests */
  37 #define GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  38
  39 /* Timeout for waiting for gets to complete */
  40 #define GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  41
  42 /* Timeout for waiting for puts to complete */
  43 #define PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  44
  45 #define SECONDS_PER_PEER_START 45
  46
  47 /* If number of peers not in config file, use this number */
  48 #define DEFAULT_NUM_PEERS 5
  49
  50 #define TEST_DATA_SIZE 8
  51
  52 #define MAX_OUTSTANDING_PUTS 10
  53
  54 #define MAX_OUTSTANDING_GETS 10
  55
  56 #define PATH_TRACKING GNUNET_YES
  57
  58 /* Structs */
  59
  60 struct TestPutContext
  61 {
  62   /**
  63    * This is a linked list
  64    */
  65   struct TestPutContext *next;
  66
  67   /**
  68    * Handle to the first peers DHT service (via the API)
  69    */
  70   struct GNUNET_DHT_Handle *dht_handle;
  71
  72   /**
  73    *  Handle to the PUT peer daemon
  74    */
  75   struct GNUNET_TESTING_Daemon *daemon;
  76
  77   /**
  78    *  Identifier for this PUT
  79    */
  80   uint32_t uid;
  81
  82   /**
  83    * Task for disconnecting DHT handles
  84    */
  85   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
  86 };
  87
  88 struct TestGetContext
  89 {
  90   /* This is a linked list */
  91   struct TestGetContext *next;
  92
  93   /**
  94    * Handle to the first peers DHT service (via the API)
  95    */
  96   struct GNUNET_DHT_Handle *dht_handle;
  97
  98   /**
  99    * Handle for the DHT get request
 100    */
 101   struct GNUNET_DHT_GetHandle *get_handle;
 102
 103   /**
 104    *  Handle to the GET peer daemon
 105    */
 106   struct GNUNET_TESTING_Daemon *daemon;
 107
 108   /**
 109    *  Identifier for this GET
 110    */
 111   uint32_t uid;
 112
 113   /**
 114    * Task for disconnecting DHT handles (and stopping GET)
 115    */
 116   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 117
 118   /**
 119    * Whether or not this request has been fulfilled already.
 120    */
 121   int succeeded;
 122 };
 123
 124 /* Globals */
 125
 126 /**
 127  * List of GETS to perform
 128  */
 129 struct TestGetContext *all_gets;
 130
 131 /**
 132  * List of PUTS to perform
 133  */
 134 struct TestPutContext *all_puts;
 135
 136 /**
 137  * Directory to store temp data in, defined in config file
 138  */
 139 static char *test_directory;
 140
 141 /**
 142  * Variable used to store the number of connections we should wait for.
 143  */
 144 static unsigned int expected_connections;
 145
 146 /**
 147  * Variable used to keep track of how many peers aren't yet started.
 148  */
 149 static unsigned long long peers_left;
 150
 151 /**
 152  * Handle to the set of all peers run for this test.
 153  */
 154 static struct GNUNET_TESTING_PeerGroup *pg;
 155
 156
 157 /**
 158  * Total number of peers to run, set based on config file.
 159  */
 160 static unsigned long long num_peers;
 161
 162 /**
 163  * Total number of items to insert.
 164  */
 165 static unsigned long long num_puts;
 166
 167 /**
 168  * Total number of items to attempt to get.
 169  */
 170 static unsigned long long num_gets;
 171
 172 /**
 173  * How many puts do we currently have in flight?
 174  */
 175 static unsigned long long outstanding_puts;
 176
 177 /**
 178  * How many puts are done?
 179  */
 180 static unsigned long long puts_completed;
 181
 182 /**
 183  * How many puts do we currently have in flight?
 184  */
 185 static unsigned long long outstanding_gets;
 186
 187 /**
 188  * How many gets are done?
 189  */
 190 static unsigned long long gets_completed;
 191
 192 /**
 193  * How many gets failed?
 194  */
 195 static unsigned long long gets_failed;
 196
 197 /**
 198  * Global used to count how many connections we have currently
 199  * been notified about (how many times has topology_callback been called
 200  * with success?)
 201  */
 202 static unsigned int total_connections;
 203
 204 /**
 205  * Global used to count how many failed connections we have
 206  * been notified about (how many times has topology_callback
 207  * been called with failure?)
 208  */
 209 static unsigned int failed_connections;
 210
 211 enum GNUNET_DHT_RouteOption route_option;
 212
 213 /* Task handle to use to schedule test failure */
 214 GNUNET_SCHEDULER_TaskIdentifier die_task;
 215
 216 static char *blacklist_transports;
 217
 218 static enum GNUNET_TESTING_Topology topology;
 219
 220 static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 221
 222 static enum GNUNET_TESTING_Topology connection_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 223
 224 static enum GNUNET_TESTING_TopologyOption connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 225
 226 static double connect_topology_option_modifier = 0.0;
 227
 228 /* Global return value (0 for success, anything else for failure) */
 229 static int ok;
 230
 231 /**
 232  * Check whether peers successfully shut down.
 233  */
 234 void shutdown_callback (void *cls,
 235                         const char *emsg)
 236 {
 237   if (emsg != NULL)
 238     {
 239       if (ok == 0)
 240         ok = 2;
 241     }
 242 }
 243
 244 /**
 245  * Task to release DHT handles for PUT
 246  */
 247 static void
 248 put_disconnect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 249 {
 250   struct TestPutContext *test_put = cls;
 251   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 252   GNUNET_DHT_disconnect(test_put->dht_handle);
 253   test_put->dht_handle = NULL;
 254 }
 255
 256 /**
 257  * Function scheduled to be run on the successful completion of this
 258  * testcase.
 259  */
 260 static void
 261 finish_testing (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 262 {
 263   GNUNET_assert (pg != NULL);
 264   struct TestPutContext *test_put = all_puts;
 265   struct TestGetContext *test_get = all_gets;
 266
 267   while (test_put != NULL)
 268     {
 269       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 270         GNUNET_SCHEDULER_cancel(test_put->disconnect_task);
 271       if (test_put->dht_handle != NULL)
 272         GNUNET_DHT_disconnect(test_put->dht_handle);
 273       test_put = test_put->next;
 274     }
 275
 276   while (test_get != NULL)
 277     {
 278       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 279         GNUNET_SCHEDULER_cancel(test_get->disconnect_task);
 280       if (test_get->get_handle != NULL)
 281         GNUNET_DHT_get_stop(test_get->get_handle);
 282       if (test_get->dht_handle != NULL)
 283         GNUNET_DHT_disconnect(test_get->dht_handle);
 284       test_get = test_get->next;
 285     }
 286
 287   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 288   ok = 0;
 289 }
 290
 291
 292 /**
 293  * Check if the get_handle is being used, if so stop the request.  Either
 294  * way, schedule the end_badly_cont function which actually shuts down the
 295  * test.
 296  */
 297 static void
 298 end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 299 {
 300   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failing test with error: `%s'!\n", (char *)cls);
 301
 302   struct TestPutContext *test_put = all_puts;
 303   struct TestGetContext *test_get = all_gets;
 304
 305   while (test_put != NULL)
 306     {
 307       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 308         GNUNET_SCHEDULER_cancel(test_put->disconnect_task);
 309       if (test_put->dht_handle != NULL)
 310         GNUNET_DHT_disconnect(test_put->dht_handle);
 311       test_put = test_put->next;
 312     }
 313
 314   while (test_get != NULL)
 315     {
 316       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 317         GNUNET_SCHEDULER_cancel(test_get->disconnect_task);
 318       if (test_get->get_handle != NULL)
 319         GNUNET_DHT_get_stop(test_get->get_handle);
 320       if (test_get->dht_handle != NULL)
 321         GNUNET_DHT_disconnect(test_get->dht_handle);
 322       test_get = test_get->next;
 323     }
 324
 325   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 326   ok = 1;
 327 }
 328
 329 /**
 330  * Task to release DHT handle associated with GET request.
 331  */
 332 static void
 333 get_stop_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 334 {
 335   struct TestGetContext *test_get = cls;
 336   outstanding_gets--; /* GET is really finished */
 337   GNUNET_DHT_disconnect(test_get->dht_handle);
 338   test_get->dht_handle = NULL;
 339
 340   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
 341   if ((gets_completed == num_gets) && (outstanding_gets == 0))/* All gets successful */
 342     {
 343       GNUNET_SCHEDULER_cancel (die_task);
 344       //GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5), &get_topology, NULL);
 345       GNUNET_SCHEDULER_add_now(&finish_testing, NULL);
 346     }
 347   else if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0)) /* Had some failures */
 348     {
 349       GNUNET_SCHEDULER_cancel(die_task);
 350       GNUNET_SCHEDULER_add_now(&end_badly, "not all gets succeeded!\n");
 351     }
 352 }
 353
 354 /**
 355  * Task to release get handle.
 356  */
 357 static void
 358 get_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 359 {
 360   struct TestGetContext *test_get = cls;
 361   GNUNET_HashCode search_key; /* Key stored under */
 362   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 363
 364   memset(original_data, test_get->uid, sizeof(original_data));
 365   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 366
 367   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
 368     {
 369       gets_failed++;
 370       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Get from peer %s for key %s failed!\n", test_get->daemon->shortname, GNUNET_h2s(&search_key));
 371     }
 372   GNUNET_assert(test_get->get_handle != NULL);
 373   GNUNET_DHT_get_stop(test_get->get_handle);
 374   GNUNET_SCHEDULER_add_now (&get_stop_finished, test_get);
 375   test_get->get_handle = NULL;
 376   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 377 }
 378
 379 /**
 380  * Iterator called if the GET request initiated returns a response.
 381  *
 382  * @param cls closure
 383  * @param exp when will this value expire
 384  * @param key key of the result
 385  * @param type type of the result
 386  * @param size number of bytes in data
 387  * @param data pointer to the result data
 388  */
 389 void get_result_iterator (void *cls,
 390                           struct GNUNET_TIME_Absolute exp,
 391                           const GNUNET_HashCode * key,
 392                           const struct GNUNET_PeerIdentity * const *get_path,
 393                           const struct GNUNET_PeerIdentity * const *put_path,
 394                           enum GNUNET_BLOCK_Type type,
 395                           size_t size,
 396                           const void *data)
 397 {
 398   struct TestGetContext *test_get = cls;
 399   GNUNET_HashCode search_key; /* Key stored under */
 400   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 401   unsigned int i;
 402   memset(original_data, test_get->uid, sizeof(original_data));
 403   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 404
 405   if (test_get->succeeded == GNUNET_YES)
 406     return; /* Get has already been successful, probably ending now */
 407
 408 #if PATH_TRACKING
 409   if (put_path != NULL)
 410     {
 411       fprintf(stderr, "PUT Path: ");
 412       for (i = 0; put_path[i] != NULL; i++)
 413         fprintf(stderr, "%s%s", i == 0 ? "" : "->", GNUNET_i2s(put_path[i]));
 414       fprintf(stderr, "\n");
 415     }
 416   if (get_path != NULL)
 417     {
 418       fprintf(stderr, "GET Path: ");
 419       for (i = 0; get_path[i] != NULL; i++)
 420         fprintf(stderr, "%s%s", i == 0 ? "" : "->", GNUNET_i2s(get_path[i]));
 421       fprintf(stderr, "\n");
 422     }
 423 #endif
 424
 425   if ((0 != memcmp(&search_key, key, sizeof (GNUNET_HashCode))) || (0 != memcmp(original_data, data, sizeof(original_data))))
 426     {
 427       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Key or data is not the same as was inserted!\n");
 428     }
 429   else
 430     {
 431       gets_completed++;
 432       test_get->succeeded = GNUNET_YES;
 433     }
 434
 435   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
 436   GNUNET_SCHEDULER_cancel(test_get->disconnect_task);
 437   GNUNET_SCHEDULER_add_continuation(&get_stop_task, test_get, GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 438 }
 439
 440
 441 /**
 442  * Set up some data, and call API PUT function
 443  */
 444 static void
 445 do_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 446 {
 447   struct TestGetContext *test_get = cls;
 448   GNUNET_HashCode key; /* Made up key to store data under */
 449   char data[TEST_DATA_SIZE]; /* Made up data to store */
 450
 451   if (test_get == NULL)
 452     return; /* End of the list */
 453   memset(data, test_get->uid, sizeof(data));
 454   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 455
 456   if (outstanding_gets > MAX_OUTSTANDING_GETS)
 457     {
 458       GNUNET_SCHEDULER_add_delayed (GET_DELAY, &do_get, test_get);
 459       return;
 460     }
 461
 462   test_get->dht_handle = GNUNET_DHT_connect(test_get->daemon->cfg, 10);
 463   /* Insert the data at the first peer */
 464   GNUNET_assert(test_get->dht_handle != NULL);
 465   outstanding_gets++;
 466   test_get->get_handle = GNUNET_DHT_get_start(test_get->dht_handle,
 467                                               GNUNET_TIME_UNIT_FOREVER_REL,
 468                                               GNUNET_BLOCK_TYPE_TEST,
 469                                               &key,
 470                                               DEFAULT_GET_REPLICATION,
 471                                               route_option,
 472                                               NULL, 0,
 473                                               NULL, 0,
 474                                               &get_result_iterator,
 475                                               test_get);
 476 #if VERBOSE
 477   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
 478              test_get->uid,
 479              test_get->daemon->shortname);
 480 #endif
 481   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed(GET_TIMEOUT, &get_stop_task, test_get);
 482   GNUNET_SCHEDULER_add_now (&do_get, test_get->next);
 483 }
 484
 485 /**
 486  * Called when the PUT request has been transmitted to the DHT service.
 487  * Schedule the GET request for some time in the future.
 488  */
 489 static void
 490 put_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 491 {
 492   struct TestPutContext *test_put = cls;
 493   outstanding_puts--;
 494   puts_completed++;
 495
 496   GNUNET_SCHEDULER_cancel(test_put->disconnect_task);
 497   test_put->disconnect_task = GNUNET_SCHEDULER_add_now(&put_disconnect_task, test_put);
 498   if (puts_completed == num_puts)
 499     {
 500       GNUNET_assert(outstanding_puts == 0);
 501       GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10), &do_get, all_gets);
 502       return;
 503     }
 504 }
 505
 506 /**
 507  * Set up some data, and call API PUT function
 508  */
 509 static void
 510 do_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 511 {
 512   struct TestPutContext *test_put = cls;
 513   GNUNET_HashCode key; /* Made up key to store data under */
 514   char data[TEST_DATA_SIZE]; /* Made up data to store */
 515
 516   if (test_put == NULL)
 517     return; /* End of list */
 518
 519   memset(data, test_put->uid, sizeof(data));
 520   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 521
 522   if (outstanding_puts > MAX_OUTSTANDING_PUTS)
 523     {
 524       GNUNET_SCHEDULER_add_delayed (PUT_DELAY, &do_put, test_put);
 525       return;
 526     }
 527
 528 #if VERBOSE
 529     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
 530                test_put->uid,
 531                test_put->daemon->shortname);
 532 #endif
 533   test_put->dht_handle = GNUNET_DHT_connect(test_put->daemon->cfg, 10);
 534
 535   GNUNET_assert(test_put->dht_handle != NULL);
 536   outstanding_puts++;
 537   GNUNET_DHT_put(test_put->dht_handle,
 538                  &key,
 539                  DEFAULT_PUT_REPLICATION,
 540                  route_option,
 541                  GNUNET_BLOCK_TYPE_TEST,
 542                  sizeof(data), data,
 543                  GNUNET_TIME_UNIT_FOREVER_ABS,
 544                  GNUNET_TIME_UNIT_FOREVER_REL,
 545                  &put_finished, test_put);
 546   test_put->disconnect_task = GNUNET_SCHEDULER_add_delayed(GNUNET_TIME_relative_get_forever(), &put_disconnect_task, test_put);
 547   GNUNET_SCHEDULER_add_now(&do_put, test_put->next);
 548 }
 549
 550
 551 /**
 552  * Set up some all of the put and get operations we want
 553  * to do.  Allocate data structure for each, add to list,
 554  * then call actual insert functions.
 555  */
 556 static void
 557 setup_puts_and_gets (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 558 {
 559   int i;
 560   uint32_t temp_daemon;
 561   struct TestPutContext *test_put;
 562   struct TestGetContext *test_get;
 563   int remember[num_puts][num_peers];
 564
 565   for (i = 0; i < num_puts; i++)
 566     {
 567       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
 568       test_put->uid = i;
 569       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 570       test_put->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 571       test_put->next = all_puts;
 572       all_puts = test_put;
 573     }
 574
 575   for (i = 0; i < num_gets; i++)
 576     {
 577       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
 578       test_get->uid = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_puts);
 579       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 580       while (remember[test_get->uid][temp_daemon] == 1)
 581         temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 582       test_get->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 583       remember[test_get->uid][temp_daemon] = 1;
 584       test_get->next = all_gets;
 585       all_gets = test_get;
 586     }
 587
 588   GNUNET_SCHEDULER_add_now (&do_put, all_puts);
 589 }
 590
 591
 592 /**
 593  * This function is called whenever a connection attempt is finished between two of
 594  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
 595  * number of times this function is called should equal the number returned
 596  * from the GNUNET_TESTING_connect_topology call.
 597  *
 598  * The emsg variable is NULL on success (peers connected), and non-NULL on
 599  * failure (peers failed to connect).
 600  */
 601 void
 602 topology_callback (void *cls,
 603                    const struct GNUNET_PeerIdentity *first,
 604                    const struct GNUNET_PeerIdentity *second,
 605                    uint32_t distance,
 606                    const struct GNUNET_CONFIGURATION_Handle *first_cfg,
 607                    const struct GNUNET_CONFIGURATION_Handle *second_cfg,
 608                    struct GNUNET_TESTING_Daemon *first_daemon,
 609                    struct GNUNET_TESTING_Daemon *second_daemon,
 610                    const char *emsg)
 611 {
 612   if (emsg == NULL)
 613     {
 614       total_connections++;
 615 #if VERBOSE > 1
 616       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
 617                  first_daemon->shortname,
 618                  second_daemon->shortname,
 619                  distance);
 620 #endif
 621     }
 622 #if VERBOSE
 623   else
 624     {
 625       failed_connections++;
 626       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
 627                   first_daemon->shortname,
 628                   second_daemon->shortname, emsg);
 629     }
 630 #endif
 631
 632   if (total_connections == expected_connections)
 633     {
 634 #if VERBOSE
 635       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 636                   "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
 637                   total_connections);
 638 #endif
 639       GNUNET_SCHEDULER_cancel (die_task);
 640       die_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
 641                                                &end_badly, "from setup puts/gets");
 642
 643       GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2), &setup_puts_and_gets, NULL);
 644     }
 645   else if (total_connections + failed_connections == expected_connections)
 646     {
 647       GNUNET_SCHEDULER_cancel (die_task);
 648       die_task = GNUNET_SCHEDULER_add_now (&end_badly, "from topology_callback (too many failed connections)");
 649     }
 650 }
 651
 652 static void
 653 peers_started_callback (void *cls,
 654        const struct GNUNET_PeerIdentity *id,
 655        const struct GNUNET_CONFIGURATION_Handle *cfg,
 656        struct GNUNET_TESTING_Daemon *d, const char *emsg)
 657 {
 658   if (emsg != NULL)
 659     {
 660       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to start daemon with error: `%s'\n",
 661                   emsg);
 662       return;
 663     }
 664   GNUNET_assert (id != NULL);
 665
 666 #if VERBOSE
 667   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
 668               (num_peers - peers_left) + 1, num_peers);
 669 #endif
 670
 671   peers_left--;
 672   if (peers_left == 0)
 673     {
 674
 675 #if VERBOSE
 676       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 677                   "All %d daemons started, now connecting peers!\n",
 678                   num_peers);
 679 #endif
 680       GNUNET_SCHEDULER_cancel (die_task);
 681
 682       expected_connections = -1;
 683       if ((pg != NULL) && (peers_left == 0))
 684         {
 685           expected_connections = GNUNET_TESTING_connect_topology (pg, connection_topology, connect_topology_option, connect_topology_option_modifier, NULL, NULL);
 686 #if VERBOSE
 687           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 688                       "Have %d expected connections\n", expected_connections);
 689 #endif
 690         }
 691
 692       if (expected_connections == GNUNET_SYSERR)
 693         {
 694           die_task = GNUNET_SCHEDULER_add_now (&end_badly, "from connect topology (bad return)");
 695         }
 696
 697       die_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
 698                                                &end_badly, "from connect topology (timeout)");
 699
 700       ok = 0;
 701     }
 702 }
 703
 704 static void
 705 create_topology ()
 706 {
 707   peers_left = num_peers; /* Reset counter */
 708   if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology, blacklist_transports) != GNUNET_SYSERR)
 709     {
 710 #if VERBOSE
 711       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 712                   "Topology set up, now starting peers!\n");
 713 #endif
 714       GNUNET_TESTING_daemons_continue_startup(pg);
 715     }
 716   else
 717     {
 718       GNUNET_SCHEDULER_cancel (die_task);
 719       die_task = GNUNET_SCHEDULER_add_now (&end_badly, "from create topology (bad return)");
 720     }
 721   GNUNET_SCHEDULER_cancel (die_task);
 722   die_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
 723                                            &end_badly, "from continue startup (timeout)");
 724 }
 725
 726 /**
 727  * Callback indicating that the hostkey was created for a peer.
 728  *
 729  * @param cls NULL
 730  * @param id the peer identity
 731  * @param d the daemon handle (pretty useless at this point, remove?)
 732  * @param emsg non-null on failure
 733  */
 734 void hostkey_callback (void *cls,
 735                        const struct GNUNET_PeerIdentity *id,
 736                        struct GNUNET_TESTING_Daemon *d,
 737                        const char *emsg)
 738 {
 739   if (emsg != NULL)
 740     {
 741       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg);
 742     }
 743
 744 #if VERBOSE > 1
 745     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 746                 "Hostkey (%d/%d) created for peer `%s'\n",
 747                 num_peers - peers_left, num_peers, GNUNET_i2s(id));
 748 #endif
 749
 750
 751     peers_left--;
 752     if (peers_left == 0)
 753       {
 754 #if VERBOSE
 755         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 756                     "All %d hostkeys created, now creating topology!\n",
 757                     num_peers);
 758 #endif
 759         GNUNET_SCHEDULER_cancel (die_task);
 760         /* Set up task in case topology creation doesn't finish
 761          * within a reasonable amount of time */
 762         die_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT,
 763                                                  &end_badly, "from create_topology");
 764         GNUNET_SCHEDULER_add_now(&create_topology, NULL);
 765         ok = 0;
 766       }
 767 }
 768
 769
 770 static void
 771 run (void *cls,
 772      char *const *args,
 773      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
 774 {
 775   char * topology_str;
 776   char * connect_topology_str;
 777   char * blacklist_topology_str;
 778   char * connect_topology_option_str;
 779   char * connect_topology_option_modifier_string;
 780
 781 #if PATH_TRACKING
 782   route_option = GNUNET_DHT_RO_RECORD_ROUTE;
 783 #else
 784   route_option = GNUNET_DHT_RO_NONE;
 785 #endif
 786
 787   /* Get path from configuration file */
 788   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string(cfg, "paths", "servicehome", &test_directory))
 789     {
 790       ok = 404;
 791       return;
 792     }
 793
 794   if ((GNUNET_YES ==
 795       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "topology",
 796                                             &topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&topology, topology_str)))
 797     {
 798       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 799                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "TOPOLOGY");
 800       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
 801     }
 802
 803   if ((GNUNET_YES ==
 804       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology",
 805                                             &connect_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&connection_topology, connect_topology_str)))
 806     {
 807       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 808                   "Invalid connect topology `%s' given for section %s option %s\n", connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
 809     }
 810   GNUNET_free_non_null(connect_topology_str);
 811   if ((GNUNET_YES ==
 812       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology_option",
 813                                             &connect_topology_option_str)) && (GNUNET_NO == GNUNET_TESTING_topology_option_get(&connect_topology_option, connect_topology_option_str)))
 814     {
 815       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 816                   "Invalid connect topology option `%s' given for section %s option %s\n", connect_topology_option_str, "TESTING", "CONNECT_TOPOLOGY_OPTION");
 817       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
 818     }
 819   GNUNET_free_non_null(connect_topology_option_str);
 820   if (GNUNET_YES ==
 821         GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "connect_topology_option_modifier",
 822                                                &connect_topology_option_modifier_string))
 823     {
 824       if (sscanf(connect_topology_option_modifier_string, "%lf", &connect_topology_option_modifier) != 1)
 825       {
 826         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 827         _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
 828         connect_topology_option_modifier_string,
 829         "connect_topology_option_modifier",
 830         "TESTING");
 831       }
 832       GNUNET_free (connect_topology_option_modifier_string);
 833     }
 834
 835   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "blacklist_transports",
 836                                          &blacklist_transports))
 837     blacklist_transports = NULL;
 838
 839   if ((GNUNET_YES ==
 840       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "blacklist_topology",
 841                                             &blacklist_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&blacklist_topology, blacklist_topology_str)))
 842     {
 843       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 844                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
 845     }
 846   GNUNET_free_non_null(topology_str);
 847   GNUNET_free_non_null(blacklist_topology_str);
 848
 849   /* Get number of peers to start from configuration */
 850   if (GNUNET_SYSERR ==
 851       GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers",
 852                                              &num_peers))
 853     num_peers = DEFAULT_NUM_PEERS;
 854
 855   if (GNUNET_SYSERR ==
 856       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_puts",
 857                                              &num_puts))
 858     num_puts = DEFAULT_NUM_PEERS;
 859
 860   if (GNUNET_SYSERR ==
 861       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_gets",
 862                                              &num_gets))
 863     num_gets = DEFAULT_NUM_PEERS;
 864
 865   /* Set peers_left so we know when all peers started */
 866   peers_left = num_peers;
 867
 868   /* Set up a task to end testing if peer start fails */
 869   die_task = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers),
 870                                            &end_badly, "didn't generate all hostkeys within a reasonable amount of time!!!");
 871
 872   pg = GNUNET_TESTING_daemons_start (cfg,
 873                                      peers_left, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), &hostkey_callback, NULL, &peers_started_callback, NULL,
 874                                      &topology_callback, NULL, NULL);
 875
 876 }
 877
 878 static int
 879 check ()
 880 {
 881   int ret;
 882   /* Arguments for GNUNET_PROGRAM_run */
 883   char *const argv[] = {"test-dht-multipeer", /* Name to give running binary */
 884     "-c",
 885     "test_dht_multipeer_data.conf", /* Config file to use */
 886 #if VERBOSE
 887     "-L", "DEBUG",
 888 #endif
 889     NULL
 890   };
 891   struct GNUNET_GETOPT_CommandLineOption options[] = {
 892     GNUNET_GETOPT_OPTION_END
 893   };
 894   /* Run the run function as a new program */
 895   ret = GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
 896                       argv, "test-dht-multipeer", "nohelp",
 897                       options, &run, &ok);
 898   if (ret != GNUNET_OK)
 899     {
 900       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "`test-dht-multipeer': Failed with error code %d\n", ret);
 901     }
 902   return ok;
 903 }
 904
 905 int
 906 main (int argc, char *argv[])
 907 {
 908   int ret;
 909
 910   GNUNET_log_setup ("test-dht-multipeer",
 911 #if VERBOSE
 912                     "DEBUG",
 913 #else
 914                     "WARNING",
 915 #endif
 916                     NULL);
 917   ret = check ();
 918   /**
 919    * Need to remove base directory, subdirectories taken care
 920    * of by the testing framework.
 921    */
 922   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
 923     {
 924       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to remove testing directory %s\n", test_directory);
 925     }
 926   return ret;
 927 }
 928
 929 /* end of test_dht_twopeer_put_get.c */