src/dht/test_dht_multipeer.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file dht/test_dht_multipeer.c
  22  * @brief testcase for testing DHT service with
  23  *        multiple peers.
  24  */
  25 #include "platform.h"
  26 #include "gnunet_testing_lib.h"
  27 #include "gnunet_core_service.h"
  28 #include "gnunet_dht_service.h"
  29
  30 /* DEFINES */
  31 #define VERBOSE GNUNET_YES
  32
  33 /* Timeout for entire testcase */
  34 #define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  35
  36 /* Timeout for waiting for replies to get requests */
  37 #define GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  38
  39 /* Timeout for waiting for gets to complete */
  40 #define GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  41
  42 /* Timeout for waiting for puts to complete */
  43 #define PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  44
  45 #define SECONDS_PER_PEER_START 45
  46
  47 /* If number of peers not in config file, use this number */
  48 #define DEFAULT_NUM_PEERS 5
  49
  50 #define TEST_DATA_SIZE 8
  51
  52 #define MAX_OUTSTANDING_PUTS 10
  53
  54 #define MAX_OUTSTANDING_GETS 10
  55
  56 /* Structs */
  57
  58 struct TestPutContext
  59 {
  60   /**
  61    * This is a linked list
  62    */
  63   struct TestPutContext *next;
  64
  65   /**
  66    * Handle to the first peers DHT service (via the API)
  67    */
  68   struct GNUNET_DHT_Handle *dht_handle;
  69
  70   /**
  71    *  Handle to the PUT peer daemon
  72    */
  73   struct GNUNET_TESTING_Daemon *daemon;
  74
  75   /**
  76    *  Identifier for this PUT
  77    */
  78   uint32_t uid;
  79
  80   /**
  81    * Task for disconnecting DHT handles
  82    */
  83   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
  84 };
  85
  86 struct TestGetContext
  87 {
  88   /* This is a linked list */
  89   struct TestGetContext *next;
  90
  91   /**
  92    * Handle to the first peers DHT service (via the API)
  93    */
  94   struct GNUNET_DHT_Handle *dht_handle;
  95
  96   /**
  97    * Handle for the DHT get request
  98    */
  99   struct GNUNET_DHT_GetHandle *get_handle;
 100
 101   /**
 102    *  Handle to the GET peer daemon
 103    */
 104   struct GNUNET_TESTING_Daemon *daemon;
 105
 106   /**
 107    *  Identifier for this GET
 108    */
 109   uint32_t uid;
 110
 111   /**
 112    * Task for disconnecting DHT handles (and stopping GET)
 113    */
 114   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 115
 116   /**
 117    * Whether or not this request has been fulfilled already.
 118    */
 119   int succeeded;
 120 };
 121
 122 /* Globals */
 123
 124 /**
 125  * List of GETS to perform
 126  */
 127 struct TestGetContext *all_gets;
 128
 129 /**
 130  * List of PUTS to perform
 131  */
 132 struct TestPutContext *all_puts;
 133
 134 /**
 135  * Directory to store temp data in, defined in config file
 136  */
 137 static char *test_directory;
 138
 139 /**
 140  * Variable used to store the number of connections we should wait for.
 141  */
 142 static unsigned int expected_connections;
 143
 144 /**
 145  * Variable used to keep track of how many peers aren't yet started.
 146  */
 147 static unsigned long long peers_left;
 148
 149 /**
 150  * Handle to the set of all peers run for this test.
 151  */
 152 static struct GNUNET_TESTING_PeerGroup *pg;
 153
 154
 155 /**
 156  * Global scheduler, used for all GNUNET_SCHEDULER_* functions.
 157  */
 158 static struct GNUNET_SCHEDULER_Handle *sched;
 159
 160 /**
 161  * Total number of peers to run, set based on config file.
 162  */
 163 static unsigned long long num_peers;
 164
 165 /**
 166  * Total number of items to insert.
 167  */
 168 static unsigned long long num_puts;
 169
 170 /**
 171  * Total number of items to attempt to get.
 172  */
 173 static unsigned long long num_gets;
 174
 175 /**
 176  * How many puts do we currently have in flight?
 177  */
 178 static unsigned long long outstanding_puts;
 179
 180 /**
 181  * How many puts are done?
 182  */
 183 static unsigned long long puts_completed;
 184
 185 /**
 186  * How many puts do we currently have in flight?
 187  */
 188 static unsigned long long outstanding_gets;
 189
 190 /**
 191  * How many gets are done?
 192  */
 193 static unsigned long long gets_completed;
 194
 195 /**
 196  * How many gets failed?
 197  */
 198 static unsigned long long gets_failed;
 199
 200 /**
 201  * Global used to count how many connections we have currently
 202  * been notified about (how many times has topology_callback been called
 203  * with success?)
 204  */
 205 static unsigned int total_connections;
 206
 207 /**
 208  * Global used to count how many failed connections we have
 209  * been notified about (how many times has topology_callback
 210  * been called with failure?)
 211  */
 212 static unsigned int failed_connections;
 213
 214 /* Task handle to use to schedule test failure */
 215 GNUNET_SCHEDULER_TaskIdentifier die_task;
 216
 217 static char *blacklist_transports;
 218
 219 static enum GNUNET_TESTING_Topology topology;
 220
 221 static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 222
 223 static enum GNUNET_TESTING_Topology connection_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 224
 225 static enum GNUNET_TESTING_TopologyOption connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 226
 227 static double connect_topology_option_modifier = 0.0;
 228
 229 /* Global return value (0 for success, anything else for failure) */
 230 static int ok;
 231
 232
 233 /**
 234  * Check whether peers successfully shut down.
 235  */
 236 void shutdown_callback (void *cls,
 237                         const char *emsg)
 238 {
 239   if (emsg != NULL)
 240     {
 241       if (ok == 0)
 242         ok = 2;
 243     }
 244 }
 245
 246 /**
 247  * Task to release DHT handles for PUT
 248  */
 249 static void
 250 put_disconnect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 251 {
 252   struct TestPutContext *test_put = cls;
 253   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 254   GNUNET_DHT_disconnect(test_put->dht_handle);
 255   test_put->dht_handle = NULL;
 256 }
 257
 258 /**
 259  * Function scheduled to be run on the successful completion of this
 260  * testcase.
 261  */
 262 static void
 263 finish_testing (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 264 {
 265   GNUNET_assert (pg != NULL);
 266   struct TestPutContext *test_put = all_puts;
 267   struct TestGetContext *test_get = all_gets;
 268
 269   while (test_put != NULL)
 270     {
 271       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 272         GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 273       if (test_put->dht_handle != NULL)
 274         GNUNET_DHT_disconnect(test_put->dht_handle);
 275       test_put = test_put->next;
 276     }
 277
 278   while (test_get != NULL)
 279     {
 280       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 281         GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 282       if (test_get->get_handle != NULL)
 283         GNUNET_DHT_get_stop(test_get->get_handle);
 284       if (test_get->dht_handle != NULL)
 285         GNUNET_DHT_disconnect(test_get->dht_handle);
 286       test_get = test_get->next;
 287     }
 288
 289   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 290   ok = 0;
 291 }
 292
 293
 294 /**
 295  * Check if the get_handle is being used, if so stop the request.  Either
 296  * way, schedule the end_badly_cont function which actually shuts down the
 297  * test.
 298  */
 299 static void
 300 end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 301 {
 302   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failing test with error: `%s'!\n", (char *)cls);
 303
 304   struct TestPutContext *test_put = all_puts;
 305   struct TestGetContext *test_get = all_gets;
 306
 307   while (test_put != NULL)
 308     {
 309       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 310         GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 311       if (test_put->dht_handle != NULL)
 312         GNUNET_DHT_disconnect(test_put->dht_handle);
 313       test_put = test_put->next;
 314     }
 315
 316   while (test_get != NULL)
 317     {
 318       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 319         GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 320       if (test_get->get_handle != NULL)
 321         GNUNET_DHT_get_stop(test_get->get_handle);
 322       if (test_get->dht_handle != NULL)
 323         GNUNET_DHT_disconnect(test_get->dht_handle);
 324       test_get = test_get->next;
 325     }
 326
 327   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 328   ok = 1;
 329 }
 330
 331 /**
 332  * Task to release DHT handle associated with GET request.
 333  */
 334 static void
 335 get_stop_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 336 {
 337   struct TestGetContext *test_get = cls;
 338   outstanding_gets--; /* GET is really finished */
 339   GNUNET_DHT_disconnect(test_get->dht_handle);
 340   test_get->dht_handle = NULL;
 341
 342   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
 343   if ((gets_completed == num_gets) && (outstanding_gets == 0))/* All gets successful */
 344     {
 345       GNUNET_SCHEDULER_cancel (sched, die_task);
 346       //GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5), &get_topology, NULL);
 347       GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL);
 348     }
 349   else if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0)) /* Had some failures */
 350     {
 351       GNUNET_SCHEDULER_cancel(sched, die_task);
 352       GNUNET_SCHEDULER_add_now(sched, &end_badly, "not all gets succeeded!\n");
 353     }
 354 }
 355
 356 /**
 357  * Task to release get handle.
 358  */
 359 static void
 360 get_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 361 {
 362   struct TestGetContext *test_get = cls;
 363   GNUNET_HashCode search_key; /* Key stored under */
 364   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 365
 366   memset(original_data, test_get->uid, sizeof(original_data));
 367   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 368
 369   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
 370     {
 371       gets_failed++;
 372       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Get from peer %s for key %s failed!\n", test_get->daemon->shortname, GNUNET_h2s(&search_key));
 373     }
 374   GNUNET_assert(test_get->get_handle != NULL);
 375   GNUNET_DHT_get_stop(test_get->get_handle);
 376   GNUNET_SCHEDULER_add_now (sched, &get_stop_finished, test_get);
 377   test_get->get_handle = NULL;
 378   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 379 }
 380
 381 /**
 382  * Iterator called if the GET request initiated returns a response.
 383  *
 384  * @param cls closure
 385  * @param exp when will this value expire
 386  * @param key key of the result
 387  * @param type type of the result
 388  * @param size number of bytes in data
 389  * @param data pointer to the result data
 390  */
 391 void get_result_iterator (void *cls,
 392                           struct GNUNET_TIME_Absolute exp,
 393                           const GNUNET_HashCode * key,
 394                           uint32_t type,
 395                           uint32_t size,
 396                           const void *data)
 397 {
 398   struct TestGetContext *test_get = cls;
 399   GNUNET_HashCode search_key; /* Key stored under */
 400   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 401
 402   memset(original_data, test_get->uid, sizeof(original_data));
 403   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 404
 405   if (test_get->succeeded == GNUNET_YES)
 406     return; /* Get has already been successful, probably ending now */
 407
 408   if ((0 != memcmp(&search_key, key, sizeof (GNUNET_HashCode))) || (0 != memcmp(original_data, data, sizeof(original_data))))
 409     {
 410       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Key or data is not the same as was inserted!\n");
 411     }
 412   else
 413     {
 414       gets_completed++;
 415       test_get->succeeded = GNUNET_YES;
 416     }
 417
 418   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
 419   GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 420   GNUNET_SCHEDULER_add_continuation(sched, &get_stop_task, test_get, GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 421 }
 422
 423 /**
 424  * Continuation telling us GET request was sent.
 425  */
 426 static void
 427 get_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 428 {
 429   // Is there something to be done here?
 430   if (tc->reason != GNUNET_SCHEDULER_REASON_PREREQ_DONE)
 431     return;
 432 }
 433
 434 /**
 435  * Set up some data, and call API PUT function
 436  */
 437 static void
 438 do_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 439 {
 440   struct TestGetContext *test_get = cls;
 441   GNUNET_HashCode key; /* Made up key to store data under */
 442   char data[TEST_DATA_SIZE]; /* Made up data to store */
 443
 444   if (test_get == NULL)
 445     return; /* End of the list */
 446   memset(data, test_get->uid, sizeof(data));
 447   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 448
 449   if (outstanding_gets > MAX_OUTSTANDING_GETS)
 450     {
 451       GNUNET_SCHEDULER_add_delayed (sched, GET_DELAY, &do_get, test_get);
 452       return;
 453     }
 454
 455   test_get->dht_handle = GNUNET_DHT_connect(sched, test_get->daemon->cfg, 10);
 456   /* Insert the data at the first peer */
 457   GNUNET_assert(test_get->dht_handle != NULL);
 458   outstanding_gets++;
 459   test_get->get_handle = GNUNET_DHT_get_start(test_get->dht_handle,
 460                                               GNUNET_TIME_relative_get_forever(),
 461                                               1,
 462                                               &key,
 463                                               &get_result_iterator,
 464                                               test_get,
 465                                               &get_continuation,
 466                                               test_get);
 467 #if VERBOSE
 468   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
 469              test_get->uid,
 470              test_get->daemon->shortname);
 471 #endif
 472   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GET_TIMEOUT, &get_stop_task, test_get);
 473   GNUNET_SCHEDULER_add_now (sched, &do_get, test_get->next);
 474 }
 475
 476 /**
 477  * Called when the PUT request has been transmitted to the DHT service.
 478  * Schedule the GET request for some time in the future.
 479  */
 480 static void
 481 put_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 482 {
 483   struct TestPutContext *test_put = cls;
 484   outstanding_puts--;
 485   puts_completed++;
 486
 487   GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 488   test_put->disconnect_task = GNUNET_SCHEDULER_add_now(sched, &put_disconnect_task, test_put);
 489   if (puts_completed == num_puts)
 490     {
 491       GNUNET_assert(outstanding_puts == 0);
 492       GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10), &do_get, all_gets);
 493       return;
 494     }
 495 }
 496
 497 /**
 498  * Set up some data, and call API PUT function
 499  */
 500 static void
 501 do_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 502 {
 503   struct TestPutContext *test_put = cls;
 504   GNUNET_HashCode key; /* Made up key to store data under */
 505   char data[TEST_DATA_SIZE]; /* Made up data to store */
 506
 507   if (test_put == NULL)
 508     return; /* End of list */
 509
 510   memset(data, test_put->uid, sizeof(data));
 511   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 512
 513   if (outstanding_puts > MAX_OUTSTANDING_PUTS)
 514     {
 515       GNUNET_SCHEDULER_add_delayed (sched, PUT_DELAY, &do_put, test_put);
 516       return;
 517     }
 518
 519 #if VERBOSE
 520     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
 521                test_put->uid,
 522                test_put->daemon->shortname);
 523 #endif
 524   test_put->dht_handle = GNUNET_DHT_connect(sched, test_put->daemon->cfg, 10);
 525
 526   GNUNET_assert(test_put->dht_handle != NULL);
 527   outstanding_puts++;
 528   GNUNET_DHT_put(test_put->dht_handle,
 529                  &key,
 530                  1,
 531                  sizeof(data), data,
 532                  GNUNET_TIME_absolute_get_forever(),
 533                  GNUNET_TIME_relative_get_forever(),
 534                  &put_finished, test_put);
 535   test_put->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_get_forever(), &put_disconnect_task, test_put);
 536   GNUNET_SCHEDULER_add_now(sched, &do_put, test_put->next);
 537 }
 538
 539
 540 /**
 541  * Set up some all of the put and get operations we want
 542  * to do.  Allocate data structure for each, add to list,
 543  * then call actual insert functions.
 544  */
 545 static void
 546 setup_puts_and_gets (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 547 {
 548   int i;
 549   uint32_t temp_daemon;
 550   struct TestPutContext *test_put;
 551   struct TestGetContext *test_get;
 552   int remember[num_puts][num_peers];
 553
 554   for (i = 0; i < num_puts; i++)
 555     {
 556       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
 557       test_put->uid = i;
 558       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 559       test_put->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 560       test_put->next = all_puts;
 561       all_puts = test_put;
 562     }
 563
 564   for (i = 0; i < num_gets; i++)
 565     {
 566       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
 567       test_get->uid = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_puts);
 568       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 569       while (remember[test_get->uid][temp_daemon] == 1)
 570         temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 571       test_get->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 572       remember[test_get->uid][temp_daemon] = 1;
 573       test_get->next = all_gets;
 574       all_gets = test_get;
 575     }
 576
 577   GNUNET_SCHEDULER_add_now (sched, &do_put, all_puts);
 578 }
 579
 580
 581 /**
 582  * This function is called whenever a connection attempt is finished between two of
 583  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
 584  * number of times this function is called should equal the number returned
 585  * from the GNUNET_TESTING_connect_topology call.
 586  *
 587  * The emsg variable is NULL on success (peers connected), and non-NULL on
 588  * failure (peers failed to connect).
 589  */
 590 void
 591 topology_callback (void *cls,
 592                    const struct GNUNET_PeerIdentity *first,
 593                    const struct GNUNET_PeerIdentity *second,
 594                    uint32_t distance,
 595                    const struct GNUNET_CONFIGURATION_Handle *first_cfg,
 596                    const struct GNUNET_CONFIGURATION_Handle *second_cfg,
 597                    struct GNUNET_TESTING_Daemon *first_daemon,
 598                    struct GNUNET_TESTING_Daemon *second_daemon,
 599                    const char *emsg)
 600 {
 601   if (emsg == NULL)
 602     {
 603       total_connections++;
 604 #if VERBOSE > 1
 605       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
 606                  first_daemon->shortname,
 607                  second_daemon->shortname,
 608                  distance);
 609 #endif
 610     }
 611 #if VERBOSE
 612   else
 613     {
 614       failed_connections++;
 615       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
 616                   first_daemon->shortname,
 617                   second_daemon->shortname, emsg);
 618     }
 619 #endif
 620
 621   if (total_connections == expected_connections)
 622     {
 623 #if VERBOSE
 624       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 625                   "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
 626                   total_connections);
 627 #endif
 628       GNUNET_SCHEDULER_cancel (sched, die_task);
 629       die_task = GNUNET_SCHEDULER_add_delayed (sched, TIMEOUT,
 630                                                &end_badly, "from setup puts/gets");
 631
 632       GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2), &setup_puts_and_gets, NULL);
 633     }
 634   else if (total_connections + failed_connections == expected_connections)
 635     {
 636       GNUNET_SCHEDULER_cancel (sched, die_task);
 637       die_task = GNUNET_SCHEDULER_add_now (sched,
 638                                            &end_badly, "from topology_callback (too many failed connections)");
 639     }
 640 }
 641
 642 static void
 643 peers_started_callback (void *cls,
 644        const struct GNUNET_PeerIdentity *id,
 645        const struct GNUNET_CONFIGURATION_Handle *cfg,
 646        struct GNUNET_TESTING_Daemon *d, const char *emsg)
 647 {
 648   if (emsg != NULL)
 649     {
 650       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to start daemon with error: `%s'\n",
 651                   emsg);
 652       return;
 653     }
 654   GNUNET_assert (id != NULL);
 655
 656 #if VERBOSE
 657   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
 658               (num_peers - peers_left) + 1, num_peers);
 659 #endif
 660
 661   peers_left--;
 662   if (peers_left == 0)
 663     {
 664
 665 #if VERBOSE
 666       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 667                   "All %d daemons started, now connecting peers!\n",
 668                   num_peers);
 669 #endif
 670       GNUNET_SCHEDULER_cancel (sched, die_task);
 671
 672       expected_connections = -1;
 673       if ((pg != NULL) && (peers_left == 0))
 674         {
 675           expected_connections = GNUNET_TESTING_connect_topology (pg, connection_topology, connect_topology_option, connect_topology_option_modifier);
 676 #if VERBOSE
 677           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 678                       "Have %d expected connections\n", expected_connections);
 679 #endif
 680         }
 681
 682       if (expected_connections == GNUNET_SYSERR)
 683         {
 684           die_task = GNUNET_SCHEDULER_add_now (sched,
 685                                                &end_badly, "from connect topology (bad return)");
 686         }
 687
 688       die_task = GNUNET_SCHEDULER_add_delayed (sched,
 689                                                TIMEOUT,
 690                                                &end_badly, "from connect topology (timeout)");
 691
 692       ok = 0;
 693     }
 694 }
 695
 696 static void
 697 create_topology ()
 698 {
 699   peers_left = num_peers; /* Reset counter */
 700   if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology, blacklist_transports) != GNUNET_SYSERR)
 701     {
 702 #if VERBOSE
 703       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 704                   "Topology set up, now starting peers!\n");
 705 #endif
 706       GNUNET_TESTING_daemons_continue_startup(pg);
 707     }
 708   else
 709     {
 710       GNUNET_SCHEDULER_cancel (sched, die_task);
 711       die_task = GNUNET_SCHEDULER_add_now (sched,
 712                                            &end_badly, "from create topology (bad return)");
 713     }
 714   GNUNET_SCHEDULER_cancel (sched, die_task);
 715   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 716                                            TIMEOUT,
 717                                            &end_badly, "from continue startup (timeout)");
 718 }
 719
 720 /**
 721  * Callback indicating that the hostkey was created for a peer.
 722  *
 723  * @param cls NULL
 724  * @param id the peer identity
 725  * @param d the daemon handle (pretty useless at this point, remove?)
 726  * @param emsg non-null on failure
 727  */
 728 void hostkey_callback (void *cls,
 729                        const struct GNUNET_PeerIdentity *id,
 730                        struct GNUNET_TESTING_Daemon *d,
 731                        const char *emsg)
 732 {
 733   if (emsg != NULL)
 734     {
 735       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg);
 736     }
 737
 738 #if VERBOSE > 1
 739     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 740                 "Hostkey (%d/%d) created for peer `%s'\n",
 741                 num_peers - peers_left, num_peers, GNUNET_i2s(id));
 742 #endif
 743
 744
 745     peers_left--;
 746     if (peers_left == 0)
 747       {
 748 #if VERBOSE
 749         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 750                     "All %d hostkeys created, now creating topology!\n",
 751                     num_peers);
 752 #endif
 753         GNUNET_SCHEDULER_cancel (sched, die_task);
 754         /* Set up task in case topology creation doesn't finish
 755          * within a reasonable amount of time */
 756         die_task = GNUNET_SCHEDULER_add_delayed (sched,
 757                                                  TIMEOUT,
 758                                                  &end_badly, "from create_topology");
 759         GNUNET_SCHEDULER_add_now(sched, &create_topology, NULL);
 760         ok = 0;
 761       }
 762 }
 763
 764
 765 static void
 766 run (void *cls,
 767      struct GNUNET_SCHEDULER_Handle *s,
 768      char *const *args,
 769      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
 770 {
 771   char * topology_str;
 772   char * connect_topology_str;
 773   char * blacklist_topology_str;
 774   char * connect_topology_option_str;
 775   char * connect_topology_option_modifier_string;
 776   sched = s;
 777
 778   /* Get path from configuration file */
 779   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string(cfg, "paths", "servicehome", &test_directory))
 780     {
 781       ok = 404;
 782       return;
 783     }
 784
 785   if ((GNUNET_YES ==
 786       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "topology",
 787                                             &topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&topology, topology_str)))
 788     {
 789       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 790                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "TOPOLOGY");
 791       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
 792     }
 793
 794   if ((GNUNET_YES ==
 795       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology",
 796                                             &connect_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&connection_topology, connect_topology_str)))
 797     {
 798       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 799                   "Invalid connect topology `%s' given for section %s option %s\n", connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
 800     }
 801   GNUNET_free_non_null(connect_topology_str);
 802   if ((GNUNET_YES ==
 803       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology_option",
 804                                             &connect_topology_option_str)) && (GNUNET_NO == GNUNET_TESTING_topology_option_get(&connect_topology_option, connect_topology_option_str)))
 805     {
 806       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 807                   "Invalid connect topology option `%s' given for section %s option %s\n", connect_topology_option_str, "TESTING", "CONNECT_TOPOLOGY_OPTION");
 808       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
 809     }
 810   GNUNET_free_non_null(connect_topology_option_str);
 811   if (GNUNET_YES ==
 812         GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "connect_topology_option_modifier",
 813                                                &connect_topology_option_modifier_string))
 814     {
 815       if (sscanf(connect_topology_option_modifier_string, "%lf", &connect_topology_option_modifier) != 1)
 816       {
 817         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 818         _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
 819         connect_topology_option_modifier_string,
 820         "connect_topology_option_modifier",
 821         "TESTING");
 822       }
 823       GNUNET_free (connect_topology_option_modifier_string);
 824     }
 825
 826   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "blacklist_transports",
 827                                          &blacklist_transports))
 828     blacklist_transports = NULL;
 829
 830   if ((GNUNET_YES ==
 831       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "blacklist_topology",
 832                                             &blacklist_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&blacklist_topology, blacklist_topology_str)))
 833     {
 834       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 835                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
 836     }
 837   GNUNET_free_non_null(topology_str);
 838   GNUNET_free_non_null(blacklist_topology_str);
 839
 840   /* Get number of peers to start from configuration */
 841   if (GNUNET_SYSERR ==
 842       GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers",
 843                                              &num_peers))
 844     num_peers = DEFAULT_NUM_PEERS;
 845
 846   if (GNUNET_SYSERR ==
 847       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_puts",
 848                                              &num_puts))
 849     num_puts = DEFAULT_NUM_PEERS;
 850
 851   if (GNUNET_SYSERR ==
 852       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_gets",
 853                                              &num_gets))
 854     num_gets = DEFAULT_NUM_PEERS;
 855
 856   /* Set peers_left so we know when all peers started */
 857   peers_left = num_peers;
 858
 859   /* Set up a task to end testing if peer start fails */
 860   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 861                                            GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers),
 862                                            &end_badly, "didn't generate all hostkeys within a reasonable amount of time!!!");
 863
 864   pg = GNUNET_TESTING_daemons_start (sched, cfg,
 865                                      peers_left, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), &hostkey_callback, NULL, &peers_started_callback, NULL,
 866                                      &topology_callback, NULL, NULL);
 867
 868 }
 869
 870 static int
 871 check ()
 872 {
 873   int ret;
 874   /* Arguments for GNUNET_PROGRAM_run */
 875   char *const argv[] = {"test-dht-multipeer", /* Name to give running binary */
 876     "-c",
 877     "test_dht_multipeer_data.conf", /* Config file to use */
 878 #if VERBOSE
 879     "-L", "DEBUG",
 880 #endif
 881     NULL
 882   };
 883   struct GNUNET_GETOPT_CommandLineOption options[] = {
 884     GNUNET_GETOPT_OPTION_END
 885   };
 886   /* Run the run function as a new program */
 887   ret = GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
 888                       argv, "test-dht-multipeer", "nohelp",
 889                       options, &run, &ok);
 890   if (ret != GNUNET_OK)
 891     {
 892       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "`test-dht-multipeer': Failed with error code %d\n", ret);
 893     }
 894   return ok;
 895 }
 896
 897 int
 898 main (int argc, char *argv[])
 899 {
 900   int ret;
 901
 902   GNUNET_log_setup ("test-dht-multipeer",
 903 #if VERBOSE
 904                     "DEBUG",
 905 #else
 906                     "WARNING",
 907 #endif
 908                     NULL);
 909   ret = check ();
 910   /**
 911    * Need to remove base directory, subdirectories taken care
 912    * of by the testing framework.
 913    */
 914   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
 915     {
 916       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to remove testing directory %s\n", test_directory);
 917     }
 918   return ret;
 919 }
 920
 921 /* end of test_dht_twopeer_put_get.c */