src/dht/test_dht_multipeer.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2009 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20 /**
  21  * @file dht/test_dht_multipeer.c
  22  * @brief testcase for testing DHT service with
  23  *        multiple peers.
  24  */
  25 #include "platform.h"
  26 #include "gnunet_testing_lib.h"
  27 #include "gnunet_core_service.h"
  28 #include "gnunet_dht_service.h"
  29
  30 /* DEFINES */
  31 #define VERBOSE GNUNET_YES
  32
  33 /* Timeout for entire testcase */
  34 #define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5)
  35
  36 /* Timeout for waiting for replies to get requests */
  37 #define GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90)
  38
  39 /* Timeout for waiting for gets to complete */
  40 #define GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  41
  42 /* Timeout for waiting for puts to complete */
  43 #define PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1)
  44
  45 #define SECONDS_PER_PEER_START 45
  46
  47 /* If number of peers not in config file, use this number */
  48 #define DEFAULT_NUM_PEERS 5
  49
  50 #define TEST_DATA_SIZE 8
  51
  52 #define MAX_OUTSTANDING_PUTS 10
  53
  54 #define MAX_OUTSTANDING_GETS 10
  55
  56 /* Structs */
  57
  58 struct TestPutContext
  59 {
  60   /* This is a linked list */
  61   struct TestPutContext *next;
  62
  63   /**
  64    * Handle to the first peers DHT service (via the API)
  65    */
  66   struct GNUNET_DHT_Handle *dht_handle;
  67
  68   /**
  69    *  Handle to the PUT peer daemon
  70    */
  71   struct GNUNET_TESTING_Daemon *daemon;
  72
  73   /**
  74    *  Identifier for this PUT
  75    */
  76   uint32_t uid;
  77
  78   /**
  79    * Task for disconnecting DHT handles
  80    */
  81   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
  82 };
  83
  84 struct TestGetContext
  85 {
  86   /* This is a linked list */
  87   struct TestGetContext *next;
  88
  89   /**
  90    * Handle to the first peers DHT service (via the API)
  91    */
  92   struct GNUNET_DHT_Handle *dht_handle;
  93
  94   /**
  95    * Handle for the DHT get request
  96    */
  97   struct GNUNET_DHT_GetHandle *get_handle;
  98
  99   /**
 100    *  Handle to the GET peer daemon
 101    */
 102   struct GNUNET_TESTING_Daemon *daemon;
 103
 104   /**
 105    *  Identifier for this GET
 106    */
 107   uint32_t uid;
 108
 109   /**
 110    * Task for disconnecting DHT handles (and stopping GET)
 111    */
 112   GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
 113
 114   /**
 115    * Whether or not this request has been fulfilled already.
 116    */
 117   int succeeded;
 118 };
 119
 120 /* Globals */
 121
 122 /**
 123  * List of GETS to perform
 124  */
 125 struct TestGetContext *all_gets;
 126
 127 /**
 128  * List of PUTS to perform
 129  */
 130 struct TestPutContext *all_puts;
 131
 132 /**
 133  * Directory to store temp data in, defined in config file
 134  */
 135 static char *test_directory;
 136
 137 /**
 138  * Variable used to store the number of connections we should wait for.
 139  */
 140 static unsigned int expected_connections;
 141
 142 /**
 143  * Variable used to keep track of how many peers aren't yet started.
 144  */
 145 static unsigned long long peers_left;
 146
 147 /**
 148  * Handle to the set of all peers run for this test.
 149  */
 150 static struct GNUNET_TESTING_PeerGroup *pg;
 151
 152
 153 /**
 154  * Global scheduler, used for all GNUNET_SCHEDULER_* functions.
 155  */
 156 static struct GNUNET_SCHEDULER_Handle *sched;
 157
 158 /**
 159  * Total number of peers to run, set based on config file.
 160  */
 161 static unsigned long long num_peers;
 162
 163 /**
 164  * Total number of items to insert.
 165  */
 166 static unsigned long long num_puts;
 167
 168 /**
 169  * Total number of items to attempt to get.
 170  */
 171 static unsigned long long num_gets;
 172
 173 /**
 174  * How many puts do we currently have in flight?
 175  */
 176 static unsigned long long outstanding_puts;
 177
 178 /**
 179  * How many puts are done?
 180  */
 181 static unsigned long long puts_completed;
 182
 183 /**
 184  * How many puts do we currently have in flight?
 185  */
 186 static unsigned long long outstanding_gets;
 187
 188 /**
 189  * How many gets are done?
 190  */
 191 static unsigned long long gets_completed;
 192
 193 /**
 194  * How many gets failed?
 195  */
 196 static unsigned long long gets_failed;
 197
 198 /**
 199  * Global used to count how many connections we have currently
 200  * been notified about (how many times has topology_callback been called
 201  * with success?)
 202  */
 203 static unsigned int total_connections;
 204
 205 /**
 206  * Global used to count how many failed connections we have
 207  * been notified about (how many times has topology_callback
 208  * been called with failure?)
 209  */
 210 static unsigned int failed_connections;
 211
 212 /* Task handle to use to schedule test failure */
 213 GNUNET_SCHEDULER_TaskIdentifier die_task;
 214
 215 static char *blacklist_transports;
 216
 217 static enum GNUNET_TESTING_Topology topology;
 218
 219 static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */
 220
 221 static enum GNUNET_TESTING_Topology connection_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */
 222
 223 static enum GNUNET_TESTING_TopologyOption connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL;
 224
 225 static double connect_topology_option_modifier = 0.0;
 226
 227 /* Global return value (0 for success, anything else for failure) */
 228 static int ok;
 229
 230
 231 /**
 232  * Check whether peers successfully shut down.
 233  */
 234 void shutdown_callback (void *cls,
 235                         const char *emsg)
 236 {
 237   if (emsg != NULL)
 238     {
 239       if (ok == 0)
 240         ok = 2;
 241     }
 242 }
 243
 244 /**
 245  * Task to release DHT handles for PUT
 246  */
 247 static void
 248 put_disconnect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 249 {
 250   struct TestPutContext *test_put = cls;
 251   test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 252   GNUNET_DHT_disconnect(test_put->dht_handle);
 253   test_put->dht_handle = NULL;
 254 }
 255
 256 /**
 257  * Function scheduled to be run on the successful completion of this
 258  * testcase.
 259  */
 260 static void
 261 finish_testing (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 262 {
 263   GNUNET_assert (pg != NULL);
 264   struct TestPutContext *test_put = all_puts;
 265   struct TestGetContext *test_get = all_gets;
 266
 267   while (test_put != NULL)
 268     {
 269       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 270         GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 271       if (test_put->dht_handle != NULL)
 272         GNUNET_DHT_disconnect(test_put->dht_handle);
 273       test_put = test_put->next;
 274     }
 275
 276   while (test_get != NULL)
 277     {
 278       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 279         GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 280       if (test_get->get_handle != NULL)
 281         GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL);
 282       if (test_get->dht_handle != NULL)
 283         GNUNET_DHT_disconnect(test_get->dht_handle);
 284       test_get = test_get->next;
 285     }
 286
 287   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 288   ok = 0;
 289 }
 290
 291
 292 /**
 293  * Check if the get_handle is being used, if so stop the request.  Either
 294  * way, schedule the end_badly_cont function which actually shuts down the
 295  * test.
 296  */
 297 static void
 298 end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 299 {
 300   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failing test with error: `%s'!\n", (char *)cls);
 301
 302   struct TestPutContext *test_put = all_puts;
 303   struct TestGetContext *test_get = all_gets;
 304
 305   while (test_put != NULL)
 306     {
 307       if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 308         GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 309       if (test_put->dht_handle != NULL)
 310         GNUNET_DHT_disconnect(test_put->dht_handle);
 311       test_put = test_put->next;
 312     }
 313
 314   while (test_get != NULL)
 315     {
 316       if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK)
 317         GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 318       if (test_get->get_handle != NULL)
 319         GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL);
 320       if (test_get->dht_handle != NULL)
 321         GNUNET_DHT_disconnect(test_get->dht_handle);
 322       test_get = test_get->next;
 323     }
 324
 325   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
 326   ok = 1;
 327 }
 328
 329 /**
 330  * Task to release DHT handle associated with GET request.
 331  */
 332 static void
 333 get_stop_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 334 {
 335   struct TestGetContext *test_get = cls;
 336   outstanding_gets--; /* GET is really finished */
 337   GNUNET_DHT_disconnect(test_get->dht_handle);
 338   test_get->dht_handle = NULL;
 339
 340   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed);
 341   if ((gets_completed == num_gets) && (outstanding_gets == 0))/* All gets successful */
 342     {
 343       GNUNET_SCHEDULER_cancel (sched, die_task);
 344       //GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5), &get_topology, NULL);
 345       GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL);
 346     }
 347   else if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0)) /* Had some failures */
 348     {
 349       GNUNET_SCHEDULER_cancel(sched, die_task);
 350       GNUNET_SCHEDULER_add_now(sched, &end_badly, "not all gets succeeded!\n");
 351     }
 352 }
 353
 354 /**
 355  * Task to release get handle.
 356  */
 357 static void
 358 get_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 359 {
 360   struct TestGetContext *test_get = cls;
 361   GNUNET_HashCode search_key; /* Key stored under */
 362   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 363
 364   memset(original_data, test_get->uid, sizeof(original_data));
 365   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 366
 367   if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT)
 368     {
 369       gets_failed++;
 370       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Get from peer %s for key %s failed!\n", test_get->daemon->shortname, GNUNET_h2s(&search_key));
 371     }
 372   GNUNET_assert(test_get->get_handle != NULL);
 373   GNUNET_DHT_get_stop(test_get->get_handle, &get_stop_finished, test_get);
 374   test_get->get_handle = NULL;
 375   test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK;
 376 }
 377
 378 /**
 379  * Iterator called if the GET request initiated returns a response.
 380  *
 381  * @param cls closure
 382  * @param exp when will this value expire
 383  * @param key key of the result
 384  * @param type type of the result
 385  * @param size number of bytes in data
 386  * @param data pointer to the result data
 387  */
 388 void get_result_iterator (void *cls,
 389                           struct GNUNET_TIME_Absolute exp,
 390                           const GNUNET_HashCode * key,
 391                           uint32_t type,
 392                           uint32_t size,
 393                           const void *data)
 394 {
 395   struct TestGetContext *test_get = cls;
 396   GNUNET_HashCode search_key; /* Key stored under */
 397   char original_data[TEST_DATA_SIZE]; /* Made up data to store */
 398
 399   memset(original_data, test_get->uid, sizeof(original_data));
 400   GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key);
 401
 402   if (test_get->succeeded == GNUNET_YES)
 403     return; /* Get has already been successful, probably ending now */
 404
 405   if ((0 != memcmp(&search_key, key, sizeof (GNUNET_HashCode))) || (0 != memcmp(original_data, data, sizeof(original_data))))
 406     {
 407       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Key or data is not the same as was inserted!\n");
 408     }
 409   else
 410     {
 411       gets_completed++;
 412       test_get->succeeded = GNUNET_YES;
 413     }
 414
 415   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n");
 416   GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task);
 417   GNUNET_SCHEDULER_add_continuation(sched, &get_stop_task, test_get, GNUNET_SCHEDULER_REASON_PREREQ_DONE);
 418 }
 419
 420 /**
 421  * Continuation telling us GET request was sent.
 422  */
 423 static void
 424 get_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 425 {
 426   // Is there something to be done here?
 427   if (tc->reason != GNUNET_SCHEDULER_REASON_PREREQ_DONE)
 428     return;
 429 }
 430
 431 /**
 432  * Set up some data, and call API PUT function
 433  */
 434 static void
 435 do_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 436 {
 437   struct TestGetContext *test_get = cls;
 438   GNUNET_HashCode key; /* Made up key to store data under */
 439   char data[TEST_DATA_SIZE]; /* Made up data to store */
 440
 441   if (test_get == NULL)
 442     return; /* End of the list */
 443   memset(data, test_get->uid, sizeof(data));
 444   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 445
 446   if (outstanding_gets > MAX_OUTSTANDING_GETS)
 447     {
 448       GNUNET_SCHEDULER_add_delayed (sched, GET_DELAY, &do_get, test_get);
 449       return;
 450     }
 451
 452   test_get->dht_handle = GNUNET_DHT_connect(sched, test_get->daemon->cfg, 10);
 453   /* Insert the data at the first peer */
 454   GNUNET_assert(test_get->dht_handle != NULL);
 455   outstanding_gets++;
 456   test_get->get_handle = GNUNET_DHT_get_start(test_get->dht_handle,
 457                                               GNUNET_TIME_relative_get_forever(),
 458                                               1,
 459                                               &key,
 460                                               &get_result_iterator,
 461                                               test_get,
 462                                               &get_continuation,
 463                                               test_get);
 464 #if VERBOSE
 465   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n",
 466              test_get->uid,
 467              test_get->daemon->shortname);
 468 #endif
 469   test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GET_TIMEOUT, &get_stop_task, test_get);
 470   GNUNET_SCHEDULER_add_now (sched, &do_get, test_get->next);
 471 }
 472
 473 /**
 474  * Called when the PUT request has been transmitted to the DHT service.
 475  * Schedule the GET request for some time in the future.
 476  */
 477 static void
 478 put_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 479 {
 480   struct TestPutContext *test_put = cls;
 481   outstanding_puts--;
 482   puts_completed++;
 483
 484   GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task);
 485   test_put->disconnect_task = GNUNET_SCHEDULER_add_now(sched, &put_disconnect_task, test_put);
 486   if (puts_completed == num_puts)
 487     {
 488       GNUNET_assert(outstanding_puts == 0);
 489       GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10), &do_get, all_gets);
 490       return;
 491     }
 492 }
 493
 494 /**
 495  * Set up some data, and call API PUT function
 496  */
 497 static void
 498 do_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 499 {
 500   struct TestPutContext *test_put = cls;
 501   GNUNET_HashCode key; /* Made up key to store data under */
 502   char data[TEST_DATA_SIZE]; /* Made up data to store */
 503
 504   if (test_put == NULL)
 505     return; /* End of list */
 506
 507   memset(data, test_put->uid, sizeof(data));
 508   GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key);
 509
 510   if (outstanding_puts > MAX_OUTSTANDING_PUTS)
 511     {
 512       GNUNET_SCHEDULER_add_delayed (sched, PUT_DELAY, &do_put, test_put);
 513       return;
 514     }
 515
 516 #if VERBOSE
 517     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n",
 518                test_put->uid,
 519                test_put->daemon->shortname);
 520 #endif
 521   test_put->dht_handle = GNUNET_DHT_connect(sched, test_put->daemon->cfg, 10);
 522
 523   GNUNET_assert(test_put->dht_handle != NULL);
 524   outstanding_puts++;
 525   GNUNET_DHT_put(test_put->dht_handle,
 526                  &key,
 527                  1,
 528                  sizeof(data), data,
 529                  GNUNET_TIME_absolute_get_forever(),
 530                  GNUNET_TIME_relative_get_forever(),
 531                  &put_finished, test_put);
 532   test_put->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_get_forever(), &put_disconnect_task, test_put);
 533   GNUNET_SCHEDULER_add_now(sched, &do_put, test_put->next);
 534 }
 535
 536
 537 /**
 538  * Set up some all of the put and get operations we want
 539  * to do.  Allocate data structure for each, add to list,
 540  * then call actual insert functions.
 541  */
 542 static void
 543 setup_puts_and_gets (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc)
 544 {
 545   int i;
 546   uint32_t temp_daemon;
 547   struct TestPutContext *test_put;
 548   struct TestGetContext *test_get;
 549   int remember[num_puts][num_peers];
 550
 551   for (i = 0; i < num_puts; i++)
 552     {
 553       test_put = GNUNET_malloc(sizeof(struct TestPutContext));
 554       test_put->uid = i;
 555       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 556       test_put->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 557       test_put->next = all_puts;
 558       all_puts = test_put;
 559     }
 560
 561   for (i = 0; i < num_gets; i++)
 562     {
 563       test_get = GNUNET_malloc(sizeof(struct TestGetContext));
 564       test_get->uid = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_puts);
 565       temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 566       while (remember[test_get->uid][temp_daemon] == 1)
 567         temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers);
 568       test_get->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon);
 569       remember[test_get->uid][temp_daemon] = 1;
 570       test_get->next = all_gets;
 571       all_gets = test_get;
 572     }
 573
 574   GNUNET_SCHEDULER_add_now (sched, &do_put, all_puts);
 575 }
 576
 577
 578 /**
 579  * This function is called whenever a connection attempt is finished between two of
 580  * the started peers (started with GNUNET_TESTING_daemons_start).  The total
 581  * number of times this function is called should equal the number returned
 582  * from the GNUNET_TESTING_connect_topology call.
 583  *
 584  * The emsg variable is NULL on success (peers connected), and non-NULL on
 585  * failure (peers failed to connect).
 586  */
 587 void
 588 topology_callback (void *cls,
 589                    const struct GNUNET_PeerIdentity *first,
 590                    const struct GNUNET_PeerIdentity *second,
 591                    uint32_t distance,
 592                    const struct GNUNET_CONFIGURATION_Handle *first_cfg,
 593                    const struct GNUNET_CONFIGURATION_Handle *second_cfg,
 594                    struct GNUNET_TESTING_Daemon *first_daemon,
 595                    struct GNUNET_TESTING_Daemon *second_daemon,
 596                    const char *emsg)
 597 {
 598   if (emsg == NULL)
 599     {
 600       total_connections++;
 601 #if VERBOSE > 1
 602       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n",
 603                  first_daemon->shortname,
 604                  second_daemon->shortname,
 605                  distance);
 606 #endif
 607     }
 608 #if VERBOSE
 609   else
 610     {
 611       failed_connections++;
 612       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n",
 613                   first_daemon->shortname,
 614                   second_daemon->shortname, emsg);
 615     }
 616 #endif
 617
 618   if (total_connections == expected_connections)
 619     {
 620 #if VERBOSE
 621       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 622                   "Created %d total connections, which is our target number!  Starting next phase of testing.\n",
 623                   total_connections);
 624 #endif
 625       GNUNET_SCHEDULER_cancel (sched, die_task);
 626       die_task = GNUNET_SCHEDULER_add_delayed (sched, TIMEOUT,
 627                                                &end_badly, "from setup puts/gets");
 628
 629       GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2), &setup_puts_and_gets, NULL);
 630     }
 631   else if (total_connections + failed_connections == expected_connections)
 632     {
 633       GNUNET_SCHEDULER_cancel (sched, die_task);
 634       die_task = GNUNET_SCHEDULER_add_now (sched,
 635                                            &end_badly, "from topology_callback (too many failed connections)");
 636     }
 637 }
 638
 639 static void
 640 peers_started_callback (void *cls,
 641        const struct GNUNET_PeerIdentity *id,
 642        const struct GNUNET_CONFIGURATION_Handle *cfg,
 643        struct GNUNET_TESTING_Daemon *d, const char *emsg)
 644 {
 645   if (emsg != NULL)
 646     {
 647       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to start daemon with error: `%s'\n",
 648                   emsg);
 649       return;
 650     }
 651   GNUNET_assert (id != NULL);
 652
 653 #if VERBOSE
 654   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n",
 655               (num_peers - peers_left) + 1, num_peers);
 656 #endif
 657
 658   peers_left--;
 659   if (peers_left == 0)
 660     {
 661
 662 #if VERBOSE
 663       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 664                   "All %d daemons started, now connecting peers!\n",
 665                   num_peers);
 666 #endif
 667       GNUNET_SCHEDULER_cancel (sched, die_task);
 668
 669       expected_connections = -1;
 670       if ((pg != NULL) && (peers_left == 0))
 671         {
 672           expected_connections = GNUNET_TESTING_connect_topology (pg, connection_topology, connect_topology_option, connect_topology_option_modifier);
 673 #if VERBOSE
 674           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 675                       "Have %d expected connections\n", expected_connections);
 676 #endif
 677         }
 678
 679       if (expected_connections == GNUNET_SYSERR)
 680         {
 681           die_task = GNUNET_SCHEDULER_add_now (sched,
 682                                                &end_badly, "from connect topology (bad return)");
 683         }
 684
 685       die_task = GNUNET_SCHEDULER_add_delayed (sched,
 686                                                TIMEOUT,
 687                                                &end_badly, "from connect topology (timeout)");
 688
 689       ok = 0;
 690     }
 691 }
 692
 693 static void
 694 create_topology ()
 695 {
 696   peers_left = num_peers; /* Reset counter */
 697   if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology, blacklist_transports) != GNUNET_SYSERR)
 698     {
 699 #if VERBOSE
 700       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 701                   "Topology set up, now starting peers!\n");
 702 #endif
 703       GNUNET_TESTING_daemons_continue_startup(pg);
 704     }
 705   else
 706     {
 707       GNUNET_SCHEDULER_cancel (sched, die_task);
 708       die_task = GNUNET_SCHEDULER_add_now (sched,
 709                                            &end_badly, "from create topology (bad return)");
 710     }
 711   GNUNET_SCHEDULER_cancel (sched, die_task);
 712   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 713                                            TIMEOUT,
 714                                            &end_badly, "from continue startup (timeout)");
 715 }
 716
 717 /**
 718  * Callback indicating that the hostkey was created for a peer.
 719  *
 720  * @param cls NULL
 721  * @param id the peer identity
 722  * @param d the daemon handle (pretty useless at this point, remove?)
 723  * @param emsg non-null on failure
 724  */
 725 void hostkey_callback (void *cls,
 726                        const struct GNUNET_PeerIdentity *id,
 727                        struct GNUNET_TESTING_Daemon *d,
 728                        const char *emsg)
 729 {
 730   if (emsg != NULL)
 731     {
 732       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg);
 733     }
 734
 735 #if VERBOSE > 1
 736     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 737                 "Hostkey (%d/%d) created for peer `%s'\n",
 738                 num_peers - peers_left, num_peers, GNUNET_i2s(id));
 739 #endif
 740
 741
 742     peers_left--;
 743     if (peers_left == 0)
 744       {
 745 #if VERBOSE
 746         GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 747                     "All %d hostkeys created, now creating topology!\n",
 748                     num_peers);
 749 #endif
 750         GNUNET_SCHEDULER_cancel (sched, die_task);
 751         /* Set up task in case topology creation doesn't finish
 752          * within a reasonable amount of time */
 753         die_task = GNUNET_SCHEDULER_add_delayed (sched,
 754                                                  TIMEOUT,
 755                                                  &end_badly, "from create_topology");
 756         GNUNET_SCHEDULER_add_now(sched, &create_topology, NULL);
 757         ok = 0;
 758       }
 759 }
 760
 761
 762 static void
 763 run (void *cls,
 764      struct GNUNET_SCHEDULER_Handle *s,
 765      char *const *args,
 766      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
 767 {
 768   char * topology_str;
 769   char * connect_topology_str;
 770   char * blacklist_topology_str;
 771   char * connect_topology_option_str;
 772   char * connect_topology_option_modifier_string;
 773   sched = s;
 774
 775   /* Get path from configuration file */
 776   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string(cfg, "paths", "servicehome", &test_directory))
 777     {
 778       ok = 404;
 779       return;
 780     }
 781
 782   if ((GNUNET_YES ==
 783       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "topology",
 784                                             &topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&topology, topology_str)))
 785     {
 786       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 787                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "TOPOLOGY");
 788       topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */
 789     }
 790
 791   if ((GNUNET_YES ==
 792       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology",
 793                                             &connect_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&connection_topology, connect_topology_str)))
 794     {
 795       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 796                   "Invalid connect topology `%s' given for section %s option %s\n", connect_topology_str, "TESTING", "CONNECT_TOPOLOGY");
 797     }
 798   GNUNET_free_non_null(connect_topology_str);
 799   if ((GNUNET_YES ==
 800       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology_option",
 801                                             &connect_topology_option_str)) && (GNUNET_NO == GNUNET_TESTING_topology_option_get(&connect_topology_option, connect_topology_option_str)))
 802     {
 803       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 804                   "Invalid connect topology option `%s' given for section %s option %s\n", connect_topology_option_str, "TESTING", "CONNECT_TOPOLOGY_OPTION");
 805       connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */
 806     }
 807   GNUNET_free_non_null(connect_topology_option_str);
 808   if (GNUNET_YES ==
 809         GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "connect_topology_option_modifier",
 810                                                &connect_topology_option_modifier_string))
 811     {
 812       if (sscanf(connect_topology_option_modifier_string, "%lf", &connect_topology_option_modifier) != 1)
 813       {
 814         GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 815         _("Invalid value `%s' for option `%s' in section `%s': expected float\n"),
 816         connect_topology_option_modifier_string,
 817         "connect_topology_option_modifier",
 818         "TESTING");
 819       }
 820       GNUNET_free (connect_topology_option_modifier_string);
 821     }
 822
 823   if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "blacklist_transports",
 824                                          &blacklist_transports))
 825     blacklist_transports = NULL;
 826
 827   if ((GNUNET_YES ==
 828       GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "blacklist_topology",
 829                                             &blacklist_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&blacklist_topology, blacklist_topology_str)))
 830     {
 831       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 832                   "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "BLACKLIST_TOPOLOGY");
 833     }
 834   GNUNET_free_non_null(topology_str);
 835   GNUNET_free_non_null(blacklist_topology_str);
 836
 837   /* Get number of peers to start from configuration */
 838   if (GNUNET_SYSERR ==
 839       GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers",
 840                                              &num_peers))
 841     num_peers = DEFAULT_NUM_PEERS;
 842
 843   if (GNUNET_SYSERR ==
 844       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_puts",
 845                                              &num_puts))
 846     num_puts = DEFAULT_NUM_PEERS;
 847
 848   if (GNUNET_SYSERR ==
 849       GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_gets",
 850                                              &num_gets))
 851     num_gets = DEFAULT_NUM_PEERS;
 852
 853   /* Set peers_left so we know when all peers started */
 854   peers_left = num_peers;
 855
 856   /* Set up a task to end testing if peer start fails */
 857   die_task = GNUNET_SCHEDULER_add_delayed (sched,
 858                                            GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers),
 859                                            &end_badly, "didn't generate all hostkeys within a reasonable amount of time!!!");
 860
 861   pg = GNUNET_TESTING_daemons_start (sched, cfg,
 862                                      peers_left, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), &hostkey_callback, NULL, &peers_started_callback, NULL,
 863                                      &topology_callback, NULL, NULL);
 864
 865 }
 866
 867 static int
 868 check ()
 869 {
 870   int ret;
 871   /* Arguments for GNUNET_PROGRAM_run */
 872   char *const argv[] = {"test-dht-multipeer", /* Name to give running binary */
 873     "-c",
 874     "test_dht_multipeer_data.conf", /* Config file to use */
 875 #if VERBOSE
 876     "-L", "DEBUG",
 877 #endif
 878     NULL
 879   };
 880   struct GNUNET_GETOPT_CommandLineOption options[] = {
 881     GNUNET_GETOPT_OPTION_END
 882   };
 883   /* Run the run function as a new program */
 884   ret = GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1,
 885                       argv, "test-dht-multipeer", "nohelp",
 886                       options, &run, &ok);
 887   if (ret != GNUNET_OK)
 888     {
 889       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "`test-dht-multipeer': Failed with error code %d\n", ret);
 890     }
 891   return ok;
 892 }
 893
 894 int
 895 main (int argc, char *argv[])
 896 {
 897   int ret;
 898
 899   GNUNET_log_setup ("test-dht-multipeer",
 900 #if VERBOSE
 901                     "DEBUG",
 902 #else
 903                     "WARNING",
 904 #endif
 905                     NULL);
 906   ret = check ();
 907   /**
 908    * Need to remove base directory, subdirectories taken care
 909    * of by the testing framework.
 910    */
 911   if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK)
 912     {
 913       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to remove testing directory %s\n", test_directory);
 914     }
 915   return ret;
 916 }
 917
 918 /* end of test_dht_twopeer_put_get.c */