larger history, connect limit
[oweals/gnunet.git] / src / nse / nse-profiler.c
1 /*
2      This file is part of GNUnet.
3      (C) 2009 Christian Grothoff (and other contributing authors)
4
5      GNUnet is free software; you can redistribute it and/or modify
6      it under the terms of the GNU General Public License as published
7      by the Free Software Foundation; either version 3, or (at your
8      option) any later version.
9
10      GNUnet is distributed in the hope that it will be useful, but
11      WITHOUT ANY WARRANTY; without even the implied warranty of
12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13      General Public License for more details.
14
15      You should have received a copy of the GNU General Public License
16      along with GNUnet; see the file COPYING.  If not, write to the
17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18      Boston, MA 02111-1307, USA.
19 */
20 /**
21  * @file nse/nse-profiler.c
22  *
23  * @brief Profiling driver for the network size estimation service.
24  *        Generally, the profiler starts a given number of peers,
25  *        then churns some off, waits a certain amount of time, then
26  *        churns again, and repeats.
27  */
28 #include "platform.h"
29 #include "gnunet_testing_lib.h"
30 #include "gnunet_nse_service.h"
31
32 #define VERBOSE GNUNET_NO
33
34 #define CONNECT_LIMIT GNUNET_YES
35
36 struct NSEPeer
37 {
38   struct NSEPeer *prev;
39
40   struct NSEPeer *next;
41
42   struct GNUNET_TESTING_Daemon *daemon;
43
44   struct GNUNET_NSE_Handle *nse_handle;
45 };
46
47 struct StatsContext
48 {
49   GNUNET_SCHEDULER_Task task;
50   GNUNET_SCHEDULER_TaskIdentifier *task_id;
51   void *task_cls;
52   unsigned long long total_nse_bytes;
53 };
54
55 struct NSEPeer *peer_head;
56
57 struct NSEPeer *peer_tail;
58
59 /**
60  * How long until we give up on connecting the peers?
61  */
62 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1500)
63
64 static int ok;
65
66 /**
67  * Be verbose
68  */
69 static int verbose;
70
71 /**
72  * Total number of peers in the test.
73  */
74 static unsigned long long num_peers;
75
76 /**
77  * Global configuration file
78  */
79 struct GNUNET_CONFIGURATION_Handle *testing_cfg;
80
81 /**
82  * Total number of currently running peers.
83  */
84 static unsigned long long peers_running;
85
86 /**
87  * Current round we are in.
88  */
89 static unsigned long long current_round;
90
91 /**
92  * Peers desired in the next round.
93  */
94 static unsigned long long peers_next_round;
95
96 /**
97  * Total number of connections in the whole network.
98  */
99 static unsigned int total_connections;
100
101 /**
102  * The currently running peer group.
103  */
104 static struct GNUNET_TESTING_PeerGroup *pg;
105
106 /**
107  * File to report results to.
108  */
109 static struct GNUNET_DISK_FileHandle *output_file;
110
111 /**
112  * File to log connection info, statistics to.
113  */
114 static struct GNUNET_DISK_FileHandle *data_file;
115
116 /**
117  * How many data points to capture before triggering next round?
118  */
119 static struct GNUNET_TIME_Relative wait_time;
120
121 /**
122  * Task called to disconnect peers.
123  */
124 static GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
125
126 /**
127  * Task called to shutdown test.
128  */
129 static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
130
131 /**
132  * Task used to churn the network.
133  */
134 static GNUNET_SCHEDULER_TaskIdentifier churn_task;
135
136 static char *topology_file;
137
138 static char *data_filename;
139
140 static uint64_t clock_skew;
141
142 /**
143  * Check whether peers successfully shut down.
144  */
145 void
146 shutdown_callback (void *cls, const char *emsg)
147 {
148   if (emsg != NULL)
149     {
150 #if VERBOSE
151       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutdown of peers failed!\n");
152 #endif
153       if (ok == 0)
154         ok = 666;
155     }
156   else
157     {
158 #if VERBOSE
159       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
160                   "All peers successfully shut down!\n");
161 #endif
162       ok = 0;
163     }
164 }
165
166 static void
167 shutdown_task (void *cls,
168                const struct GNUNET_SCHEDULER_TaskContext *tc)
169 {
170   struct NSEPeer *pos;
171 #if VERBOSE
172   fprintf(stderr, "Ending test.\n");
173 #endif
174
175   if (disconnect_task != GNUNET_SCHEDULER_NO_TASK)
176     {
177       GNUNET_SCHEDULER_cancel(disconnect_task);
178       disconnect_task = GNUNET_SCHEDULER_NO_TASK;
179     }
180   while (NULL != (pos = peer_head))
181     {
182       if (pos->nse_handle != NULL)
183         GNUNET_NSE_disconnect(pos->nse_handle);
184       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
185       GNUNET_free(pos);
186     }
187
188   if (data_file != NULL)
189     GNUNET_DISK_file_close(data_file);
190   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
191 }
192
193 /**
194  * Callback to call when network size estimate is updated.
195  *
196  * @param cls closure
197  * @param estimate the value of the current network size estimate
198  * @param std_dev standard deviation (rounded down to nearest integer)
199  *                of the size estimation values seen
200  *
201  */
202 static void
203 handle_estimate (void *cls, double estimate, double std_dev)
204 {
205   struct NSEPeer *peer = cls;
206   char *output_buffer;
207   int size;
208   //fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
209   if (output_file != NULL)
210     {
211       size = GNUNET_asprintf(&output_buffer, "%s %u %f %f\n", GNUNET_i2s(&peer->daemon->id), peers_running, estimate, std_dev);
212       if (size != GNUNET_DISK_file_write(output_file, output_buffer, size))
213         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Unable to write to file!\n", "nse-profiler");
214     }
215   else
216     fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
217
218 }
219
220 static void
221 connect_nse_service (void *cls,
222                      const struct GNUNET_SCHEDULER_TaskContext *tc)
223 {
224   struct NSEPeer *current_peer;
225   unsigned int i;
226 #if VERBOSE
227   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
228 #endif
229   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
230   for (i = 0; i < num_peers; i++)
231     {
232 #if CONNECT_LIMIT
233       if (i % 50 != 0)
234         continue;
235 #endif
236       current_peer = GNUNET_malloc(sizeof(struct NSEPeer));
237       current_peer->daemon = GNUNET_TESTING_daemon_get(pg, i);
238       if (GNUNET_YES == GNUNET_TESTING_daemon_running(GNUNET_TESTING_daemon_get(pg, i)))
239         {
240           current_peer->nse_handle = GNUNET_NSE_connect (current_peer->daemon->cfg, &handle_estimate, current_peer);
241           GNUNET_assert(current_peer->nse_handle != NULL);
242         }
243       GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer);
244     }
245 }
246
247 static void
248 churn_peers (void *cls,
249              const struct GNUNET_SCHEDULER_TaskContext *tc);
250
251 /**
252  * Continuation called by the "get_all" and "get" functions.
253  *
254  * @param cls struct StatsContext
255  * @param success GNUNET_OK if statistics were
256  *        successfully obtained, GNUNET_SYSERR if not.
257  */
258 static void stats_finished_callback (void *cls, int success)
259 {
260   struct StatsContext *stats_context = (struct StatsContext *)cls;
261   char *buf;
262   int buf_len;
263
264   if ((GNUNET_OK == success) && (data_file != NULL)) /* Stats lookup successful, write out data */
265     {
266       buf = NULL;
267       buf_len = GNUNET_asprintf(&buf, "TOTAL_NSE_BYTES: %u\n", stats_context->total_nse_bytes);
268       if (buf_len > 0)
269         {
270           GNUNET_DISK_file_write(data_file, buf, buf_len);
271         }
272       GNUNET_free_non_null(buf);
273     }
274
275   if (stats_context->task != NULL)
276     (*stats_context->task_id) = GNUNET_SCHEDULER_add_now(stats_context->task, stats_context->task_cls);
277
278   GNUNET_free(stats_context);
279 }
280
281 /**
282  * Callback function to process statistic values.
283  *
284  * @param cls struct StatsContext
285  * @param peer the peer the statistics belong to
286  * @param subsystem name of subsystem that created the statistic
287  * @param name the name of the datum
288  * @param value the current value
289  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
290  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
291  */
292 static int statistics_iterator (void *cls,
293                                 const struct GNUNET_PeerIdentity *peer,
294                                 const char *subsystem,
295                                 const char *name,
296                                 uint64_t value,
297                                 int is_persistent)
298 {
299   struct StatsContext *stats_context = (struct StatsContext *)cls;
300   char *buf;
301
302   GNUNET_assert(0 < GNUNET_asprintf(&buf, "bytes of messages of type %d received", GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD));
303   if ((0 == strstr(subsystem, "core")) && (0 == strstr(name, buf)))
304     {
305       stats_context->total_nse_bytes += value;
306     }
307   GNUNET_free(buf);
308   return GNUNET_OK;
309 }
310
311 /**
312  * @param cls struct StatsContext
313  * @param tc task context
314  */
315 static void
316 get_statistics (void *cls,
317                 const struct GNUNET_SCHEDULER_TaskContext *tc)
318 {
319   struct StatsContext *stats_context = (struct StatsContext *)cls;
320   GNUNET_TESTING_get_statistics(pg, &stats_finished_callback, &statistics_iterator, stats_context);
321 }
322
323 static void
324 disconnect_nse_peers (void *cls,
325                       const struct GNUNET_SCHEDULER_TaskContext *tc)
326 {
327   struct NSEPeer *pos;
328   char *buf;
329   struct StatsContext *stats_context;
330   disconnect_task = GNUNET_SCHEDULER_NO_TASK;
331   pos = peer_head;
332
333   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: disconnecting nse service of peers\n");
334   while (NULL != (pos = peer_head))
335     {
336       if (pos->nse_handle != NULL)
337         {
338           GNUNET_NSE_disconnect(pos->nse_handle);
339           pos->nse_handle = NULL;
340         }
341       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
342       GNUNET_free(pos);
343     }
344
345   GNUNET_asprintf(&buf, "round%llu", current_round);
346   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", buf, &peers_next_round))
347     {
348       current_round++;
349       GNUNET_assert(churn_task == GNUNET_SCHEDULER_NO_TASK);
350       churn_task = GNUNET_SCHEDULER_add_now(&churn_peers, NULL);
351     }
352   else /* No more rounds, let's shut it down! */
353     {
354       stats_context = GNUNET_malloc(sizeof(struct StatsContext));
355       GNUNET_SCHEDULER_cancel(shutdown_handle);
356       shutdown_handle = GNUNET_SCHEDULER_NO_TASK;
357       stats_context->task = &shutdown_task;
358       stats_context->task_cls = NULL;
359       stats_context->task_id = &shutdown_handle;
360       GNUNET_SCHEDULER_add_now(&get_statistics, stats_context);
361       //shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
362     }
363   GNUNET_free(buf);
364 }
365
366 /**
367  * Prototype of a function that will be called when a
368  * particular operation was completed the testing library.
369  *
370  * @param cls unused
371  * @param emsg NULL on success
372  */
373 void topology_output_callback (void *cls, const char *emsg)
374 {
375   struct StatsContext *stats_context;
376   stats_context = GNUNET_malloc(sizeof(struct StatsContext));
377
378   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
379   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
380 }
381
382
383 /**
384  * Prototype of a function that will be called when a
385  * particular operation was completed the testing library.
386  *
387  * @param cls closure
388  * @param emsg NULL on success
389  */
390 static void
391 churn_callback (void *cls, const char *emsg)
392 {
393   char *temp_output_file;
394
395   if (emsg == NULL) /* Everything is okay! */
396     {
397       peers_running = GNUNET_TESTING_daemons_running(pg);
398       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
399                   "Round %lu, churn finished successfully.\n", current_round);
400       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
401       GNUNET_asprintf(&temp_output_file, "%s%lu.dot", topology_file, current_round);
402       GNUNET_TESTING_peergroup_topology_to_file(pg,
403                                                 temp_output_file,
404                                                 &topology_output_callback,
405                                                 NULL);
406       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Writing topology to file %s\n", temp_output_file);
407       GNUNET_free(temp_output_file);
408     }
409   else
410     {
411       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
412                   "Round %lu, churn FAILED!!\n", current_round);
413       GNUNET_SCHEDULER_cancel(shutdown_handle);
414       GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
415     }
416 }
417
418 static void
419 churn_peers (void *cls,
420                       const struct GNUNET_SCHEDULER_TaskContext *tc)
421 {
422   peers_running = GNUNET_TESTING_daemons_running(pg);
423   churn_task = GNUNET_SCHEDULER_NO_TASK;
424   if (peers_next_round == peers_running)
425     {
426       /* Nothing to do... */
427       GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
428       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
429       disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
430       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Round %lu, doing nothing!\n", current_round);
431     }
432   else
433     {
434       if (peers_next_round > num_peers)
435         {
436           GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Asked to turn on more peers than have!!\n");
437           GNUNET_SCHEDULER_cancel(shutdown_handle);
438           GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
439         }
440       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
441                   "Round %lu, turning off %lu peers, turning on %lu peers!\n",
442                   current_round,
443                   (peers_running > peers_next_round) ? peers_running
444                       - peers_next_round : 0,
445                   (peers_next_round > peers_running) ? peers_next_round
446                       - peers_running : 0);
447       GNUNET_TESTING_daemons_churn (pg,
448                                     (peers_running > peers_next_round) ? peers_running
449                                         - peers_next_round
450                                         : 0,
451                                     (peers_next_round > peers_running) ? peers_next_round
452                                         - peers_running
453                                         : 0, wait_time, &churn_callback,
454                                     NULL);
455     }
456 }
457
458
459 static void
460 my_cb (void *cls,
461        const char *emsg)
462 {
463   char *buf;
464   int buf_len;
465   if (emsg != NULL)
466     {
467       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
468                   "Peergroup callback called with error, aborting test!\n");
469       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error from testing: `%s'\n");
470       ok = 1;
471       GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
472       return;
473     }
474 #if VERBOSE
475   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
476               "Peer Group started successfully, connecting to NSE service for each peer!\n");
477 #endif
478   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Have %u connections\n", total_connections);
479   if (data_file != NULL)
480     {
481       buf = NULL;
482       buf_len = GNUNET_asprintf(&buf, "CONNECTIONS_0: %u\n", total_connections);
483       if (buf_len > 0)
484         GNUNET_DISK_file_write(data_file, buf, buf_len);
485       GNUNET_free_non_null(buf);
486     }
487   peers_running = GNUNET_TESTING_daemons_running(pg);
488   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
489   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
490 }
491
492 /**
493  * Prototype of a function that will be called whenever
494  * two daemons are connected by the testing library.
495  *
496  * @param cls closure
497  * @param first peer id for first daemon
498  * @param second peer id for the second daemon
499  * @param distance distance between the connected peers
500  * @param first_cfg config for the first daemon
501  * @param second_cfg config for the second daemon
502  * @param first_daemon handle for the first daemon
503  * @param second_daemon handle for the second daemon
504  * @param emsg error message (NULL on success)
505  */
506 void connect_cb (void *cls,
507                  const struct GNUNET_PeerIdentity *first,
508                  const struct GNUNET_PeerIdentity *second,
509                  uint32_t distance,
510                  const struct GNUNET_CONFIGURATION_Handle *first_cfg,
511                  const struct GNUNET_CONFIGURATION_Handle *second_cfg,
512                  struct GNUNET_TESTING_Daemon *first_daemon,
513                  struct GNUNET_TESTING_Daemon *second_daemon,
514                  const char *emsg)
515 {
516   char *second_id;
517
518   second_id = GNUNET_strdup(GNUNET_i2s(second));
519   if (emsg == NULL)
520     total_connections++;
521 }
522
523
524 static void
525 run (void *cls,
526      char *const *args,
527      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
528 {
529   char *temp_str;
530   unsigned long long temp_wait;
531   ok = 1;
532   testing_cfg = GNUNET_CONFIGURATION_create();
533   GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_load(testing_cfg, cfgfile));
534 #if VERBOSE
535   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n");
536   GNUNET_CONFIGURATION_set_value_string (testing_cfg, "testing",
537                                            "use_progressbars",
538                                            "YES");
539 #endif
540   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "testing", "num_peers", &num_peers))
541     {
542       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option TESTING:NUM_PEERS is required!\n");
543       return;
544     }
545
546   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", "wait_time", &temp_wait))
547     {
548       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:wait_time is required!\n");
549       return;
550     }
551
552   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "topology_output_file", &topology_file))
553     {
554       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:topology_output_file is required!\n");
555       return;
556     }
557
558   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "data_output_file", &data_filename))
559     {
560       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:data_output_file is required!\n");
561       return;
562     }
563
564   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (testing_cfg, "nse-profiler", "skew_clock"))
565     {
566       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Setting our clock as skewed...\n");
567       clock_skew = GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_TIME_UNIT_MINUTES.rel_value);
568     }
569
570
571   data_file = GNUNET_DISK_file_open (data_filename, GNUNET_DISK_OPEN_READWRITE
572                                                   | GNUNET_DISK_OPEN_CREATE,
573                                                   GNUNET_DISK_PERM_USER_READ |
574                                                   GNUNET_DISK_PERM_USER_WRITE);
575   if (data_file == NULL)
576     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", data_filename);
577   GNUNET_free(data_filename);
578
579   wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
580
581   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg, "nse-profiler", "output_file", &temp_str))
582     {
583       output_file = GNUNET_DISK_file_open (temp_str, GNUNET_DISK_OPEN_READWRITE
584                                                       | GNUNET_DISK_OPEN_CREATE,
585                                                       GNUNET_DISK_PERM_USER_READ |
586                                                       GNUNET_DISK_PERM_USER_WRITE);
587       if (output_file == NULL)
588         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", temp_str);
589     }
590   GNUNET_free_non_null(temp_str);
591
592   pg = GNUNET_TESTING_peergroup_start(testing_cfg,
593                                       num_peers,
594                                       TIMEOUT,
595                                       &connect_cb,
596                                       &my_cb, NULL,
597                                       NULL);
598   GNUNET_assert (pg != NULL);
599   shutdown_handle = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever(), &shutdown_task, NULL);
600 }
601
602
603
604 /**
605  * nse-profiler command line options
606  */
607 static struct GNUNET_GETOPT_CommandLineOption options[] = {
608   {'V', "verbose", NULL,
609    gettext_noop ("be verbose (print progress information)"),
610    0, &GNUNET_GETOPT_set_one, &verbose},
611   GNUNET_GETOPT_OPTION_END
612 };
613
614 int
615 main (int argc, char *argv[])
616 {
617   int ret;
618
619   GNUNET_log_setup ("nse-profiler",
620 #if VERBOSE
621                     "DEBUG",
622 #else
623                     "WARNING",
624 #endif
625                     NULL);
626   ret = 1;
627   GNUNET_PROGRAM_run (argc,
628                       argv, "nse-profiler", gettext_noop
629                       ("Run a test of the NSE service."),
630                       options, &run, &ok);
631
632   GNUNET_DISK_directory_remove ("/tmp/nse-profiler");
633   return ret;
634 }
635
636 /* end of nse-profiler.c */