fix
[oweals/gnunet.git] / src / nse / nse-profiler.c
1 /*
2      This file is part of GNUnet.
3      (C) 2011 Christian Grothoff (and other contributing authors)
4
5      GNUnet is free software; you can redistribute it and/or modify
6      it under the terms of the GNU General Public License as published
7      by the Free Software Foundation; either version 3, or (at your
8      option) any later version.
9
10      GNUnet is distributed in the hope that it will be useful, but
11      WITHOUT ANY WARRANTY; without even the implied warranty of
12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13      General Public License for more details.
14
15      You should have received a copy of the GNU General Public License
16      along with GNUnet; see the file COPYING.  If not, write to the
17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18      Boston, MA 02111-1307, USA.
19 */
20 /**
21  * @file nse/nse-profiler.c
22  *
23  * @brief Profiling driver for the network size estimation service.
24  *        Generally, the profiler starts a given number of peers,
25  *        then churns some off, waits a certain amount of time, then
26  *        churns again, and repeats.
27  */
28 #include "platform.h"
29 #include "gnunet_testing_lib.h"
30 #include "gnunet_nse_service.h"
31
32 #define VERBOSE GNUNET_NO
33
34 struct NSEPeer
35 {
36   struct NSEPeer *prev;
37
38   struct NSEPeer *next;
39
40   struct GNUNET_TESTING_Daemon *daemon;
41
42   struct GNUNET_NSE_Handle *nse_handle;
43 };
44
45 struct StatsContext
46 {
47   GNUNET_SCHEDULER_Task task;
48   GNUNET_SCHEDULER_TaskIdentifier *task_id;
49   void *task_cls;
50   unsigned long long total_nse_bytes;
51 };
52
53 struct NSEPeer *peer_head;
54
55 struct NSEPeer *peer_tail;
56
57 /**
58  * How long until we give up on connecting the peers?
59  */
60 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1500)
61
62 static int ok;
63
64 /**
65  * Be verbose
66  */
67 static int verbose;
68
69 /**
70  * Total number of peers in the test.
71  */
72 static unsigned long long num_peers;
73
74 /**
75  * Global configuration file
76  */
77 struct GNUNET_CONFIGURATION_Handle *testing_cfg;
78
79 /**
80  * Total number of currently running peers.
81  */
82 static unsigned long long peers_running;
83
84 /**
85  * Current round we are in.
86  */
87 static unsigned long long current_round;
88
89 /**
90  * Peers desired in the next round.
91  */
92 static unsigned long long peers_next_round;
93
94 /**
95  * Maximum number of connections to NSE services.
96  */
97 static unsigned long long connection_limit;
98
99 /**
100  * Total number of connections in the whole network.
101  */
102 static unsigned int total_connections;
103
104 /**
105  * The currently running peer group.
106  */
107 static struct GNUNET_TESTING_PeerGroup *pg;
108
109 /**
110  * File to report results to.
111  */
112 static struct GNUNET_DISK_FileHandle *output_file;
113
114 /**
115  * File to log connection info, statistics to.
116  */
117 static struct GNUNET_DISK_FileHandle *data_file;
118
119 /**
120  * How many data points to capture before triggering next round?
121  */
122 static struct GNUNET_TIME_Relative wait_time;
123
124 /**
125  * Task called to disconnect peers.
126  */
127 static GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
128
129 /**
130  * Task called to shutdown test.
131  */
132 static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
133
134 /**
135  * Task used to churn the network.
136  */
137 static GNUNET_SCHEDULER_TaskIdentifier churn_task;
138
139 static char *topology_file;
140
141 static char *data_filename;
142
143 static uint64_t clock_skew;
144
145 /**
146  * Check whether peers successfully shut down.
147  */
148 void
149 shutdown_callback (void *cls, const char *emsg)
150 {
151   if (emsg != NULL)
152     {
153 #if VERBOSE
154       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutdown of peers failed!\n");
155 #endif
156       if (ok == 0)
157         ok = 666;
158     }
159   else
160     {
161 #if VERBOSE
162       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
163                   "All peers successfully shut down!\n");
164 #endif
165       ok = 0;
166     }
167 }
168
169 static void
170 shutdown_task (void *cls,
171                const struct GNUNET_SCHEDULER_TaskContext *tc)
172 {
173   struct NSEPeer *pos;
174 #if VERBOSE
175   fprintf(stderr, "Ending test.\n");
176 #endif
177
178   if (disconnect_task != GNUNET_SCHEDULER_NO_TASK)
179     {
180       GNUNET_SCHEDULER_cancel(disconnect_task);
181       disconnect_task = GNUNET_SCHEDULER_NO_TASK;
182     }
183   while (NULL != (pos = peer_head))
184     {
185       if (pos->nse_handle != NULL)
186         GNUNET_NSE_disconnect(pos->nse_handle);
187       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
188       GNUNET_free(pos);
189     }
190
191   if (data_file != NULL)
192     GNUNET_DISK_file_close(data_file);
193   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
194 }
195
196 /**
197  * Callback to call when network size estimate is updated.
198  *
199  * @param cls closure
200  * @param estimate the value of the current network size estimate
201  * @param std_dev standard deviation (rounded down to nearest integer)
202  *                of the size estimation values seen
203  *
204  */
205 static void
206 handle_estimate (void *cls, double estimate, double std_dev)
207 {
208   struct NSEPeer *peer = cls;
209   char *output_buffer;
210   int size;
211   //fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
212   if (output_file != NULL)
213     {
214       size = GNUNET_asprintf(&output_buffer, "%s %u %f %f\n", GNUNET_i2s(&peer->daemon->id), peers_running, estimate, std_dev);
215       if (size != GNUNET_DISK_file_write(output_file, output_buffer, size))
216         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Unable to write to file!\n", "nse-profiler");
217     }
218   else
219     fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
220
221 }
222
223 static void
224 connect_nse_service (void *cls,
225                      const struct GNUNET_SCHEDULER_TaskContext *tc)
226 {
227   struct NSEPeer *current_peer;
228   unsigned int i;
229 #if VERBOSE
230   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
231 #endif
232   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
233   for (i = 0; i < num_peers; i++)
234     {
235       if ((connection_limit > 0) && (i % (num_peers / connection_limit) != 0))
236         continue;
237       current_peer = GNUNET_malloc(sizeof(struct NSEPeer));
238       current_peer->daemon = GNUNET_TESTING_daemon_get(pg, i);
239       if (GNUNET_YES == GNUNET_TESTING_daemon_running(GNUNET_TESTING_daemon_get(pg, i)))
240         {
241           current_peer->nse_handle = GNUNET_NSE_connect (current_peer->daemon->cfg, &handle_estimate, current_peer);
242           GNUNET_assert(current_peer->nse_handle != NULL);
243         }
244       GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer);
245     }
246 }
247
248 static void
249 churn_peers (void *cls,
250              const struct GNUNET_SCHEDULER_TaskContext *tc);
251
252 /**
253  * Continuation called by the "get_all" and "get" functions.
254  *
255  * @param cls struct StatsContext
256  * @param success GNUNET_OK if statistics were
257  *        successfully obtained, GNUNET_SYSERR if not.
258  */
259 static void stats_finished_callback (void *cls, int success)
260 {
261   struct StatsContext *stats_context = (struct StatsContext *)cls;
262   char *buf;
263   int buf_len;
264
265   if ((GNUNET_OK == success) && (data_file != NULL)) /* Stats lookup successful, write out data */
266     {
267       buf = NULL;
268       buf_len = GNUNET_asprintf(&buf, "TOTAL_NSE_BYTES: %u\n", stats_context->total_nse_bytes);
269       if (buf_len > 0)
270         {
271           GNUNET_DISK_file_write(data_file, buf, buf_len);
272         }
273       GNUNET_free_non_null(buf);
274     }
275
276   if (stats_context->task != NULL)
277     (*stats_context->task_id) = GNUNET_SCHEDULER_add_now(stats_context->task, stats_context->task_cls);
278
279   GNUNET_free(stats_context);
280 }
281
282 /**
283  * Callback function to process statistic values.
284  *
285  * @param cls struct StatsContext
286  * @param peer the peer the statistics belong to
287  * @param subsystem name of subsystem that created the statistic
288  * @param name the name of the datum
289  * @param value the current value
290  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
291  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
292  */
293 static int statistics_iterator (void *cls,
294                                 const struct GNUNET_PeerIdentity *peer,
295                                 const char *subsystem,
296                                 const char *name,
297                                 uint64_t value,
298                                 int is_persistent)
299 {
300   struct StatsContext *stats_context = (struct StatsContext *)cls;
301   char *buf;
302
303   GNUNET_assert(0 < GNUNET_asprintf(&buf, "bytes of messages of type %d received", GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD));
304   if ((0 == strstr(subsystem, "core")) && (0 == strstr(name, buf)))
305     {
306       stats_context->total_nse_bytes += value;
307     }
308   GNUNET_free(buf);
309   return GNUNET_OK;
310 }
311
312 /**
313  * @param cls struct StatsContext
314  * @param tc task context
315  */
316 static void
317 get_statistics (void *cls,
318                 const struct GNUNET_SCHEDULER_TaskContext *tc)
319 {
320   struct StatsContext *stats_context = (struct StatsContext *)cls;
321   GNUNET_TESTING_get_statistics(pg, &stats_finished_callback, &statistics_iterator, stats_context);
322 }
323
324 static void
325 disconnect_nse_peers (void *cls,
326                       const struct GNUNET_SCHEDULER_TaskContext *tc)
327 {
328   struct NSEPeer *pos;
329   char *buf;
330   struct StatsContext *stats_context;
331   disconnect_task = GNUNET_SCHEDULER_NO_TASK;
332   pos = peer_head;
333
334   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: disconnecting nse service of peers\n");
335   while (NULL != (pos = peer_head))
336     {
337       if (pos->nse_handle != NULL)
338         {
339           GNUNET_NSE_disconnect(pos->nse_handle);
340           pos->nse_handle = NULL;
341         }
342       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
343       GNUNET_free(pos);
344     }
345
346   GNUNET_asprintf(&buf, "round%llu", current_round);
347   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", buf, &peers_next_round))
348     {
349       current_round++;
350       GNUNET_assert(churn_task == GNUNET_SCHEDULER_NO_TASK);
351       churn_task = GNUNET_SCHEDULER_add_now(&churn_peers, NULL);
352     }
353   else /* No more rounds, let's shut it down! */
354     {
355       stats_context = GNUNET_malloc(sizeof(struct StatsContext));
356       GNUNET_SCHEDULER_cancel(shutdown_handle);
357       shutdown_handle = GNUNET_SCHEDULER_NO_TASK;
358       stats_context->task = &shutdown_task;
359       stats_context->task_cls = NULL;
360       stats_context->task_id = &shutdown_handle;
361       GNUNET_SCHEDULER_add_now(&get_statistics, stats_context);
362       //shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
363     }
364   GNUNET_free(buf);
365 }
366
367 /**
368  * Prototype of a function that will be called when a
369  * particular operation was completed the testing library.
370  *
371  * @param cls unused
372  * @param emsg NULL on success
373  */
374 void topology_output_callback (void *cls, const char *emsg)
375 {
376   struct StatsContext *stats_context;
377   stats_context = GNUNET_malloc(sizeof(struct StatsContext));
378
379   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
380   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
381 }
382
383
384 /**
385  * Prototype of a function that will be called when a
386  * particular operation was completed the testing library.
387  *
388  * @param cls closure
389  * @param emsg NULL on success
390  */
391 static void
392 churn_callback (void *cls, const char *emsg)
393 {
394   char *temp_output_file;
395
396   if (emsg == NULL) /* Everything is okay! */
397     {
398       peers_running = GNUNET_TESTING_daemons_running(pg);
399       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
400                   "Round %lu, churn finished successfully.\n", current_round);
401       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
402       GNUNET_asprintf(&temp_output_file, "%s%lu.dot", topology_file, current_round);
403       GNUNET_TESTING_peergroup_topology_to_file(pg,
404                                                 temp_output_file,
405                                                 &topology_output_callback,
406                                                 NULL);
407       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Writing topology to file %s\n", temp_output_file);
408       GNUNET_free(temp_output_file);
409     }
410   else
411     {
412       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
413                   "Round %lu, churn FAILED!!\n", current_round);
414       GNUNET_SCHEDULER_cancel(shutdown_handle);
415       GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
416     }
417 }
418
419 static void
420 churn_peers (void *cls,
421                       const struct GNUNET_SCHEDULER_TaskContext *tc)
422 {
423   peers_running = GNUNET_TESTING_daemons_running(pg);
424   churn_task = GNUNET_SCHEDULER_NO_TASK;
425   if (peers_next_round == peers_running)
426     {
427       /* Nothing to do... */
428       GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
429       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
430       disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
431       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Round %lu, doing nothing!\n", current_round);
432     }
433   else
434     {
435       if (peers_next_round > num_peers)
436         {
437           GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Asked to turn on more peers than have!!\n");
438           GNUNET_SCHEDULER_cancel(shutdown_handle);
439           GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
440         }
441       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
442                   "Round %lu, turning off %lu peers, turning on %lu peers!\n",
443                   current_round,
444                   (peers_running > peers_next_round) ? peers_running
445                       - peers_next_round : 0,
446                   (peers_next_round > peers_running) ? peers_next_round
447                       - peers_running : 0);
448       GNUNET_TESTING_daemons_churn (pg,
449                                     (peers_running > peers_next_round) ? peers_running
450                                         - peers_next_round
451                                         : 0,
452                                     (peers_next_round > peers_running) ? peers_next_round
453                                         - peers_running
454                                         : 0, wait_time, &churn_callback,
455                                     NULL);
456     }
457 }
458
459
460 static void
461 my_cb (void *cls,
462        const char *emsg)
463 {
464   char *buf;
465   int buf_len;
466   if (emsg != NULL)
467     {
468       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
469                   "Peergroup callback called with error, aborting test!\n");
470       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error from testing: `%s'\n");
471       ok = 1;
472       GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
473       return;
474     }
475 #if VERBOSE
476   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
477               "Peer Group started successfully, connecting to NSE service for each peer!\n");
478 #endif
479   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Have %u connections\n", total_connections);
480   if (data_file != NULL)
481     {
482       buf = NULL;
483       buf_len = GNUNET_asprintf(&buf, "CONNECTIONS_0: %u\n", total_connections);
484       if (buf_len > 0)
485         GNUNET_DISK_file_write(data_file, buf, buf_len);
486       GNUNET_free_non_null(buf);
487     }
488   peers_running = GNUNET_TESTING_daemons_running(pg);
489   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
490   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
491 }
492
493 /**
494  * Prototype of a function that will be called whenever
495  * two daemons are connected by the testing library.
496  *
497  * @param cls closure
498  * @param first peer id for first daemon
499  * @param second peer id for the second daemon
500  * @param distance distance between the connected peers
501  * @param first_cfg config for the first daemon
502  * @param second_cfg config for the second daemon
503  * @param first_daemon handle for the first daemon
504  * @param second_daemon handle for the second daemon
505  * @param emsg error message (NULL on success)
506  */
507 void connect_cb (void *cls,
508                  const struct GNUNET_PeerIdentity *first,
509                  const struct GNUNET_PeerIdentity *second,
510                  uint32_t distance,
511                  const struct GNUNET_CONFIGURATION_Handle *first_cfg,
512                  const struct GNUNET_CONFIGURATION_Handle *second_cfg,
513                  struct GNUNET_TESTING_Daemon *first_daemon,
514                  struct GNUNET_TESTING_Daemon *second_daemon,
515                  const char *emsg)
516 {
517   char *second_id;
518
519   second_id = GNUNET_strdup(GNUNET_i2s(second));
520   if (emsg == NULL)
521     total_connections++;
522 }
523
524
525 static void
526 run (void *cls,
527      char *const *args,
528      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
529 {
530   char *temp_str;
531   unsigned long long temp_wait;
532   ok = 1;
533   testing_cfg = GNUNET_CONFIGURATION_create();
534   GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_load(testing_cfg, cfgfile));
535 #if VERBOSE
536   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n");
537   GNUNET_CONFIGURATION_set_value_string (testing_cfg, "testing",
538                                            "use_progressbars",
539                                            "YES");
540 #endif
541   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "testing", "num_peers", &num_peers))
542     {
543       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option TESTING:NUM_PEERS is required!\n");
544       return;
545     }
546
547   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", "wait_time", &temp_wait))
548     {
549       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:wait_time is required!\n");
550       return;
551     }
552
553   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", "connection_limit", &connection_limit))
554     {
555       connection_limit = 0;
556     }
557
558   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "topology_output_file", &topology_file))
559     {
560       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:topology_output_file is required!\n");
561       return;
562     }
563
564   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "data_output_file", &data_filename))
565     {
566       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:data_output_file is required!\n");
567       return;
568     }
569
570   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (testing_cfg, "nse-profiler", "skew_clock"))
571     {
572       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Setting our clock as skewed...\n");
573       clock_skew = GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK, GNUNET_TIME_UNIT_MINUTES.rel_value);
574     }
575
576
577   data_file = GNUNET_DISK_file_open (data_filename, GNUNET_DISK_OPEN_READWRITE
578                                                   | GNUNET_DISK_OPEN_CREATE,
579                                                   GNUNET_DISK_PERM_USER_READ |
580                                                   GNUNET_DISK_PERM_USER_WRITE);
581   if (data_file == NULL)
582     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", data_filename);
583   GNUNET_free(data_filename);
584
585   wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
586
587   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg, "nse-profiler", "output_file", &temp_str))
588     {
589       output_file = GNUNET_DISK_file_open (temp_str, GNUNET_DISK_OPEN_READWRITE
590                                                       | GNUNET_DISK_OPEN_CREATE,
591                                                       GNUNET_DISK_PERM_USER_READ |
592                                                       GNUNET_DISK_PERM_USER_WRITE);
593       if (output_file == NULL)
594         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", temp_str);
595     }
596   GNUNET_free_non_null(temp_str);
597
598   pg = GNUNET_TESTING_peergroup_start(testing_cfg,
599                                       num_peers,
600                                       TIMEOUT,
601                                       &connect_cb,
602                                       &my_cb, NULL,
603                                       NULL);
604   GNUNET_assert (pg != NULL);
605   shutdown_handle = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever(), &shutdown_task, NULL);
606 }
607
608
609
610 /**
611  * nse-profiler command line options
612  */
613 static struct GNUNET_GETOPT_CommandLineOption options[] = {
614   {'V', "verbose", NULL,
615    gettext_noop ("be verbose (print progress information)"),
616    0, &GNUNET_GETOPT_set_one, &verbose},
617   GNUNET_GETOPT_OPTION_END
618 };
619
620 int
621 main (int argc, char *argv[])
622 {
623   int ret;
624
625   GNUNET_log_setup ("nse-profiler",
626 #if VERBOSE
627                     "DEBUG",
628 #else
629                     "WARNING",
630 #endif
631                     NULL);
632   ret = 1;
633   GNUNET_PROGRAM_run (argc,
634                       argv, "nse-profiler", gettext_noop
635                       ("Run a test of the NSE service."),
636                       options, &run, &ok);
637
638   GNUNET_DISK_directory_remove ("/tmp/nse-profiler");
639   return ret;
640 }
641
642 /* end of nse-profiler.c */