nse changes
[oweals/gnunet.git] / src / nse / nse-profiler.c
1 /*
2      This file is part of GNUnet.
3      (C) 2009 Christian Grothoff (and other contributing authors)
4
5      GNUnet is free software; you can redistribute it and/or modify
6      it under the terms of the GNU General Public License as published
7      by the Free Software Foundation; either version 3, or (at your
8      option) any later version.
9
10      GNUnet is distributed in the hope that it will be useful, but
11      WITHOUT ANY WARRANTY; without even the implied warranty of
12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13      General Public License for more details.
14
15      You should have received a copy of the GNU General Public License
16      along with GNUnet; see the file COPYING.  If not, write to the
17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18      Boston, MA 02111-1307, USA.
19 */
20 /**
21  * @file nse/nse-profiler.c
22  *
23  * @brief Profiling driver for the network size estimation service.
24  *        Generally, the profiler starts a given number of peers,
25  *        then churns some off, waits a certain amount of time, then
26  *        churns again, and repeats.
27  */
28 #include "platform.h"
29 #include "gnunet_testing_lib.h"
30 #include "gnunet_nse_service.h"
31
32 #define VERBOSE GNUNET_NO
33
34 struct NSEPeer
35 {
36   struct NSEPeer *prev;
37
38   struct NSEPeer *next;
39
40   struct GNUNET_TESTING_Daemon *daemon;
41
42   struct GNUNET_NSE_Handle *nse_handle;
43 };
44
45 struct StatsContext
46 {
47   GNUNET_SCHEDULER_Task task;
48   GNUNET_SCHEDULER_TaskIdentifier *task_id;
49   void *task_cls;
50   unsigned long long total_nse_bytes;
51 };
52
53 struct NSEPeer *peer_head;
54
55 struct NSEPeer *peer_tail;
56
57 /**
58  * How long until we give up on connecting the peers?
59  */
60 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1500)
61
62 static int ok;
63
64 /**
65  * Be verbose
66  */
67 static int verbose;
68
69 /**
70  * Total number of peers in the test.
71  */
72 static unsigned long long num_peers;
73
74 /**
75  * Global configuration file
76  */
77 struct GNUNET_CONFIGURATION_Handle *testing_cfg;
78
79 /**
80  * Total number of currently running peers.
81  */
82 static unsigned long long peers_running;
83
84 /**
85  * Current round we are in.
86  */
87 static unsigned long long current_round;
88
89 /**
90  * Peers desired in the next round.
91  */
92 static unsigned long long peers_next_round;
93
94 /**
95  * Total number of connections in the whole network.
96  */
97 static unsigned int total_connections;
98
99 /**
100  * The currently running peer group.
101  */
102 static struct GNUNET_TESTING_PeerGroup *pg;
103
104 /**
105  * File to report results to.
106  */
107 static struct GNUNET_DISK_FileHandle *output_file;
108
109 /**
110  * File to log connection info, statistics to.
111  */
112 static struct GNUNET_DISK_FileHandle *data_file;
113
114 /**
115  * How many data points to capture before triggering next round?
116  */
117 static struct GNUNET_TIME_Relative wait_time;
118
119 /**
120  * Task called to disconnect peers.
121  */
122 static GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
123
124 /**
125  * Task called to shutdown test.
126  */
127 static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
128
129 /**
130  * Task used to churn the network.
131  */
132 static GNUNET_SCHEDULER_TaskIdentifier churn_task;
133
134 static char *topology_file;
135
136 static char *data_filename;
137
138 /**
139  * Check whether peers successfully shut down.
140  */
141 void
142 shutdown_callback (void *cls, const char *emsg)
143 {
144   if (emsg != NULL)
145     {
146 #if VERBOSE
147       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutdown of peers failed!\n");
148 #endif
149       if (ok == 0)
150         ok = 666;
151     }
152   else
153     {
154 #if VERBOSE
155       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
156                   "All peers successfully shut down!\n");
157 #endif
158       ok = 0;
159     }
160 }
161
162 static void
163 shutdown_task (void *cls,
164                const struct GNUNET_SCHEDULER_TaskContext *tc)
165 {
166   struct NSEPeer *pos;
167 #if VERBOSE
168   fprintf(stderr, "Ending test.\n");
169 #endif
170
171   if (disconnect_task != GNUNET_SCHEDULER_NO_TASK)
172     {
173       GNUNET_SCHEDULER_cancel(disconnect_task);
174       disconnect_task = GNUNET_SCHEDULER_NO_TASK;
175     }
176   while (NULL != (pos = peer_head))
177     {
178       if (pos->nse_handle != NULL)
179         GNUNET_NSE_disconnect(pos->nse_handle);
180       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
181       GNUNET_free(pos);
182     }
183
184   if (data_file != NULL)
185     GNUNET_DISK_file_close(data_file);
186   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
187 }
188
189 /**
190  * Callback to call when network size estimate is updated.
191  *
192  * @param cls closure
193  * @param estimate the value of the current network size estimate
194  * @param std_dev standard deviation (rounded down to nearest integer)
195  *                of the size estimation values seen
196  *
197  */
198 static void
199 handle_estimate (void *cls, double estimate, double std_dev)
200 {
201   struct NSEPeer *peer = cls;
202   char *output_buffer;
203   int size;
204   //fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
205   if (output_file != NULL)
206     {
207       size = GNUNET_asprintf(&output_buffer, "%s %u %f %f\n", GNUNET_i2s(&peer->daemon->id), peers_running, estimate, std_dev);
208       if (size != GNUNET_DISK_file_write(output_file, output_buffer, size))
209         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "%s: Unable to write to file!\n", "nse-profiler");
210     }
211   else
212     fprintf(stderr, "Received network size estimate from peer %s. Size: %f std.dev. %f\n", GNUNET_i2s(&peer->daemon->id), estimate, std_dev);
213
214 }
215
216 static void
217 connect_nse_service (void *cls,
218                      const struct GNUNET_SCHEDULER_TaskContext *tc)
219 {
220   struct NSEPeer *current_peer;
221   unsigned int i;
222 #if VERBOSE
223   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
224 #endif
225   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: connecting to nse service of peers\n");
226   for (i = 0; i < num_peers; i++)
227     {
228       current_peer = GNUNET_malloc(sizeof(struct NSEPeer));
229       current_peer->daemon = GNUNET_TESTING_daemon_get(pg, i);
230       if (GNUNET_YES == GNUNET_TESTING_daemon_running(GNUNET_TESTING_daemon_get(pg, i)))
231         {
232           current_peer->nse_handle = GNUNET_NSE_connect (current_peer->daemon->cfg, &handle_estimate, current_peer);
233           GNUNET_assert(current_peer->nse_handle != NULL);
234         }
235       GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer);
236     }
237 }
238
239 static void
240 churn_peers (void *cls,
241              const struct GNUNET_SCHEDULER_TaskContext *tc);
242
243 /**
244  * Continuation called by the "get_all" and "get" functions.
245  *
246  * @param cls struct StatsContext
247  * @param success GNUNET_OK if statistics were
248  *        successfully obtained, GNUNET_SYSERR if not.
249  */
250 static void stats_finished_callback (void *cls, int success)
251 {
252   struct StatsContext *stats_context = (struct StatsContext *)cls;
253   char *buf;
254   int buf_len;
255
256   if ((GNUNET_OK == success) && (data_file != NULL)) /* Stats lookup successful, write out data */
257     {
258       buf = NULL;
259       buf_len = GNUNET_asprintf(&buf, "TOTAL_NSE_BYTES: %u\n", stats_context->total_nse_bytes);
260       if (buf_len > 0)
261         {
262           GNUNET_DISK_file_write(data_file, buf, buf_len);
263         }
264       GNUNET_free_non_null(buf);
265     }
266
267   if (stats_context->task != NULL)
268     (*stats_context->task_id) = GNUNET_SCHEDULER_add_now(stats_context->task, stats_context->task_cls);
269
270   GNUNET_free(stats_context);
271 }
272
273 /**
274  * Callback function to process statistic values.
275  *
276  * @param cls struct StatsContext
277  * @param peer the peer the statistics belong to
278  * @param subsystem name of subsystem that created the statistic
279  * @param name the name of the datum
280  * @param value the current value
281  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
282  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
283  */
284 static int statistics_iterator (void *cls,
285                                 const struct GNUNET_PeerIdentity *peer,
286                                 const char *subsystem,
287                                 const char *name,
288                                 uint64_t value,
289                                 int is_persistent)
290 {
291   struct StatsContext *stats_context = (struct StatsContext *)cls;
292   char *buf;
293
294   GNUNET_assert(0 < GNUNET_asprintf(&buf, "bytes of messages of type %d received", GNUNET_MESSAGE_TYPE_NSE_P2P_FLOOD));
295   if ((0 == strstr(subsystem, "core")) && (0 == strstr(name, buf)))
296     {
297       stats_context->total_nse_bytes += value;
298     }
299   GNUNET_free(buf);
300   return GNUNET_OK;
301 }
302
303 /**
304  * @param cls struct StatsContext
305  * @param tc task context
306  */
307 static void
308 get_statistics (void *cls,
309                 const struct GNUNET_SCHEDULER_TaskContext *tc)
310 {
311   struct StatsContext *stats_context = (struct StatsContext *)cls;
312   GNUNET_TESTING_get_statistics(pg, &stats_finished_callback, &statistics_iterator, stats_context);
313 }
314
315 static void
316 disconnect_nse_peers (void *cls,
317                       const struct GNUNET_SCHEDULER_TaskContext *tc)
318 {
319   struct NSEPeer *pos;
320   char *buf;
321   struct StatsContext *stats_context;
322   disconnect_task = GNUNET_SCHEDULER_NO_TASK;
323   pos = peer_head;
324
325   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "TEST_NSE_MULTIPEER: disconnecting nse service of peers\n");
326   while (pos != NULL)
327     {
328       if (pos->nse_handle != NULL)
329         {
330           GNUNET_NSE_disconnect(pos->nse_handle);
331           pos->nse_handle = NULL;
332         }
333       pos = pos->next;
334     }
335
336   GNUNET_asprintf(&buf, "round%llu", current_round);
337   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", buf, &peers_next_round))
338     {
339       current_round++;
340       GNUNET_assert(churn_task == GNUNET_SCHEDULER_NO_TASK);
341       churn_task = GNUNET_SCHEDULER_add_now(&churn_peers, NULL);
342     }
343   else /* No more rounds, let's shut it down! */
344     {
345       stats_context = GNUNET_malloc(sizeof(struct StatsContext));
346       GNUNET_SCHEDULER_cancel(shutdown_handle);
347       shutdown_handle = GNUNET_SCHEDULER_NO_TASK;
348       stats_context->task = &shutdown_task;
349       stats_context->task_cls = NULL;
350       stats_context->task_id = &shutdown_handle;
351       GNUNET_SCHEDULER_add_now(&get_statistics, stats_context);
352       //shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
353     }
354   GNUNET_free(buf);
355 }
356
357 /**
358  * Prototype of a function that will be called when a
359  * particular operation was completed the testing library.
360  *
361  * @param cls unused
362  * @param emsg NULL on success
363  */
364 void topology_output_callback (void *cls, const char *emsg)
365 {
366   struct StatsContext *stats_context;
367   stats_context = GNUNET_malloc(sizeof(struct StatsContext));
368
369   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
370   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
371 }
372
373
374 /**
375  * Prototype of a function that will be called when a
376  * particular operation was completed the testing library.
377  *
378  * @param cls closure
379  * @param emsg NULL on success
380  */
381 static void
382 churn_callback (void *cls, const char *emsg)
383 {
384   char *temp_output_file;
385
386   if (emsg == NULL) /* Everything is okay! */
387     {
388       peers_running = GNUNET_TESTING_daemons_running(pg);
389       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
390                   "Round %lu, churn finished successfully.\n", current_round);
391       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
392       GNUNET_asprintf(&temp_output_file, "%s%lu.dot", topology_file, current_round);
393       GNUNET_TESTING_peergroup_topology_to_file(pg,
394                                                 temp_output_file,
395                                                 &topology_output_callback,
396                                                 NULL);
397       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Writing topology to file %s\n", temp_output_file);
398       GNUNET_free(temp_output_file);
399     }
400   else
401     {
402       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
403                   "Round %lu, churn FAILED!!\n", current_round);
404       GNUNET_SCHEDULER_cancel(shutdown_handle);
405       GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
406     }
407 }
408
409 static void
410 churn_peers (void *cls,
411                       const struct GNUNET_SCHEDULER_TaskContext *tc)
412 {
413   peers_running = GNUNET_TESTING_daemons_running(pg);
414   churn_task = GNUNET_SCHEDULER_NO_TASK;
415   if (peers_next_round == peers_running)
416     {
417       /* Nothing to do... */
418       GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
419       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
420       disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
421       GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Round %lu, doing nothing!\n", current_round);
422     }
423   else
424     {
425       if (peers_next_round > num_peers)
426         {
427           GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Asked to turn on more peers than have!!\n");
428           GNUNET_SCHEDULER_cancel(shutdown_handle);
429           GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
430         }
431       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
432                   "Round %lu, turning off %lu peers, turning on %lu peers!\n",
433                   current_round,
434                   (peers_running > peers_next_round) ? peers_running
435                       - peers_next_round : 0,
436                   (peers_next_round > peers_running) ? peers_next_round
437                       - peers_running : 0);
438       GNUNET_TESTING_daemons_churn (pg,
439                                     (peers_running > peers_next_round) ? peers_running
440                                         - peers_next_round
441                                         : 0,
442                                     (peers_next_round > peers_running) ? peers_next_round
443                                         - peers_running
444                                         : 0, wait_time, &churn_callback,
445                                     NULL);
446     }
447 }
448
449
450 static void
451 my_cb (void *cls,
452        const char *emsg)
453 {
454   char *buf;
455   int buf_len;
456   if (emsg != NULL)
457     {
458       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
459                   "Peergroup callback called with error, aborting test!\n");
460       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error from testing: `%s'\n");
461       ok = 1;
462       GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
463       return;
464     }
465 #if VERBOSE
466   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
467               "Peer Group started successfully, connecting to NSE service for each peer!\n");
468 #endif
469   GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Have %u connections\n", total_connections);
470   if (data_file != NULL)
471     {
472       buf = NULL;
473       buf_len = GNUNET_asprintf(&buf, "CONNECTIONS_0: %u\n", total_connections);
474       if (buf_len > 0)
475         GNUNET_DISK_file_write(data_file, buf, buf_len);
476       GNUNET_free_non_null(buf);
477     }
478   peers_running = GNUNET_TESTING_daemons_running(pg);
479   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
480   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
481 }
482
483 /**
484  * Prototype of a function that will be called whenever
485  * two daemons are connected by the testing library.
486  *
487  * @param cls closure
488  * @param first peer id for first daemon
489  * @param second peer id for the second daemon
490  * @param distance distance between the connected peers
491  * @param first_cfg config for the first daemon
492  * @param second_cfg config for the second daemon
493  * @param first_daemon handle for the first daemon
494  * @param second_daemon handle for the second daemon
495  * @param emsg error message (NULL on success)
496  */
497 void connect_cb (void *cls,
498                  const struct GNUNET_PeerIdentity *first,
499                  const struct GNUNET_PeerIdentity *second,
500                  uint32_t distance,
501                  const struct GNUNET_CONFIGURATION_Handle *first_cfg,
502                  const struct GNUNET_CONFIGURATION_Handle *second_cfg,
503                  struct GNUNET_TESTING_Daemon *first_daemon,
504                  struct GNUNET_TESTING_Daemon *second_daemon,
505                  const char *emsg)
506 {
507   char *second_id;
508
509   second_id = GNUNET_strdup(GNUNET_i2s(second));
510   if (emsg == NULL)
511     total_connections++;
512 }
513
514
515 static void
516 run (void *cls,
517      char *const *args,
518      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
519 {
520   char *temp_str;
521   unsigned long long temp_wait;
522   ok = 1;
523   testing_cfg = GNUNET_CONFIGURATION_create();
524   GNUNET_assert(GNUNET_OK == GNUNET_CONFIGURATION_load(testing_cfg, cfgfile));
525 #if VERBOSE
526   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n");
527   GNUNET_CONFIGURATION_set_value_string (testing_cfg, "testing",
528                                            "use_progressbars",
529                                            "YES");
530 #endif
531   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "testing", "num_peers", &num_peers))
532     {
533       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option TESTING:NUM_PEERS is required!\n");
534       return;
535     }
536
537   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, "nse-profiler", "wait_time", &temp_wait))
538     {
539       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:wait_time is required!\n");
540       return;
541     }
542
543   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "topology_output_file", &topology_file))
544     {
545       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:topology_output_file is required!\n");
546       return;
547     }
548
549   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, "nse-profiler", "data_output_file", &data_filename))
550     {
551       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option nse-profiler:data_output_file is required!\n");
552       return;
553     }
554
555
556   data_file = GNUNET_DISK_file_open (data_filename, GNUNET_DISK_OPEN_READWRITE
557                                                   | GNUNET_DISK_OPEN_CREATE,
558                                                   GNUNET_DISK_PERM_USER_READ |
559                                                   GNUNET_DISK_PERM_USER_WRITE);
560   if (data_file == NULL)
561     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", data_filename);
562   GNUNET_free(data_filename);
563
564   wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
565
566   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg, "nse-profiler", "output_file", &temp_str))
567     {
568       output_file = GNUNET_DISK_file_open (temp_str, GNUNET_DISK_OPEN_READWRITE
569                                                       | GNUNET_DISK_OPEN_CREATE,
570                                                       GNUNET_DISK_PERM_USER_READ |
571                                                       GNUNET_DISK_PERM_USER_WRITE);
572       if (output_file == NULL)
573         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to open %s for output!\n", temp_str);
574     }
575   GNUNET_free_non_null(temp_str);
576
577   pg = GNUNET_TESTING_peergroup_start(testing_cfg,
578                                       num_peers,
579                                       TIMEOUT,
580                                       &connect_cb,
581                                       &my_cb, NULL,
582                                       NULL);
583   GNUNET_assert (pg != NULL);
584   shutdown_handle = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever(), &shutdown_task, NULL);
585 }
586
587
588
589 /**
590  * nse-profiler command line options
591  */
592 static struct GNUNET_GETOPT_CommandLineOption options[] = {
593   {'V', "verbose", NULL,
594    gettext_noop ("be verbose (print progress information)"),
595    0, &GNUNET_GETOPT_set_one, &verbose},
596   GNUNET_GETOPT_OPTION_END
597 };
598
599 int
600 main (int argc, char *argv[])
601 {
602   int ret;
603
604   GNUNET_log_setup ("nse-profiler",
605 #if VERBOSE
606                     "DEBUG",
607 #else
608                     "WARNING",
609 #endif
610                     NULL);
611   ret = 1;
612   GNUNET_PROGRAM_run (argc,
613                       argv, "nse-profiler", gettext_noop
614                       ("Run a test of the NSE service."),
615                       options, &run, &ok);
616
617   GNUNET_DISK_directory_remove ("/tmp/nse-profiler");
618   return ret;
619 }
620
621 /* end of nse-profiler.c */