more client code
[oweals/gnunet.git] / src / nse / gnunet-nse-profiler.c
1 /*
2      This file is part of GNUnet.
3      (C) 2011 Christian Grothoff (and other contributing authors)
4
5      GNUnet is free software; you can redistribute it and/or modify
6      it under the terms of the GNU General Public License as published
7      by the Free Software Foundation; either version 3, or (at your
8      option) any later version.
9
10      GNUnet is distributed in the hope that it will be useful, but
11      WITHOUT ANY WARRANTY; without even the implied warranty of
12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13      General Public License for more details.
14
15      You should have received a copy of the GNU General Public License
16      along with GNUnet; see the file COPYING.  If not, write to the
17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18      Boston, MA 02111-1307, USA.
19 */
20 /**
21  * @file nse/nse-profiler.c
22  *
23  * @brief Profiling driver for the network size estimation service.
24  *        Generally, the profiler starts a given number of peers,
25  *        then churns some off, waits a certain amount of time, then
26  *        churns again, and repeats.
27  */
28 #include "platform.h"
29 #include "gnunet_testing_lib.h"
30 #include "gnunet_nse_service.h"
31
32 #define VERBOSE GNUNET_NO
33
34 struct NSEPeer
35 {
36   struct NSEPeer *prev;
37
38   struct NSEPeer *next;
39
40   struct GNUNET_TESTING_Daemon *daemon;
41
42   struct GNUNET_NSE_Handle *nse_handle;
43 };
44
45
46 struct StatsContext
47 {
48   unsigned long long total_nse_bytes;
49 };
50
51
52 static struct NSEPeer *peer_head;
53
54 static struct NSEPeer *peer_tail;
55
56 /**
57  * How long until we give up on connecting the peers?
58  */
59 #define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1500)
60
61 static int ok;
62
63 /**
64  * Be verbose
65  */
66 static int verbose;
67
68 /**
69  * Total number of peers in the test.
70  */
71 static unsigned long long num_peers;
72
73 /**
74  * Global configuration file
75  */
76 static struct GNUNET_CONFIGURATION_Handle *testing_cfg;
77
78 /**
79  * Total number of currently running peers.
80  */
81 static unsigned long long peers_running;
82
83 /**
84  * Current round we are in.
85  */
86 static unsigned long long current_round;
87
88 /**
89  * Peers desired in the next round.
90  */
91 static unsigned long long peers_next_round;
92
93 /**
94  * Maximum number of connections to NSE services.
95  */
96 static unsigned long long connection_limit;
97
98 /**
99  * Total number of connections in the whole network.
100  */
101 static unsigned int total_connections;
102
103 /**
104  * The currently running peer group.
105  */
106 static struct GNUNET_TESTING_PeerGroup *pg;
107
108 /**
109  * File to report results to.
110  */
111 static struct GNUNET_DISK_FileHandle *output_file;
112
113 /**
114  * File to log connection info, statistics to.
115  */
116 static struct GNUNET_DISK_FileHandle *data_file;
117
118 /**
119  * How many data points to capture before triggering next round?
120  */
121 static struct GNUNET_TIME_Relative wait_time;
122
123 /**
124  * Task called to disconnect peers.
125  */
126 static GNUNET_SCHEDULER_TaskIdentifier disconnect_task;
127
128 /**
129  * Task called to shutdown test.
130  */
131 static GNUNET_SCHEDULER_TaskIdentifier shutdown_handle;
132
133 /**
134  * Task used to churn the network.
135  */
136 static GNUNET_SCHEDULER_TaskIdentifier churn_task;
137
138 static char *topology_file;
139
140 static char *data_filename;
141
142 static uint64_t clock_skew;
143
144 /**
145  * Check whether peers successfully shut down.
146  */
147 static void
148 shutdown_callback (void *cls, const char *emsg)
149 {
150   if (emsg != NULL)
151     {
152 #if VERBOSE
153       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutdown of peers failed!\n");
154 #endif
155       if (ok == 0)
156         ok = 666;
157     }
158   else
159     {
160 #if VERBOSE
161       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
162                   "All peers successfully shut down!\n");
163 #endif
164       ok = 0;
165     }
166 }
167
168
169 static void
170 shutdown_task (void *cls,
171                const struct GNUNET_SCHEDULER_TaskContext *tc)
172 {
173   struct NSEPeer *pos;
174 #if VERBOSE
175   fprintf(stderr, "Ending test.\n");
176 #endif
177
178   if (disconnect_task != GNUNET_SCHEDULER_NO_TASK)
179     {
180       GNUNET_SCHEDULER_cancel(disconnect_task);
181       disconnect_task = GNUNET_SCHEDULER_NO_TASK;
182     }
183   while (NULL != (pos = peer_head))
184     {
185       if (pos->nse_handle != NULL)
186         GNUNET_NSE_disconnect(pos->nse_handle);
187       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
188       GNUNET_free(pos);
189     }
190
191   if (data_file != NULL)
192     GNUNET_DISK_file_close(data_file);
193   GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
194 }
195
196
197 /**
198  * Callback to call when network size estimate is updated.
199  *
200  * @param cls closure
201  * @param timestamp server timestamp
202  * @param estimate the value of the current network size estimate
203  * @param std_dev standard deviation (rounded down to nearest integer)
204  *                of the size estimation values seen
205  *
206  */
207 static void
208 handle_estimate (void *cls, struct GNUNET_TIME_Absolute timestamp, double estimate, double std_dev)
209 {
210   struct NSEPeer *peer = cls;
211   char *output_buffer;
212   size_t size;
213
214   if (output_file != NULL)
215     {
216       size = GNUNET_asprintf(&output_buffer, 
217                              "%s %llu %llu %f %f %f\n",
218                              GNUNET_i2s(&peer->daemon->id),
219                              peers_running,
220                              timestamp.abs_value,
221                              GNUNET_NSE_log_estimate_to_n(estimate),
222                              estimate, 
223                              std_dev);
224       if (size != GNUNET_DISK_file_write(output_file, output_buffer, size))
225         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
226                    "Unable to write to file!\n");
227       GNUNET_free (output_buffer);
228     }
229   else
230     fprintf(stderr,
231             "Received network size estimate from peer %s. Size: %f std.dev. %f\n", 
232             GNUNET_i2s(&peer->daemon->id),
233             estimate,
234             std_dev);
235
236 }
237
238
239 static void
240 connect_nse_service (void *cls,
241                      const struct GNUNET_SCHEDULER_TaskContext *tc)
242 {
243   struct NSEPeer *current_peer;
244   unsigned int i;
245
246 #if VERBOSE
247   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
248               "Connecting to nse service of peers\n");
249 #endif
250   for (i = 0; i < num_peers; i++)
251     {
252       if ((connection_limit > 0) && (i % (num_peers / connection_limit) != 0))
253         continue;
254 #if VERBOSE
255       GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "nse-profiler: connecting to nse service of peer %d\n", i);
256 #endif
257       current_peer = GNUNET_malloc(sizeof(struct NSEPeer));
258       current_peer->daemon = GNUNET_TESTING_daemon_get(pg, i);
259       if (GNUNET_YES == GNUNET_TESTING_daemon_running(GNUNET_TESTING_daemon_get(pg, i)))
260         {
261           current_peer->nse_handle = GNUNET_NSE_connect (current_peer->daemon->cfg,
262                                                          &handle_estimate, 
263                                                          current_peer);
264           GNUNET_assert(current_peer->nse_handle != NULL);
265         }
266       GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer);
267     }
268 }
269
270
271 static void
272 churn_peers (void *cls,
273              const struct GNUNET_SCHEDULER_TaskContext *tc);
274
275
276 /**
277  * Continuation called by the "get_all" and "get" functions.
278  *
279  * @param cls struct StatsContext
280  * @param success GNUNET_OK if statistics were
281  *        successfully obtained, GNUNET_SYSERR if not.
282  */
283 static void 
284 stats_finished_callback (void *cls, int success)
285 {
286   struct StatsContext *stats_context = cls;
287   char *buf;
288   int buf_len;
289
290   if ( (GNUNET_OK == success) && 
291        (data_file != NULL) )
292     {
293       /* Stats lookup successful, write out data */
294       buf = NULL;
295       buf_len = GNUNET_asprintf(&buf,
296                                 "TOTAL_NSE_BYTES: %u\n", 
297                                 stats_context->total_nse_bytes);
298       if (buf_len > 0)
299         {
300           GNUNET_DISK_file_write(data_file, buf, buf_len);
301         }
302       GNUNET_free_non_null(buf);
303     }
304
305   GNUNET_assert (GNUNET_SCHEDULER_NO_TASK == shutdown_handle);
306   shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
307   GNUNET_free(stats_context);
308 }
309
310
311 /**
312  * Callback function to process statistic values.
313  *
314  * @param cls struct StatsContext
315  * @param peer the peer the statistics belong to
316  * @param subsystem name of subsystem that created the statistic
317  * @param name the name of the datum
318  * @param value the current value
319  * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not
320  * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration
321  */
322 static int 
323 statistics_iterator (void *cls,
324                      const struct GNUNET_PeerIdentity *peer,
325                      const char *subsystem,
326                      const char *name,
327                      uint64_t value,
328                      int is_persistent)
329 {
330   struct StatsContext *stats_context = cls;
331
332   if ( (0 == strstr(subsystem, "nse")) && 
333        (0 == strstr(name, "# flood messages received")) )
334     stats_context->total_nse_bytes += value;
335   return GNUNET_OK;
336 }
337
338
339 static void
340 disconnect_nse_peers (void *cls,
341                       const struct GNUNET_SCHEDULER_TaskContext *tc)
342 {
343   struct NSEPeer *pos;
344   char *buf;
345   struct StatsContext *stats_context;
346
347   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
348               "disconnecting nse service of peers\n");
349   disconnect_task = GNUNET_SCHEDULER_NO_TASK;
350   pos = peer_head;
351   while (NULL != (pos = peer_head))
352     {
353       if (pos->nse_handle != NULL)
354         {
355           GNUNET_NSE_disconnect(pos->nse_handle);
356           pos->nse_handle = NULL;
357         }
358       GNUNET_CONTAINER_DLL_remove(peer_head, peer_tail, pos);
359       GNUNET_free(pos);
360     }
361
362   GNUNET_asprintf(&buf, 
363                   "round%llu", 
364                   current_round);
365   if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_number (testing_cfg, 
366                                                           "nse-profiler",
367                                                           buf, 
368                                                           &peers_next_round))
369     {
370       current_round++;
371       GNUNET_assert(churn_task == GNUNET_SCHEDULER_NO_TASK);
372       churn_task = GNUNET_SCHEDULER_add_now(&churn_peers, NULL);
373     }
374   else /* No more rounds, let's shut it down! */
375     {
376       stats_context = GNUNET_malloc(sizeof(struct StatsContext));
377       GNUNET_SCHEDULER_cancel(shutdown_handle);
378       shutdown_handle = GNUNET_SCHEDULER_NO_TASK;
379       GNUNET_TESTING_get_statistics(pg, 
380                                     &stats_finished_callback, 
381                                     &statistics_iterator, 
382                                     stats_context);
383     }
384   GNUNET_free(buf);
385 }
386
387
388 /**
389  * FIXME.
390  *
391  * @param cls unused
392  * @param emsg NULL on success
393  */
394 static void 
395 topology_output_callback (void *cls, const char *emsg)
396 {
397   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, 
398                                                  &disconnect_nse_peers, NULL);
399   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
400 }
401
402
403 /**
404  * FIXME.
405  *
406  * @param cls closure
407  * @param emsg NULL on success
408  */
409 static void
410 churn_callback (void *cls, const char *emsg)
411 {
412   char *temp_output_file;
413
414   if (emsg == NULL) /* Everything is okay! */
415     {
416       peers_running = peers_next_round;
417       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
418                   "Round %llu, churn finished successfully.\n",
419                   current_round);
420       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
421       GNUNET_asprintf(&temp_output_file, 
422                       "%s_%llu.dot",
423                       topology_file, 
424                       current_round);
425       GNUNET_TESTING_peergroup_topology_to_file(pg,
426                                                 temp_output_file,
427                                                 &topology_output_callback,
428                                                 NULL);
429       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
430                  "Writing topology to file %s\n",
431                  temp_output_file);
432       GNUNET_free(temp_output_file);
433     }
434   else
435     {
436       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
437                   "Round %llu, churn FAILED!!\n",
438                   current_round);
439       GNUNET_SCHEDULER_cancel(shutdown_handle);
440       shutdown_handle = GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
441     }
442 }
443
444
445 static void
446 churn_peers (void *cls,
447              const struct GNUNET_SCHEDULER_TaskContext *tc)
448 {
449   /* peers_running = GNUNET_TESTING_daemons_running(pg); */
450   churn_task = GNUNET_SCHEDULER_NO_TASK;
451   if (peers_next_round == peers_running)
452     {
453       /* Nothing to do... */
454       GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
455       GNUNET_assert(disconnect_task == GNUNET_SCHEDULER_NO_TASK);
456       disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, 
457                                                      &disconnect_nse_peers, NULL);
458       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
459                  "Round %lu, doing nothing!\n", 
460                  current_round);
461     }
462   else
463     {
464       if (peers_next_round > num_peers)
465         {
466           GNUNET_log(GNUNET_ERROR_TYPE_ERROR, 
467                      "Asked to turn on more peers than we have!!\n");
468           GNUNET_SCHEDULER_cancel(shutdown_handle);
469           GNUNET_SCHEDULER_add_now(&shutdown_task, NULL);
470         }
471       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
472                   "Round %llu, turning off %llu peers, turning on %llu peers!\n",
473                   current_round,
474                   (peers_running > peers_next_round) 
475                   ? peers_running - peers_next_round 
476                   : 0,
477                   (peers_next_round > peers_running) 
478                   ? peers_next_round - peers_running 
479                   : 0);
480       GNUNET_TESTING_daemons_churn (pg, "nse",
481                                     (peers_running > peers_next_round) 
482                                     ? peers_running - peers_next_round
483                                     : 0,
484                                     (peers_next_round > peers_running) 
485                                     ? peers_next_round - peers_running
486                                     : 0, 
487                                     wait_time,
488                                     &churn_callback, NULL);
489     }
490 }
491
492
493 static void
494 nse_started_cb(void *cls, const char *emsg)
495 {
496   GNUNET_SCHEDULER_add_now(&connect_nse_service, NULL);
497   disconnect_task = GNUNET_SCHEDULER_add_delayed(wait_time, &disconnect_nse_peers, NULL);
498 }
499
500
501 static void
502 my_cb (void *cls,
503        const char *emsg)
504 {
505   char *buf;
506   int buf_len;
507   if (emsg != NULL)
508     {
509       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
510                   "Peergroup callback called with error, aborting test!\n");
511       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Error from testing: `%s'\n");
512       ok = 1;
513       GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL);
514       return;
515     }
516 #if VERBOSE
517   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
518               "Peer Group started successfully, connecting to NSE service for each peer!\n");
519 #endif
520   GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
521               "Have %u connections\n",
522               total_connections);
523   if (data_file != NULL)
524     {
525       buf = NULL;
526       buf_len = GNUNET_asprintf(&buf, 
527                                 "CONNECTIONS_0: %u\n",
528                                 total_connections);
529       if (buf_len > 0)
530         GNUNET_DISK_file_write(data_file, buf, buf_len);
531       GNUNET_free (buf);
532     }
533   peers_running = GNUNET_TESTING_daemons_running(pg);
534   GNUNET_TESTING_daemons_start_service (pg,
535                                         "nse",
536                                         wait_time,
537                                         &nse_started_cb,
538                                         NULL);
539
540 }
541
542
543 /**
544  * Function that will be called whenever two daemons are connected by
545  * the testing library.
546  *
547  * @param cls closure
548  * @param first peer id for first daemon
549  * @param second peer id for the second daemon
550  * @param distance distance between the connected peers
551  * @param first_cfg config for the first daemon
552  * @param second_cfg config for the second daemon
553  * @param first_daemon handle for the first daemon
554  * @param second_daemon handle for the second daemon
555  * @param emsg error message (NULL on success)
556  */
557 static void 
558 connect_cb (void *cls,
559             const struct GNUNET_PeerIdentity *first,
560             const struct GNUNET_PeerIdentity *second,
561             uint32_t distance,
562             const struct GNUNET_CONFIGURATION_Handle *first_cfg,
563             const struct GNUNET_CONFIGURATION_Handle *second_cfg,
564             struct GNUNET_TESTING_Daemon *first_daemon,
565             struct GNUNET_TESTING_Daemon *second_daemon,
566             const char *emsg)
567 {
568   if (emsg == NULL)
569     total_connections++;
570 }
571
572
573 static void
574 run (void *cls,
575      char *const *args,
576      const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg)
577 {
578   char *temp_str;
579   unsigned long long temp_wait;
580   struct GNUNET_TESTING_Host *hosts;
581
582   ok = 1;
583   testing_cfg = GNUNET_CONFIGURATION_create();
584 #if VERBOSE
585   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n");
586   GNUNET_CONFIGURATION_set_value_string (testing_cfg,
587                                          "testing",
588                                          "use_progressbars",
589                                          "YES");
590 #endif
591   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, 
592                                                           "testing",
593                                                           "num_peers", &num_peers))
594     {
595       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, "Option TESTING:NUM_PEERS is required!\n");
596       return;
597     }
598
599   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, 
600                                                           "nse-profiler", 
601                                                           "wait_time",
602                                                           &temp_wait))
603     {
604       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, 
605                  "Option nse-profiler:wait_time is required!\n");
606       return;
607     }
608
609   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_number (testing_cfg, 
610                                                           "nse-profiler", "connection_limit",
611                                                           &connection_limit))
612     {
613       connection_limit = 0;
614     }
615
616   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, 
617                                                           "nse-profiler", "topology_output_file", 
618                                                           &topology_file))
619     {
620       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, 
621                  "Option nse-profiler:topology_output_file is required!\n");
622       return;
623     }
624
625   if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (testing_cfg, 
626                                                           "nse-profiler", "data_output_file",
627                                                           &data_filename))
628     {
629       GNUNET_log(GNUNET_ERROR_TYPE_ERROR, 
630                  "Option nse-profiler:data_output_file is required!\n");
631       return;
632     }
633
634   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno (testing_cfg, 
635                                                           "nse-profiler", 
636                                                           "skew_clock"))
637     {
638       GNUNET_log(GNUNET_ERROR_TYPE_WARNING,
639                  "Setting our clock as skewed...\n");
640       clock_skew = GNUNET_CRYPTO_random_u64(GNUNET_CRYPTO_QUALITY_WEAK, 
641                                             GNUNET_TIME_UNIT_MINUTES.rel_value);
642     }
643
644
645   data_file = GNUNET_DISK_file_open (data_filename,
646                                      GNUNET_DISK_OPEN_READWRITE
647                                      | GNUNET_DISK_OPEN_CREATE,
648                                      GNUNET_DISK_PERM_USER_READ |
649                                      GNUNET_DISK_PERM_USER_WRITE);
650   if (data_file == NULL)
651     GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
652                "Failed to open %s for output!\n", 
653                data_filename);
654   GNUNET_free(data_filename);
655
656   wait_time = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_wait);
657
658   if (GNUNET_YES == GNUNET_CONFIGURATION_get_value_string(cfg,
659                                                           "nse-profiler", 
660                                                           "output_file",
661                                                           &temp_str))
662     {
663       output_file = GNUNET_DISK_file_open (temp_str, GNUNET_DISK_OPEN_READWRITE
664                                                       | GNUNET_DISK_OPEN_CREATE,
665                                                       GNUNET_DISK_PERM_USER_READ |
666                                                       GNUNET_DISK_PERM_USER_WRITE);
667       if (output_file == NULL)
668         GNUNET_log(GNUNET_ERROR_TYPE_WARNING, 
669                    "Failed to open %s for output!\n",
670                    temp_str);
671     }
672   GNUNET_free_non_null(temp_str);
673
674   hosts = GNUNET_TESTING_hosts_load (testing_cfg);
675
676   pg = GNUNET_TESTING_peergroup_start(testing_cfg,
677                                       num_peers,
678                                       TIMEOUT,
679                                       &connect_cb,
680                                       &my_cb, NULL,
681                                       hosts);
682   GNUNET_assert (pg != NULL);
683   shutdown_handle = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_get_forever(),
684                                                   &shutdown_task,
685                                                   NULL);
686 }
687
688
689
690 /**
691  * nse-profiler command line options
692  */
693 static struct GNUNET_GETOPT_CommandLineOption options[] = {
694   {'V', "verbose", NULL,
695    gettext_noop ("be verbose (print progress information)"),
696    0, &GNUNET_GETOPT_set_one, &verbose},
697   GNUNET_GETOPT_OPTION_END
698 };
699
700
701 int
702 main (int argc, char *argv[])
703 {
704   GNUNET_log_setup ("nse-profiler",
705 #if VERBOSE
706                     "DEBUG",
707 #else
708                     "WARNING",
709 #endif
710                     NULL);
711   GNUNET_PROGRAM_run (argc,
712                       argv, "nse-profiler", 
713                       gettext_noop ("Measure quality and performance of the NSE service."),
714                       options, 
715                       &run, NULL);
716 #if REMOVE_DIR
717   GNUNET_DISK_directory_remove ("/tmp/nse-profiler");
718 #endif
719   return ok;
720 }
721
722 /* end of nse-profiler.c */