X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fnse%2Fgnunet-nse-profiler.c;h=b8fa293d58de6faf86dcf5095a660b044eac89a9;hb=80d2de6cdc4d253c7fbc6a4bc067d856aab9cca9;hp=eae5e96700c5a5c9b7f4d9ce90a022d67571d0cf;hpb=f7d297d8be140f6507001a509e69bf15d2ea727a;p=oweals%2Fgnunet.git diff --git a/src/nse/gnunet-nse-profiler.c b/src/nse/gnunet-nse-profiler.c index eae5e9670..b8fa293d5 100644 --- a/src/nse/gnunet-nse-profiler.c +++ b/src/nse/gnunet-nse-profiler.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2011, 2012 Christian Grothoff (and other contributing authors) + Copyright (C) 2011, 2012 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -14,8 +14,8 @@ You should have received a copy of the GNU General Public License along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 59 Temple Place - Suite 330, - Boston, MA 02111-1307, USA. + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. */ /** * @file nse/gnunet-nse-profiler.c @@ -24,11 +24,11 @@ * Generally, the profiler starts a given number of peers, * then churns some off, waits a certain amount of time, then * churns again, and repeats. - * - * TODO: - * - need to check for leaks (especially FD leaks) - * - need to TEST + * @author Christian Grothoff + * @author Nathan Evans + * @author Sree Harsha Totakura */ + #include "platform.h" #include "gnunet_testbed_service.h" #include "gnunet_nse_service.h" @@ -71,39 +71,15 @@ struct NSEPeer */ struct GNUNET_TESTBED_Operation *nse_op; -}; - - -/** - * Context for the stats task? - */ -struct StatsContext -{ - /** - * How many messages have peers received during the test. + * Testbed operation to connect to statistics service */ - unsigned long long total_nse_received_messages; + struct GNUNET_TESTBED_Operation *stat_op; /** - * How many messages have peers send during the test (should be == received). + * Handle to the statistics service */ - unsigned long long total_nse_transmitted_messages; - - /** - * How many messages have travelled an edge in both directions. - */ - unsigned long long total_nse_cross; - - /** - * How many extra messages per edge (corrections) have been received. - */ - unsigned long long total_nse_extra; - - /** - * How many messages have been discarded. - */ - unsigned long long total_discarded; + struct GNUNET_STATISTICS_Handle *sh; }; @@ -127,6 +103,11 @@ struct OpListEntry */ struct GNUNET_TESTBED_Operation *op; + /** + * Depending on whether we start or stop NSE service at the peer set this to 1 + * or -1 + */ + int delta; }; @@ -152,7 +133,7 @@ static int verbose; /** * Name of the file with the hosts to run the test over (configuration option) - */ + */ static char *hosts_file; /** @@ -198,7 +179,7 @@ static struct GNUNET_CONFIGURATION_Handle *testing_cfg; /** * The shutdown task */ -static GNUNET_SCHEDULER_TaskIdentifier shutdown_task_id; +static struct GNUNET_SCHEDULER_Task * shutdown_task_id; /** * Maximum number of connections to NSE services. @@ -246,11 +227,6 @@ static struct OpListEntry *oplist_head; */ static struct OpListEntry *oplist_tail; -/** - * The get stats operation - */ -static struct GNUNET_TESTBED_Operation *get_stats_op; - /** * Are we shutting down */ @@ -271,6 +247,8 @@ close_monitor_connections () { if (NULL != pos->nse_op) GNUNET_TESTBED_operation_done (pos->nse_op); + if (NULL != pos->stat_op) + GNUNET_TESTBED_operation_done (pos->stat_op); GNUNET_CONTAINER_DLL_remove (peer_head, peer_tail, pos); GNUNET_free (pos); } @@ -292,20 +270,22 @@ close_monitor_connections () static void shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - shutdown_task_id = GNUNET_SCHEDULER_NO_TASK; + shutdown_task_id = NULL; if (GNUNET_YES == shutting_down) return; shutting_down = GNUNET_YES; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Ending test.\n"); + LOG_DEBUG ("Ending test.\n"); close_monitor_connections (); - if (NULL != get_stats_op) - { - GNUNET_TESTBED_operation_done (get_stats_op); - get_stats_op = NULL; - } - // FIXME: what about closing other files!? if (NULL != data_file) + { GNUNET_DISK_file_close (data_file); + data_file = NULL; + } + if (NULL != output_file) + { + GNUNET_DISK_file_close (output_file); + output_file = NULL; + } if (NULL != testing_cfg) GNUNET_CONFIGURATION_destroy (testing_cfg); testing_cfg = NULL; @@ -318,7 +298,7 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) static void shutdown_now () { - if (GNUNET_SCHEDULER_NO_TASK != shutdown_task_id) + if (NULL != shutdown_task_id) GNUNET_SCHEDULER_cancel (shutdown_task_id); shutdown_task_id = GNUNET_SCHEDULER_add_now (&shutdown_task, NULL); } @@ -332,10 +312,9 @@ shutdown_now () * @param estimate the value of the current network size estimate * @param std_dev standard deviation (rounded down to nearest integer) * of the size estimation values seen - * */ static void -handle_estimate (void *cls, +handle_estimate (void *cls, struct GNUNET_TIME_Absolute timestamp, double estimate, double std_dev) { @@ -350,11 +329,11 @@ handle_estimate (void *cls, peer, estimate, std_dev); return; } - size = GNUNET_snprintf (output_buffer, + size = GNUNET_snprintf (output_buffer, sizeof (output_buffer), "%p %llu %llu %f %f %f\n", peer, peers_running, - timestamp.abs_value, + (unsigned long long) timestamp.abs_value_us, GNUNET_NSE_log_estimate_to_n (estimate), estimate, std_dev); if (size != GNUNET_DISK_file_write (output_file, output_buffer, size)) @@ -366,7 +345,7 @@ handle_estimate (void *cls, /** * Adapter function called to establish a connection to * NSE service. - * + * * @param cls closure (the 'struct NSEPeer') * @param cfg configuration of the peer to connect to; will be available until * GNUNET_TESTBED_operation_done() is called on the operation returned @@ -386,11 +365,11 @@ nse_connect_adapter (void *cls, /** * Adapter function called to destroy a connection to * NSE service. - * + * * @param cls closure * @param op_result service handle returned from the connect adapter */ -static void +static void nse_disconnect_adapter (void *cls, void *op_result) { @@ -398,6 +377,119 @@ nse_disconnect_adapter (void *cls, } +/** + * Callback function to process statistic values. + * + * @param cls `struct NSEPeer` + * @param subsystem name of subsystem that created the statistic + * @param name the name of the datum + * @param value the current value + * @param is_persistent #GNUNET_YES if the value is persistent, #GNUNET_NO if not + * @return #GNUNET_OK to continue, #GNUNET_SYSERR to abort iteration + */ +static int +stat_iterator (void *cls, + const char *subsystem, + const char *name, + uint64_t value, int is_persistent) +{ + char *output_buffer; + struct GNUNET_TIME_Absolute now; + size_t size; + unsigned int flag; + + GNUNET_assert (NULL != data_file); + now = GNUNET_TIME_absolute_get (); + flag = strcasecmp (subsystem, "core"); + if (0 != flag) + flag = 1; + size = GNUNET_asprintf (&output_buffer, "%llu %llu %u\n", + now.abs_value_us / 1000LL / 1000LL, + value, flag); + if (size != GNUNET_DISK_file_write (data_file, output_buffer, size)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Unable to write to file!\n"); + GNUNET_free (output_buffer); + return GNUNET_SYSERR; + } + GNUNET_free (output_buffer); + return GNUNET_OK; +} + + +/** + * Called to open a connection to the peer's statistics + * + * @param cls peer context + * @param cfg configuration of the peer to connect to; will be available until + * GNUNET_TESTBED_operation_done() is called on the operation returned + * from GNUNET_TESTBED_service_connect() + * @return service handle to return in 'op_result', NULL on error + */ +static void * +stat_connect_adapter (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct NSEPeer *peer = cls; + + peer->sh = GNUNET_STATISTICS_create ("nse-profiler", cfg); + return peer->sh; +} + + +/** + * Called to disconnect from peer's statistics service + * + * @param cls peer context + * @param op_result service handle returned from the connect adapter + */ +static void +stat_disconnect_adapter (void *cls, void *op_result) +{ + struct NSEPeer *peer = cls; + + GNUNET_break (GNUNET_OK == GNUNET_STATISTICS_watch_cancel + (peer->sh, "core", "# peers connected", + stat_iterator, peer)); + GNUNET_break (GNUNET_OK == GNUNET_STATISTICS_watch_cancel + (peer->sh, "nse", "# peers connected", + stat_iterator, peer)); + GNUNET_STATISTICS_destroy (op_result, GNUNET_NO); + peer->sh = NULL; +} + + +/** + * Called after successfully opening a connection to a peer's statistics + * service; we register statistics monitoring for CORE and NSE here. + * + * @param cls the callback closure from functions generating an operation + * @param op the operation that has been finished + * @param ca_result the service handle returned from GNUNET_TESTBED_ConnectAdapter() + * @param emsg error message in case the operation has failed; will be NULL if + * operation has executed successfully. + */ +static void +stat_comp_cb (void *cls, struct GNUNET_TESTBED_Operation *op, + void *ca_result, const char *emsg ) +{ + struct GNUNET_STATISTICS_Handle *sh = ca_result; + struct NSEPeer *peer = cls; + + if (NULL != emsg) + { + GNUNET_break (0); + return; + } + GNUNET_break (GNUNET_OK == GNUNET_STATISTICS_watch + (sh, "core", "# peers connected", + stat_iterator, peer)); + GNUNET_break (GNUNET_OK == GNUNET_STATISTICS_watch + (sh, "nse", "# peers connected", + stat_iterator, peer)); +} + + /** * Task run to connect to the NSE and statistics services to a subset of * all of the running peers. @@ -410,19 +502,18 @@ connect_nse_service () unsigned int connections; if (0 == connection_limit) - return; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Connecting to nse service of peers\n"); + return; + LOG_DEBUG ("Connecting to nse service of peers\n"); connections = 0; for (i = 0; i < num_peers_in_round[current_round]; i++) { - if ((num_peers_in_round[current_round] > connection_limit) && + if ((num_peers_in_round[current_round] > connection_limit) && (0 != (i % (num_peers_in_round[current_round] / connection_limit)))) continue; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "nse-profiler: connecting to nse service of peer %d\n", i); - current_peer = GNUNET_malloc (sizeof (struct NSEPeer)); + LOG_DEBUG ("Connecting to nse service of peer %d\n", i); + current_peer = GNUNET_new (struct NSEPeer); current_peer->daemon = daemons[i]; - current_peer->nse_op + current_peer->nse_op = GNUNET_TESTBED_service_connect (NULL, current_peer->daemon, "nse", @@ -430,6 +521,16 @@ connect_nse_service () &nse_connect_adapter, &nse_disconnect_adapter, current_peer); + if (NULL != data_file) + current_peer->stat_op + = GNUNET_TESTBED_service_connect (NULL, + current_peer->daemon, + "statistics", + stat_comp_cb, + current_peer, + &stat_connect_adapter, + &stat_disconnect_adapter, + current_peer); GNUNET_CONTAINER_DLL_insert (peer_head, peer_tail, current_peer); if (++connections == connection_limit) break; @@ -444,139 +545,10 @@ connect_nse_service () * @param tc scheduler context (unused) */ static void -next_round (void *cls, +next_round (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); -/** - * Continuation called by the "get_all" and "get" functions at the - * end of a round. Obtains the final statistics and writes them to - * the file, then either starts the next round, or, if this was the - * last round, terminates the run. - * - * @param cls struct StatsContext - * @param op operation handle - * @param emsg error message, NULL on success - */ -static void -stats_finished_callback (void *cls, - struct GNUNET_TESTBED_Operation *op, - const char *emsg) -{ - struct StatsContext *stats_context = cls; - char buf[512]; - size_t buf_len; - - GNUNET_TESTBED_operation_done (get_stats_op); - get_stats_op = NULL; - if (NULL != emsg) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to get statistics: %s\n", - emsg); - GNUNET_SCHEDULER_shutdown (); - GNUNET_free (stats_context); - return; - } - LOG_DEBUG ("Finished collecting statistics\n"); - if (NULL != data_file) - { - /* Stats lookup successful, write out data */ - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "TOTAL_NSE_RECEIVED_MESSAGES_%u: %u \n", - current_round, - stats_context->total_nse_received_messages); - GNUNET_DISK_file_write (data_file, buf, buf_len); - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "TOTAL_NSE_TRANSMITTED_MESSAGES_%u: %u\n", - current_round, - stats_context->total_nse_transmitted_messages); - GNUNET_DISK_file_write (data_file, buf, buf_len); - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "TOTAL_NSE_CROSS_%u: %u \n", - current_round, - stats_context->total_nse_cross); - GNUNET_DISK_file_write (data_file, buf, buf_len); - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "TOTAL_NSE_EXTRA_%u: %u \n", - current_round, - stats_context->total_nse_extra); - GNUNET_DISK_file_write (data_file, buf, buf_len); - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "TOTAL_NSE_DISCARDED_%u: %u \n", - current_round, - stats_context->total_discarded); - GNUNET_DISK_file_write (data_file, buf, buf_len); - } - GNUNET_SCHEDULER_add_now (&next_round, NULL); - GNUNET_free (stats_context); -} - - -/** - * Callback function to process statistic values. - * - * @param cls struct StatsContext - * @param peer the peer the statistics belong to - * @param subsystem name of subsystem that created the statistic - * @param name the name of the datum - * @param value the current value - * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not - * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration - */ -static int -statistics_iterator (void *cls, - const struct GNUNET_TESTBED_Peer *peer, - const char *subsystem, const char *name, uint64_t value, - int is_persistent) -{ - struct StatsContext *stats_context = cls; - char buf[512]; - size_t buf_len; - - if (0 != strcasecmp (subsystem, "nse")) - return GNUNET_SYSERR; - if (0 == strcmp (name, "# flood messages received")) - { - stats_context->total_nse_received_messages += value; - if ( (verbose > 1) && - (NULL != data_file) ) - { - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "%p %u RECEIVED\n", - peer, value); - GNUNET_DISK_file_write (data_file, buf, buf_len); - } - } - if (0 == strcmp (name, "# flood messages transmitted")) - { - stats_context->total_nse_transmitted_messages += value; - if ( (verbose > 1) && - (NULL != data_file) ) - { - buf_len = - GNUNET_snprintf (buf, sizeof (buf), - "%p %u TRANSMITTED\n", - peer, value); - GNUNET_DISK_file_write (data_file, buf, buf_len); - } - } - if (0 == strcmp (name, "# cross messages")) - stats_context->total_nse_cross += value; - if (0 == strcmp (name, "# extra messages")) - stats_context->total_nse_extra += value; - if (0 == strcmp (name, "# flood messages discarded (clock skew too large)")) - stats_context->total_discarded += value; - return GNUNET_OK; -} - - /** * We're at the end of a round. Stop monitoring, write total * number of connections to log and get full stats. Then trigger @@ -586,32 +558,14 @@ statistics_iterator (void *cls, * @param tc unused */ static void -finish_round (void *cls, +finish_round (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - struct StatsContext *stats_context; - char buf[1024]; - size_t buf_len; - if (0 != (GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason)) return; LOG (GNUNET_ERROR_TYPE_INFO, "Have %u connections\n", total_connections); - if (NULL != data_file) - { - buf_len = GNUNET_snprintf (buf, sizeof (buf), - "CONNECTIONS_0: %u\n", - total_connections); - GNUNET_DISK_file_write (data_file, buf, buf_len); - } - close_monitor_connections (); - stats_context = GNUNET_malloc (sizeof (struct StatsContext)); - get_stats_op = - GNUNET_TESTBED_get_statistics (num_peers_in_round[current_round], - daemons, - "nse", NULL, - &statistics_iterator, - &stats_finished_callback, - stats_context); + close_monitor_connections (); + GNUNET_SCHEDULER_add_now (&next_round, NULL); } @@ -639,25 +593,34 @@ make_oplist_entry () { struct OpListEntry *entry; - entry = GNUNET_malloc (sizeof (struct OpListEntry)); + entry = GNUNET_new (struct OpListEntry); GNUNET_CONTAINER_DLL_insert_tail (oplist_head, oplist_tail, entry); return entry; } /** - * Functions of this signature are called when a peer has been successfully - * started or stopped. + * Callback to be called when NSE service is started or stopped at peers * * @param cls NULL + * @param op the operation handle * @param emsg NULL on success; otherwise an error description */ -static void -peer_churn_cb (void *cls, const char *emsg) +static void +manage_service_cb (void *cls, struct GNUNET_TESTBED_Operation *op, + const char *emsg) { struct OpListEntry *entry = cls; - + GNUNET_TESTBED_operation_done (entry->op); + if (NULL != emsg) + { + LOG (GNUNET_ERROR_TYPE_ERROR, "Failed to start/stop NSE at a peer\n"); + GNUNET_SCHEDULER_shutdown (); + return; + } + GNUNET_assert (0 != entry->delta); + peers_running += entry->delta; GNUNET_CONTAINER_DLL_remove (oplist_head, oplist_tail, entry); GNUNET_free (entry); if (num_peers_in_round[current_round] == peers_running) @@ -668,9 +631,6 @@ peer_churn_cb (void *cls, const char *emsg) /** * Adjust the number of running peers to match the required number of running * peers for the round - * - * @param - * @return */ static void adjust_running_peers () @@ -682,15 +642,25 @@ adjust_running_peers () for (i=peers_running;iop = GNUNET_TESTBED_peer_start (NULL, daemons[i], - &peer_churn_cb, entry); + entry->delta = 1; + entry->op = GNUNET_TESTBED_peer_manage_service (NULL, + daemons[i], + "nse", + &manage_service_cb, + entry, + 1); } /* stop peers if we have too many */ for (i=num_peers_in_round[current_round];iop = GNUNET_TESTBED_peer_stop (NULL, daemons[i], - &peer_churn_cb, entry); + entry->delta = -1; + entry->op = GNUNET_TESTBED_peer_manage_service (NULL, + daemons[i], + "nse", + &manage_service_cb, + entry, + 0); } } @@ -703,13 +673,13 @@ adjust_running_peers () * @param tc unused */ static void -next_round (void *cls, +next_round (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { if (0 != (GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason)) return; - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "disconnecting nse service of peers\n"); - current_round++; + LOG_DEBUG ("Disconnecting nse service of peers\n"); + current_round++; if (current_round == num_rounds) { /* this was the last round, terminate */ @@ -735,17 +705,11 @@ next_round (void *cls, * @param event information on what is happening */ static void -master_controller_cb (void *cls, +master_controller_cb (void *cls, const struct GNUNET_TESTBED_EventInformation *event) { switch (event->type) { - case GNUNET_TESTBED_ET_PEER_START: - peers_running++; - break; - case GNUNET_TESTBED_ET_PEER_STOP: - peers_running--; - break; case GNUNET_TESTBED_ET_CONNECT: total_connections++; break; @@ -762,14 +726,22 @@ master_controller_cb (void *cls, * Signature of a main function for a testcase. * * @param cls NULL + * @param h the run handle * @param num_peers_ number of peers in 'peers' * @param peers handle to peers run in the testbed. NULL upon timeout (see * GNUNET_TESTBED_test_run()). + * @param links_succeeded the number of overlay link connection attempts that + * succeeded + * @param links_failed the number of overlay link connection attempts that + * failed */ -static void +static void test_master (void *cls, + struct GNUNET_TESTBED_RunHandle *h, unsigned int num_peers_, - struct GNUNET_TESTBED_Peer **peers) + struct GNUNET_TESTBED_Peer **peers, + unsigned int links_succeeded, + unsigned int links_failed) { if (NULL == peers) { @@ -803,11 +775,11 @@ run (void *cls, char *const *args, const char *cfgfile, { char *tok; uint64_t event_mask; - unsigned int num; + unsigned int num; ok = 1; testing_cfg = GNUNET_CONFIGURATION_dup (cfg); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting daemons.\n"); + LOG_DEBUG ("Starting daemons.\n"); if (NULL == num_peer_spec) { fprintf (stderr, "You need to specify the number of peers to run\n"); @@ -834,7 +806,7 @@ run (void *cls, char *const *args, const char *cfgfile, return; } if ( (NULL != data_filename) && - (NULL == (data_file = + (NULL == (data_file = GNUNET_DISK_file_open (data_filename, GNUNET_DISK_OPEN_READWRITE | GNUNET_DISK_OPEN_TRUNCATE | @@ -867,7 +839,7 @@ run (void *cls, char *const *args, const char *cfgfile, NULL, /* master_controller_cb cls */ &test_master, NULL); /* test_master cls */ - shutdown_task_id = + shutdown_task_id = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_FOREVER_REL, &shutdown_task, NULL); }