From d6e728b2b869ee521216deae75a767165d3e1001 Mon Sep 17 00:00:00 2001 From: Supriti Singh Date: Thu, 7 Aug 2014 14:17:48 +0000 Subject: [PATCH] - Collect bandwidth statistics in profiler - Get average put/get length in profiler - Check for GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE,instead of GNUNET_SERVER_MAX_MESSAGE_SIZE --- src/dht/gnunet-service-xdht.c | 2 +- src/dht/gnunet-service-xdht_neighbours.c | 111 ++++++++++--- src/dht/gnunet_dht_profiler.c | 190 +++++++++-------------- 3 files changed, 165 insertions(+), 138 deletions(-) diff --git a/src/dht/gnunet-service-xdht.c b/src/dht/gnunet-service-xdht.c index 3f44c2a69..61f534988 100644 --- a/src/dht/gnunet-service-xdht.c +++ b/src/dht/gnunet-service-xdht.c @@ -168,7 +168,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, GDS_block_context = GNUNET_BLOCK_context_create (GDS_cfg); GDS_stats = GNUNET_STATISTICS_create ("dht", GDS_cfg); GDS_ROUTING_init (); - GDS_NSE_init (); //FIXME:not used in x0vine, remove + GDS_NSE_init (); GDS_DATACACHE_init (); GDS_HELLO_init (); GDS_CLIENTS_init (server); diff --git a/src/dht/gnunet-service-xdht_neighbours.c b/src/dht/gnunet-service-xdht_neighbours.c index 3f280134e..139d1971b 100644 --- a/src/dht/gnunet-service-xdht_neighbours.c +++ b/src/dht/gnunet-service-xdht_neighbours.c @@ -936,10 +936,10 @@ core_transmit_notify (void *cls, size_t size, void *buf) while ((NULL != (pending = peer->head)) && (size - off >= (msize = ntohs (pending->msg->size)))) { - /*GNUNET_STATISTICS_update (GDS_stats, + GNUNET_STATISTICS_update (GDS_stats, gettext_noop ("# Bytes transmitted to other peers"), msize, - GNUNET_NO);*/ + GNUNET_NO); memcpy (&cbuf[off], pending->msg, msize); off += msize; peer->pending_count--; @@ -978,11 +978,7 @@ process_friend_queue (struct FriendInfo *peer) { return; } - GNUNET_STATISTICS_update (GDS_stats, - gettext_noop - ("# Bytes of bandwidth requested from core"), - ntohs (pending->msg->size), GNUNET_NO); - + peer->th = GNUNET_CORE_notify_transmit_ready (core_api, GNUNET_NO, pending->importance, @@ -1030,7 +1026,7 @@ GDS_NEIGHBOURS_send_trail_setup (struct GNUNET_PeerIdentity source_peer, msize = sizeof (struct PeerTrailSetupMessage) + (trail_length * sizeof (struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1099,7 +1095,7 @@ GDS_NEIGHBOURS_send_trail_setup_result (struct GNUNET_PeerIdentity querying_peer msize = sizeof (struct PeerTrailSetupResultMessage) + (trail_length * sizeof (struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1171,7 +1167,7 @@ GDS_NEIGHBOURS_send_trail_rejection (struct GNUNET_PeerIdentity source_peer, msize = sizeof (struct PeerTrailRejectionMessage) + (trail_length * sizeof (struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1237,7 +1233,7 @@ GDS_NEIGHBOURS_send_verify_successor_message (struct GNUNET_PeerIdentity source_ msize = sizeof (struct PeerVerifySuccessorMessage) + (trail_length * sizeof (struct GNUNET_PeerIdentity));; - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1291,7 +1287,7 @@ GDS_NEIGHBOURS_send_trail_teardown (struct GNUNET_HashCode trail_id, msize = sizeof (struct PeerTrailTearDownMessage); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1358,7 +1354,7 @@ GDS_NEIGHBOURS_send_verify_successor_result (struct GNUNET_PeerIdentity querying msize = sizeof (struct PeerVerifySuccessorResultMessage) + (trail_length * sizeof(struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1420,7 +1416,7 @@ GDS_NEIGHBOURS_send_notify_new_successor (struct GNUNET_PeerIdentity source_peer msize = sizeof (struct PeerNotifyNewSuccessorMessage) + (successor_trail_length * sizeof(struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1481,7 +1477,7 @@ GDS_NEIGHBOURS_send_add_trail (struct GNUNET_PeerIdentity source_peer, msize = sizeof (struct PeerAddTrailMessage) + (trail_length * sizeof(struct GNUNET_PeerIdentity)); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -1533,7 +1529,7 @@ GDS_NEIGHBOURS_send_trail_compression (struct GNUNET_PeerIdentity source_peer, msize = sizeof (struct PeerTrailCompressionMessage); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -2113,13 +2109,13 @@ GDS_NEIGHBOURS_send_put (const struct GNUNET_HashCode *key, msize = put_path_length * sizeof (struct GNUNET_PeerIdentity) + data_size + sizeof (struct PeerPutMessage); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { put_path_length = 0; msize = data_size + sizeof (struct PeerPutMessage); } - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -2227,7 +2223,7 @@ GDS_NEIGHBOURS_send_get (const struct GNUNET_HashCode *key, /* In this case we don't make get_path_length = 0, as we need get path to * return the message back to querying client. */ - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { GNUNET_break (0); return; @@ -2333,12 +2329,19 @@ GDS_NEIGHBOURS_send_get_result (const struct GNUNET_HashCode *key, data_size + sizeof (struct PeerGetResultMessage); - if (msize >= GNUNET_SERVER_MAX_MESSAGE_SIZE) + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) { - GNUNET_break (0); + put_path_length = 0; + msize = msize - put_path_length; return; } + if (msize >= GNUNET_CONSTANTS_MAX_ENCRYPTED_MESSAGE_SIZE) + { + GNUNET_break(0); + return; + } + current_path_index = 0; if(get_path_length > 0) { @@ -3518,6 +3521,11 @@ handle_dht_p2p_put (void *cls, const struct GNUNET_PeerIdentity *peer, return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), (int64_t) msize, + GNUNET_NO); + best_known_dest = put->best_known_destination; put_path = (struct GNUNET_PeerIdentity *) &put[1]; payload = &put_path[putlen]; @@ -3687,6 +3695,11 @@ handle_dht_p2p_get (void *cls, const struct GNUNET_PeerIdentity *peer, return GNUNET_YES; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + /* Add sender to get path */ struct GNUNET_PeerIdentity gp[get_length + 1]; if (get_length > 0) @@ -3795,6 +3808,11 @@ handle_dht_p2p_get_result (void *cls, const struct GNUNET_PeerIdentity *peer, return GNUNET_YES; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + put_path = (const struct GNUNET_PeerIdentity *) &get_result[1]; get_path = &put_path[putlen]; payload = (const void *) &get_path[getlen]; @@ -3930,6 +3948,11 @@ handle_dht_p2p_trail_setup (void *cls, const struct GNUNET_PeerIdentity *peer, return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail_peer_list = (const struct GNUNET_PeerIdentity *)&trail_setup[1]; current_dest = trail_setup->best_known_destination; trail_id = trail_setup->trail_id; @@ -4187,6 +4210,11 @@ handle_dht_p2p_trail_setup_result(void *cls, const struct GNUNET_PeerIdentity *p return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + is_predecessor = ntohl (trail_result->is_predecessor); querying_peer = trail_result->querying_peer; finger_identity = trail_result->finger_identity; @@ -4693,12 +4721,17 @@ handle_dht_p2p_verify_successor(void *cls, GNUNET_break_op (0); return GNUNET_OK; } - + + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail_id = vsm->trail_id; source_peer = vsm->source_peer; successor = vsm->successor; trail = (struct GNUNET_PeerIdentity *)&vsm[1]; - + /* I am NOT the successor of source_peer. Pass the message to next_hop on * the trail. */ @@ -5024,6 +5057,11 @@ handle_dht_p2p_verify_successor_result(void *cls, return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail = (const struct GNUNET_PeerIdentity *) &vsrm[1]; querying_peer = vsrm->querying_peer; trail_direction = ntohl (vsrm->trail_direction); @@ -5098,6 +5136,11 @@ handle_dht_p2p_notify_new_successor(void *cls, return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail = (struct GNUNET_PeerIdentity *) &nsm[1]; source = nsm->source_peer; new_successor = nsm->new_successor; @@ -5189,7 +5232,12 @@ handle_dht_p2p_trail_setup_rejection (void *cls, GNUNET_break_op (0); return GNUNET_OK; } - + + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail_peer_list = (const struct GNUNET_PeerIdentity *)&trail_rejection[1]; is_predecessor = ntohl (trail_rejection->is_predecessor); congestion_timeout = trail_rejection->congestion_time; @@ -5318,6 +5366,11 @@ handle_dht_p2p_trail_compression (void *cls, const struct GNUNET_PeerIdentity *p return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail_compression = (const struct PeerTrailCompressionMessage *) message; trail_id = trail_compression->trail_id; @@ -5385,6 +5438,11 @@ handle_dht_p2p_trail_teardown (void *cls, const struct GNUNET_PeerIdentity *peer return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail_teardown = (const struct PeerTrailTearDownMessage *) message; trail_direction = ntohl (trail_teardown->trail_direction); trail_id = trail_teardown->trail_id; @@ -5468,6 +5526,11 @@ handle_dht_p2p_add_trail (void *cls, const struct GNUNET_PeerIdentity *peer, return GNUNET_OK; } + GNUNET_STATISTICS_update (GDS_stats, + gettext_noop + ("# Bytes received from other peers"), msize, + GNUNET_NO); + trail = (const struct GNUNET_PeerIdentity *)&add_trail[1]; destination_peer = add_trail->destination_peer; source_peer = add_trail->source_peer; diff --git a/src/dht/gnunet_dht_profiler.c b/src/dht/gnunet_dht_profiler.c index f88749524..95528b3f9 100644 --- a/src/dht/gnunet_dht_profiler.c +++ b/src/dht/gnunet_dht_profiler.c @@ -205,7 +205,6 @@ static unsigned int n_gets_fail; */ static unsigned int replication; -#if 0 /** * Testbed Operation (to get stats). */ @@ -215,7 +214,30 @@ static struct GNUNET_TESTBED_Operation *stats_op; * Testbed peer handles. */ static struct GNUNET_TESTBED_Peer **testbed_handles; -#endif + +/** + * Total number of messages sent by peer. + */ +static uint64_t outgoing_bandwidth; + +/** + * Total number of messages received by peer. + */ +static uint64_t incoming_bandwidth; + +/** + * Average number of hops taken to do put. + */ +static unsigned int average_put_path_length; + +/** + * Average number of hops taken to do get. + */ +static unsigned int average_get_path_length; + +static unsigned int total_put_path_length; + +static unsigned int total_get_path_length; /** * Shutdown task. Cleanup all resources and operations. * @@ -251,10 +273,13 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_free (a_ctx); a_ctx = NULL; } + if(NULL != stats_op) + GNUNET_TESTBED_operation_done (stats_op); + stats_op = NULL; GNUNET_free_non_null (a_ac); } -#if 0 + /** * Stats callback. Finish the stats testbed operation and when all stats have * been iterated, shutdown the test. @@ -266,9 +291,12 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) */ static void bandwidth_stats_cont (void *cls, - struct GNUNET_TESTBED_Operation *op, const char *emsg) + struct GNUNET_TESTBED_Operation *op, + const char *emsg) { - + INFO ("# Outgoing bandwidth: %u\n", outgoing_bandwidth); + INFO ("# Incoming bandwidth: %u\n", incoming_bandwidth); + GNUNET_SCHEDULER_shutdown (); } @@ -284,35 +312,29 @@ bandwidth_stats_cont (void *cls, * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration */ static int -bandwidth_stats_iterator (void *cls, const struct GNUNET_TESTBED_Peer *peer, - const char *subsystem, const char *name, - uint64_t value, int is_persistent) +bandwidth_stats_iterator (void *cls, + const struct GNUNET_TESTBED_Peer *peer, + const char *subsystem, + const char *name, + uint64_t value, + int is_persistent) { - return GNUNET_OK; + static const char *s_sent = "# Bytes transmitted to other peers"; + static const char *s_recv = "# Bytes received from other peers"; + + if (0 == strncmp (s_sent, name, strlen (s_sent))) + outgoing_bandwidth = outgoing_bandwidth + value; + else if (0 == strncmp(s_recv, name, strlen (s_recv))) + incoming_bandwidth = incoming_bandwidth + value; + else + return GNUNET_OK; + DEBUG ("Bandwith - Out: %lu; In: %lu\n", + (unsigned long) outgoing_bandwidth, + (unsigned long) incoming_bandwidth); + return GNUNET_OK; } -/** - * Task that collects bandwidth used by all the peers. - * - * @param cls Closure (NULL). - * @param tc Task Context. - */ -static void -collect_bandwidth_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - if ((GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason) != 0) - return; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Start collecting bandwidth statistics...\n"); - //FIXME: what is the name of transport subsystem? - stats_op = GNUNET_TESTBED_get_statistics (n_active, testbed_handles, - NULL, NULL, - bandwidth_stats_iterator, - bandwidth_stats_cont, NULL); -} -#endif - static void summarize () { @@ -322,9 +344,19 @@ summarize () INFO ("# GETS made: %u\n", n_gets); INFO ("# GETS succeeded: %u\n", n_gets_ok); INFO ("# GETS failed: %u\n", n_gets_fail); - //FIXME: is this the right place to call b/w stats? - //GNUNET_SCHEDULER_add_now (&collect_bandwidth_stats, NULL); - GNUNET_SCHEDULER_shutdown (); + INFO ("# average_put_path_length: %u\n", average_put_path_length); + INFO ("# average_get_path_length: %u\n", average_get_path_length); + + if (NULL == testbed_handles) + { + INFO ("No peers found\n"); + return; + } + /* Collect Stats*/ + stats_op = GNUNET_TESTBED_get_statistics (n_active, testbed_handles, + "dht", NULL, + bandwidth_stats_iterator, + bandwidth_stats_cont, NULL); } @@ -338,19 +370,17 @@ static void cancel_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { struct ActiveContext *ac = cls; - struct Context *ctx = ac->ctx; ac->delay_task = GNUNET_SCHEDULER_NO_TASK; GNUNET_assert (NULL != ac->dht_get); GNUNET_DHT_get_stop (ac->dht_get); ac->dht_get = NULL; - GNUNET_TESTBED_operation_done (ctx->op); - ctx->op = NULL; n_gets_fail++; /* If profiling is complete, summarize */ if (n_active == n_gets_fail + n_gets_ok) summarize (); + } @@ -384,7 +414,6 @@ get_iter (void *cls, { struct ActiveContext *ac = cls; struct ActiveContext *get_ac = ac->get_ac; - struct Context *ctx = ac->ctx; /* Check the keys of put and get match or not. */ GNUNET_assert (0 == memcmp (key, &get_ac->hash, sizeof (struct GNUNET_HashCode))); @@ -396,12 +425,17 @@ get_iter (void *cls, ac->dht_get = NULL; GNUNET_SCHEDULER_cancel (ac->delay_task); ac->delay_task = GNUNET_SCHEDULER_NO_TASK; - GNUNET_TESTBED_operation_done (ctx->op); - ctx->op = NULL; + + total_put_path_length = total_put_path_length + put_path_length; + total_get_path_length = total_get_path_length + get_path_length; /* Summarize if profiling is complete */ if (n_active == n_gets_fail + n_gets_ok) + { + average_put_path_length = total_put_path_length/n_active; + average_get_path_length = total_get_path_length/n_active; summarize (); + } } @@ -468,71 +502,6 @@ put_cont (void *cls, int success) ac->delay_task = GNUNET_SCHEDULER_add_delayed (delay, &delayed_get, ac); } -#if 0 -/** - * Stats callback. Finish the stats testbed operation and when all stats have - * been iterated, shutdown the test. - * - * @param cls closure - * @param op the operation that has been finished - * @param emsg error message in case the operation has failed; will be NULL if - * operation has executed successfully. - */ -static void -finger_stats_cont (void *cls, - struct GNUNET_TESTBED_Operation *op, - const char *emsg) -{ - -} - - -/** - * Process statistic values. - * - * @param cls closure - * @param peer the peer the statistic belong to - * @param subsystem name of subsystem that created the statistic - * @param name the name of the datum - * @param value the current value - * @param is_persistent GNUNET_YES if the value is persistent, GNUNET_NO if not - * @return GNUNET_OK to continue, GNUNET_SYSERR to abort iteration - */ -static int -finger_stats_iterator (void *cls, const struct GNUNET_TESTBED_Peer *peer, - const char *subsystem, const char *name, - uint64_t value, int is_persistent) -{ - uint32_t i; - - i = GNUNET_TESTBED_get_index (peer); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, " STATS %u - %s [%s]: %llu\n", - i, subsystem, name, value); - - return GNUNET_OK; -} - - -/** - * Task check that keepalives were sent and received. - * - * @param cls Closure (NULL). - * @param tc Task Context. - */ -static void -collect_finger_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - if ((GNUNET_SCHEDULER_REASON_SHUTDOWN & tc->reason) != 0) - return; - - GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Start collecting statistics...\n"); - /* FIXME: Write subsystem name. */ - stats_op = GNUNET_TESTBED_get_statistics (n_active, testbed_handles, - "dht", NULL, - finger_stats_iterator, - finger_stats_cont, NULL); -} -#endif /** * Task to do DHT PUTS @@ -545,12 +514,6 @@ delayed_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { struct ActiveContext *ac = cls; - /*FIXME: Before doing anything else, first collect statistics from each peer - DHT and check if circle is formed. If yes then go ahead with more puts, - else wait for 'delay' time. This function does not return anything, so we - should have some way to notify that circle is done or we need to wait.*/ - //GNUNET_SCHEDULER_add_now(collect_finger_stats,NULL); - ac->delay_task = GNUNET_SCHEDULER_NO_TASK; /* Generate and DHT PUT some random data */ ac->put_data_size = 16; /* minimum */ @@ -696,7 +659,8 @@ test_run (void *cls, { unsigned int cnt; unsigned int ac_cnt; - + + testbed_handles = peers; if (NULL == peers) { /* exit */ @@ -802,8 +766,8 @@ main (int argc, char *const *argv) if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) return 2; - delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 5); /* default delay */ - timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30); /* default timeout */ + delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 30); /* default delay */ + timeout = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1); /* default timeout */ replication = 1; /* default replication */ rc = 0; if (GNUNET_OK != -- 2.25.1