X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fregex%2Fgnunet-regex-profiler.c;h=638f1a1a43bd2cc1d78eb39d013df1e82fa756d9;hb=f119c5a88b06a05349f311c27f27be9a2760bf3f;hp=ce0a2ecb18fb5c63e2448e22eeea9530b073dc6a;hpb=6a9bc072e64367297498b93cd6d975995203d2e9;p=oweals%2Fgnunet.git diff --git a/src/regex/gnunet-regex-profiler.c b/src/regex/gnunet-regex-profiler.c index ce0a2ecb1..638f1a1a4 100644 --- a/src/regex/gnunet-regex-profiler.c +++ b/src/regex/gnunet-regex-profiler.c @@ -31,14 +31,13 @@ #include "platform.h" #include "gnunet_applications.h" #include "gnunet_util_lib.h" -#include "gnunet_regex_lib.h" +#include "regex_internal_lib.h" #include "gnunet_arm_service.h" #include "gnunet_dht_service.h" #include "gnunet_testbed_service.h" #define FIND_TIMEOUT \ GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90) -#define SEARCHES_IN_PARALLEL 5 /** * DLL of operations @@ -151,7 +150,7 @@ struct RegexPeer /** * Handle to a running regex search. */ - struct GNUNET_REGEX_search_handle *search_handle; + struct REGEX_INTERNAL_Search *search_handle; /** * Testbed operation handle for DHT. @@ -163,11 +162,6 @@ struct RegexPeer */ struct GNUNET_STATISTICS_Handle *stats_handle; - /** - * Testbed operation handle for the statistics service. - */ - struct GNUNET_TESTBED_Operation *stats_op_handle; - /** * The starting time of a profiling step. */ @@ -184,6 +178,10 @@ struct RegexPeer struct GNUNET_TESTBED_Operation *daemon_op; }; +/** + * Set when shutting down to avoid making more queries. + */ +static int in_shutdown; /** * The array of peers; we fill this as the peers are given to us by the testbed @@ -271,14 +269,14 @@ static char *hosts_file; static char *strings_file; /** - * Search strings. + * Search strings (num_peers of them). */ static char **search_strings; /** - * Number of search strings. + * How many searches are we going to start in parallel */ -static int num_search_strings; +static long long unsigned int init_parallel_searches; /** * How many searches are running in parallel @@ -286,9 +284,9 @@ static int num_search_strings; static unsigned int parallel_searches; /** - * Number of peers found with search strings. + * Number of strings found in the published regexes. */ -static unsigned int peers_found; +static unsigned int strings_found; /** * Index of peer to start next announce/search. @@ -382,17 +380,6 @@ stats_connect_cb (void *cls, const char *emsg); -/** - * Task to collect all statistics from s, will shutdown the - * profiler, when done. - * - * @param cls NULL - * @param tc the task context - */ -static void -do_collect_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); - - /** * Start announcing the next regex in the DHT. * @@ -453,15 +440,13 @@ do_shutdown (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) if (NULL != peers[peer_cnt].op_handle) GNUNET_TESTBED_operation_done (peers[peer_cnt].op_handle); - if (NULL != peers[peer_cnt].stats_op_handle) - GNUNET_TESTBED_operation_done (peers[peer_cnt].stats_op_handle); } if (NULL != data_file) GNUNET_DISK_file_close (data_file); for (search_str_cnt = 0; - search_str_cnt < num_search_strings && NULL != search_strings; + search_str_cnt < num_peers && NULL != search_strings; search_str_cnt++) { GNUNET_free_non_null (search_strings[search_str_cnt]); @@ -493,7 +478,7 @@ do_abort (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { unsigned long i = (unsigned long) cls; - GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting %lu...\n", i); + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Aborting from line %lu...\n", i); abort_task = GNUNET_SCHEDULER_NO_TASK; result = GNUNET_SYSERR; if (GNUNET_SCHEDULER_NO_TASK != shutdown_task) @@ -603,10 +588,10 @@ stats_cb (void *cls, return; } - GNUNET_assert (NULL != peer->stats_op_handle); + GNUNET_assert (NULL != peer->op_handle); - GNUNET_TESTBED_operation_done (peer->stats_op_handle); - peer->stats_op_handle = NULL; + GNUNET_TESTBED_operation_done (peer->op_handle); + peer->op_handle = NULL; peer_cnt++; peer = &peers[peer_cnt]; @@ -615,10 +600,11 @@ stats_cb (void *cls, { struct GNUNET_TIME_Relative delay = { 100 }; shutdown_task = GNUNET_SCHEDULER_add_delayed (delay, &do_shutdown, NULL); + result = GNUNET_OK; } else { - peer->stats_op_handle = + peer->op_handle = GNUNET_TESTBED_service_connect (NULL, peer->peer_handle, "statistics", @@ -685,7 +671,7 @@ do_collect_stats (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_assert (NULL != peer->peer_handle); - peer->stats_op_handle = + peer->op_handle = GNUNET_TESTBED_service_connect (NULL, peer->peer_handle, "statistics", @@ -716,7 +702,7 @@ find_string (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); * Method called when we've found a peer that announced a regex * that matches our search string. Now get the statistics. * - * @param cls Closure provided in GNUNET_REGEX_search. + * @param cls Closure provided in REGEX_INTERNAL_search. * @param id Peer providing a regex that matches the string. * @param get_path Path of the get request. * @param get_path_length Lenght of get_path. @@ -743,14 +729,15 @@ regex_found_handler (void *cls, return; } - peers_found++; + strings_found++; parallel_searches--; if (GNUNET_SCHEDULER_NO_TASK != peer->timeout) { GNUNET_SCHEDULER_cancel (peer->timeout); peer->timeout = GNUNET_SCHEDULER_NO_TASK; - GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); + if (GNUNET_NO == in_shutdown) + GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); } if (NULL == id) @@ -758,7 +745,7 @@ regex_found_handler (void *cls, // FIXME not possible right now GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "String matching timed out for string %s on peer %u (%i/%i)\n", - peer->search_str, peer->id, peers_found, num_search_strings); + peer->search_str, peer->id, strings_found, num_peers); peer->search_str_matched = GNUNET_SYSERR; } else @@ -769,7 +756,7 @@ regex_found_handler (void *cls, "String %s found on peer %u after %s (%i/%i) (%u||)\n", peer->search_str, peer->id, GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO), - peers_found, num_search_strings, parallel_searches); + strings_found, num_peers, parallel_searches); peer->search_str_matched = GNUNET_YES; @@ -795,7 +782,7 @@ regex_found_handler (void *cls, GNUNET_TESTBED_operation_done (peer->op_handle); peer->op_handle = NULL; - if (peers_found == num_search_strings) + if (strings_found == num_peers) { prof_time = GNUNET_TIME_absolute_get_duration (prof_start_time); GNUNET_log (GNUNET_ERROR_TYPE_INFO, @@ -803,7 +790,10 @@ regex_found_handler (void *cls, GNUNET_STRINGS_relative_time_to_string (prof_time, GNUNET_NO)); if (GNUNET_SCHEDULER_NO_TASK != search_timeout_task) + { GNUNET_SCHEDULER_cancel (search_timeout_task); + search_timeout_task = GNUNET_SCHEDULER_NO_TASK; + } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Collecting stats and shutting down.\n"); GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL); @@ -819,14 +809,16 @@ regex_found_handler (void *cls, * @param tc the task context */ static void -search_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +search_timed_out (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) { + unsigned int i; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Finding matches to all strings did not succeed after %s.\n", GNUNET_STRINGS_relative_time_to_string (search_timeout_time, GNUNET_NO)); GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Found %i of %i strings\n", peers_found, num_search_strings); + "Found %i of %i strings\n", strings_found, num_peers); GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Search timed out after %s." @@ -834,6 +826,15 @@ search_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) GNUNET_STRINGS_relative_time_to_string (search_timeout_time, GNUNET_NO)); + in_shutdown = GNUNET_YES; + for (i = 0; i < num_peers; i++) + { + if (NULL != peers[i].op_handle) + { + GNUNET_TESTBED_operation_done (peers[i].op_handle); + peers[i].op_handle = NULL; + } + } GNUNET_SCHEDULER_add_now (&do_collect_stats, NULL); } @@ -846,7 +847,7 @@ search_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) * @param tc TaskContext. */ static void -find_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +find_timed_out (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { struct RegexPeer *p = cls; @@ -855,10 +856,11 @@ find_timeout (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) if ((tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) != 0) return; GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Searching for string \"%s\" on peer %d timed out. Starting new search.\n", + "Searching for string \"%s\" on peer %d timed out.\n", p->search_str, p->id); - GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); + if (GNUNET_NO == in_shutdown) + GNUNET_SCHEDULER_add_now (&announce_next_regex, NULL); } @@ -874,7 +876,8 @@ find_string (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) unsigned int search_peer = (unsigned int) (long) cls; if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) || - search_peer >= num_search_strings) + search_peer >= num_peers || + GNUNET_YES == in_shutdown) return; GNUNET_log (GNUNET_ERROR_TYPE_INFO, @@ -894,7 +897,7 @@ find_string (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) &peers[search_peer]); GNUNET_assert (NULL != peers[search_peer].op_handle); peers[search_peer].timeout = GNUNET_SCHEDULER_add_delayed (FIND_TIMEOUT, - &find_timeout, + &find_timed_out, &peers[search_peer]); } @@ -915,15 +918,13 @@ daemon_started (void *cls, struct GNUNET_TESTBED_Operation *op, struct RegexPeer *peer = (struct RegexPeer *) cls; unsigned long search_peer; unsigned int i; - unsigned int me; GNUNET_TESTBED_operation_done (peer->daemon_op); peer->daemon_op = NULL; - me = peer - peers; if (NULL != emsg) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, - "Failed to start/stop daemon at peer %u: %s\n", me, emsg); + "Failed to start/stop daemon at peer %u: %s\n", peer->id, emsg); GNUNET_abort (); } @@ -936,7 +937,7 @@ daemon_started (void *cls, struct GNUNET_TESTBED_Operation *op, if (i > num_peers) GNUNET_abort (); /* we ran out of peers, must be a bug */ } - peers[search_peer].search_str = search_strings[me]; + peers[search_peer].search_str = search_strings[peer->id]; peers[search_peer].search_str_matched = GNUNET_NO; GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply( reannounce_period_max, @@ -960,7 +961,7 @@ do_announce (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting announce.\n"); - for (i = 0; i < SEARCHES_IN_PARALLEL; i++) + for (i = 0; i < init_parallel_searches; i++) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, " scheduling announce %u\n", @@ -981,9 +982,22 @@ announce_next_regex (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { struct RegexPeer *peer; - if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN) || - next_search >= num_peers) + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + return; + if (next_search >= num_peers) + { + if (strings_found != num_peers) + { + struct GNUNET_TIME_Relative new_delay; + if (GNUNET_SCHEDULER_NO_TASK != search_timeout_task) + GNUNET_SCHEDULER_cancel (search_timeout_task); + new_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15); + search_timeout_task = GNUNET_SCHEDULER_add_delayed (new_delay, + &search_timed_out, + NULL); + } return; + } GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Starting daemon %u\n", next_search); peer = &peers[next_search]; @@ -1025,7 +1039,7 @@ dht_connect_cb (void *cls, struct GNUNET_TESTBED_Operation *op, GNUNET_assert (peer->dht_handle == ca_result); peer->search_str_matched = GNUNET_NO; - peer->search_handle = GNUNET_REGEX_search (peer->dht_handle, + peer->search_handle = REGEX_INTERNAL_search (peer->dht_handle, peer->search_str, ®ex_found_handler, peer, NULL); @@ -1067,7 +1081,7 @@ dht_da (void *cls, void *op_result) if (NULL != peer->search_handle) { - GNUNET_REGEX_search_cancel (peer->search_handle); + REGEX_INTERNAL_search_cancel (peer->search_handle); peer->search_handle = NULL; } @@ -1084,7 +1098,7 @@ dht_da (void *cls, void *op_result) * * @param cls NULL * @param num_peers_ number of peers in 'peers' - * @param peers handle to peers run in the testbed. NULL upon timeout (see + * @param testbed_peers handle to peers run in the testbed. NULL upon timeout (see * GNUNET_TESTBED_test_run()). * @param links_succeeded the number of overlay link connection attempts that * succeeded @@ -1117,11 +1131,49 @@ test_master (void *cls, { peers[i].peer_handle = testbed_peers[i]; } - GNUNET_SCHEDULER_add_now (&do_announce, NULL); - abort_task = - GNUNET_SCHEDULER_add_delayed (search_timeout_time, - &do_abort, - (void*) __LINE__); + if (GNUNET_NO == + GNUNET_CONFIGURATION_get_value_yesno (cfg, "DHT", "DISABLE_TRY_CONNECT")) + { + struct GNUNET_TIME_Relative settle_time; + + settle_time = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, + 10 * num_peers); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Waiting for DHT for %s to settle new connections.\n\n", + GNUNET_STRINGS_relative_time_to_string(settle_time, GNUNET_NO)); + GNUNET_SCHEDULER_add_delayed (settle_time, &do_announce, NULL); + } + else + { + GNUNET_SCHEDULER_add_now (&do_announce, NULL); + } + search_timeout_task = + GNUNET_SCHEDULER_add_delayed (search_timeout_time, &search_timed_out, NULL); +} + +/** + * Function that will be called whenever something in the testbed changes. + * + * @param cls closure, NULL + * @param event information on what is happening + */ +static void +master_controller_cb (void *cls, + const struct GNUNET_TESTBED_EventInformation *event) +{ + switch (event->type) + { + case GNUNET_TESTBED_ET_CONNECT: + printf("."); + break; + case GNUNET_TESTBED_ET_PEER_START: + printf("#"); + break; + default: + break; + } + fflush(stdout); } @@ -1216,7 +1268,10 @@ run (void *cls, char *const *args, const char *cfgfile, { unsigned int nsearchstrs; unsigned int i; - + struct GNUNET_TIME_Relative abort_time; + + in_shutdown = GNUNET_NO; + /* Check config */ if (NULL == config) { @@ -1236,6 +1291,16 @@ run (void *cls, char *const *args, const char *cfgfile, shutdown_task = GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); return; } + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "REGEXPROFILER", + "PARALLEL_SEARCHES", + &init_parallel_searches)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Configuration option \"PARALLEL_SEARCHES\" missing." + " Using default (%d)\n", 10); + init_parallel_searches = 10; + } if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_time (cfg, "REGEXPROFILER", "REANNOUNCE_PERIOD_MAX", @@ -1248,12 +1313,6 @@ run (void *cls, char *const *args, const char *cfgfile, } /* Check arguments */ - if (NULL == hosts_file) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No hosts-file specified on command line. Exiting.\n")); - return; - } if (NULL == policy_dir) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, @@ -1285,23 +1344,25 @@ run (void *cls, char *const *args, const char *cfgfile, } nsearchstrs = load_search_strings (strings_file, &search_strings, - num_search_strings); - if (num_search_strings != nsearchstrs) + num_peers); + if (num_peers != nsearchstrs) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("Error loading search strings." - "Given file does not contain enough strings. Exiting.\n")); + "Error loading search strings.\n"); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "File (%s) does not contain enough strings (%u/%u).\n", + strings_file, nsearchstrs, num_peers); shutdown_task = GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); return; } - if (0 >= num_search_strings || NULL == search_strings) + if (0 >= num_peers || NULL == search_strings) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("Error loading search strings. Exiting.\n")); shutdown_task = GNUNET_SCHEDULER_add_now (&do_shutdown, NULL); return; } - for (i = 0; i < num_search_strings; i++) + for (i = 0; i < num_peers; i++) GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "search string: %s\n", search_strings[i]); @@ -1329,24 +1390,39 @@ run (void *cls, char *const *args, const char *cfgfile, peers[i].id = i; } + GNUNET_CONFIGURATION_set_value_number (cfg, + "TESTBED", "OVERLAY_RANDOM_LINKS", + num_peers * 20); + GNUNET_CONFIGURATION_set_value_number (cfg, + "DHT", "FORCE_NSE", + (long long unsigned) + (log (num_peers) / log (2.0))); event_mask = 0LL; -/* For feedback about the start process activate these and pass master_cb +/* For feedback about the start process activate these and pass master_cb */ event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_START); - event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP); +// event_mask |= (1LL << GNUNET_TESTBED_ET_PEER_STOP); event_mask |= (1LL << GNUNET_TESTBED_ET_CONNECT); - event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT);*/ +// event_mask |= (1LL << GNUNET_TESTBED_ET_DISCONNECT); prof_start_time = GNUNET_TIME_absolute_get (); - GNUNET_TESTBED_run (args[0], + GNUNET_TESTBED_run (hosts_file, cfg, num_peers, event_mask, - NULL, /* master_controller_cb, */ + &master_controller_cb, NULL, /* master_controller_cb cls */ &test_master, NULL); /* test_master cls */ + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_time (cfg, "TESTBED", + "SETUP_TIMEOUT", + &abort_time)) + { + abort_time = + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15); + } + abort_time = GNUNET_TIME_relative_add (abort_time, GNUNET_TIME_UNIT_MINUTES); abort_task = - GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_relative_multiply - (GNUNET_TIME_UNIT_MINUTES, 5), + GNUNET_SCHEDULER_add_delayed (abort_time, &do_abort, (void*) __LINE__); } @@ -1365,22 +1441,19 @@ main (int argc, char *const *argv) static const struct GNUNET_GETOPT_CommandLineOption options[] = { {'o', "output-file", "FILENAME", gettext_noop ("name of the file for writing statistics"), - 1, &GNUNET_GETOPT_set_string, &data_filename}, + GNUNET_YES, &GNUNET_GETOPT_set_string, &data_filename}, {'t', "matching-timeout", "TIMEOUT", - gettext_noop ("wait TIMEOUT before considering a string match as failed"), - GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &search_timeout_time }, - {'n', "num-search-strings", "COUNT", - gettext_noop ("number of search strings to read from search strings file"), - GNUNET_YES, &GNUNET_GETOPT_set_uint, &num_search_strings }, + gettext_noop ("wait TIMEOUT before ending the experiment"), + GNUNET_YES, &GNUNET_GETOPT_set_relative_time, &search_timeout_time}, {'p', "policy-dir", "DIRECTORY", gettext_noop ("directory with policy files"), - GNUNET_YES, &GNUNET_GETOPT_set_filename, &policy_dir }, + GNUNET_YES, &GNUNET_GETOPT_set_filename, &policy_dir}, {'s', "strings-file", "FILENAME", gettext_noop ("name of file with input strings"), - GNUNET_YES, &GNUNET_GETOPT_set_filename, &strings_file }, + GNUNET_YES, &GNUNET_GETOPT_set_filename, &strings_file}, {'H', "hosts-file", "FILENAME", gettext_noop ("name of file with hosts' names"), - GNUNET_YES, &GNUNET_GETOPT_set_filename, &hosts_file }, + GNUNET_YES, &GNUNET_GETOPT_set_filename, &hosts_file}, GNUNET_GETOPT_OPTION_END }; int ret; @@ -1390,7 +1463,7 @@ main (int argc, char *const *argv) result = GNUNET_SYSERR; ret = GNUNET_PROGRAM_run (argc, argv, - "gnunet-regex-profiler [OPTIONS] hosts-file policy-dir search-strings-file", + "gnunet-regex-profiler", _("Profiler for regex"), options, &run, NULL); if (GNUNET_OK != ret)