From dd333f69e55bd7e2b94b4bb281e9b0c0b63c23f2 Mon Sep 17 00:00:00 2001 From: Martin Schanzenbach Date: Thu, 22 Mar 2012 16:36:39 +0000 Subject: [PATCH] -parallel background lookups --- src/gns/Makefile.am | 17 ++- src/gns/gnunet-service-gns.c | 39 +++++-- src/gns/gnunet-service-gns_resolver.c | 150 +++++++++++++++++++++----- src/gns/gnunet-service-gns_resolver.h | 15 ++- src/gns/test_gns_simple_lookup.c | 18 +--- src/gns/test_gns_simple_lookup.conf | 3 +- 6 files changed, 188 insertions(+), 54 deletions(-) diff --git a/src/gns/Makefile.am b/src/gns/Makefile.am index f41de7e3c..2b62cec53 100644 --- a/src/gns/Makefile.am +++ b/src/gns/Makefile.am @@ -38,7 +38,8 @@ check_PROGRAMS = \ test_gns_simple_mx_lookup \ test_gns_simple_zkey_lookup \ test_gns_dht_delegated_lookup \ - test_gns_pseu_shorten + test_gns_pseu_shorten \ + test_gns_max_queries # test_gns_simple_lookup @@ -169,6 +170,20 @@ test_gns_pseu_shorten_DEPENDENCIES = \ $(top_builddir)/src/gns/libgnunetgns.la \ $(top_builddir)/src/testing/libgnunettesting.la + +test_gns_max_queries_SOURCES = \ + test_gns_max_queries.c +test_gns_max_queries_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/namestore/libgnunetnamestore.la \ + $(top_builddir)/src/gns/libgnunetgns.la \ + $(top_builddir)/src/testing/libgnunettesting.la +test_gns_max_queries_DEPENDENCIES = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/namestore/libgnunetnamestore.la \ + $(top_builddir)/src/gns/libgnunetgns.la \ + $(top_builddir)/src/testing/libgnunettesting.la + gnunet_gns_SOURCES = \ gnunet-gns.c gnunet_gns_LDADD = \ diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c index 37d53bfd3..dff300bb7 100644 --- a/src/gns/gnunet-service-gns.c +++ b/src/gns/gnunet-service-gns.c @@ -19,10 +19,6 @@ */ /** - * - * TODO: - * - Write xquery and block plugin - * - The smaller FIXME issues all around * * @file gns/gnunet-service-gns.c * @brief GNUnet GNS service @@ -155,6 +151,9 @@ GNUNET_SCHEDULER_TaskIdentifier zone_update_taskid = GNUNET_SCHEDULER_NO_TASK; /* automatic pkey import for name shortening */ static int auto_import_pkey; +/* lookup timeout */ +static struct GNUNET_TIME_Relative default_lookup_timeout; + /** * Task run during shutdown. * @@ -174,6 +173,7 @@ shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_SERVER_notification_context_destroy (nc); gns_interceptor_stop(); + gns_resolver_cleanup(); GNUNET_NAMESTORE_disconnect(namestore_handle, 1); GNUNET_DHT_disconnect(dht_handle); @@ -723,14 +723,14 @@ handle_lookup(void *cls, { gns_resolver_lookup_record(zone_hash, clh->type, name, zone_key, - GNUNET_GNS_DEFAULT_LOOKUP_TIMEOUT, + default_lookup_timeout, &send_lookup_response, clh); } else { gns_resolver_lookup_record(zone_hash, clh->type, name, NULL, - GNUNET_GNS_DEFAULT_LOOKUP_TIMEOUT, + default_lookup_timeout, &send_lookup_response, clh); } } @@ -753,6 +753,8 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, char* keyfile; struct GNUNET_CRYPTO_RsaPublicKeyBinaryEncoded pkey; + unsigned long long max_parallel_bg_queries = 0; + unsigned long long default_lookup_timeout_secs = 0; static const struct GNUNET_SERVER_MessageHandler handlers[] = { {&handle_shorten, NULL, GNUNET_MESSAGE_TYPE_GNS_SHORTEN, 0}, @@ -813,8 +815,31 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, auto_import_pkey = GNUNET_YES; } + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number(c, "gns", + "MAX_PARALLEL_BACKGROUND_QUERIES", + &max_parallel_bg_queries)) + { + GNUNET_log(GNUNET_ERROR_TYPE_INFO, + "Number of allowed parallel background queries: %d\n", + max_parallel_bg_queries); + } + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number(c, "gns", + "DEFAULT_LOOKUP_TIMEOUT", + &default_lookup_timeout_secs)) + { + GNUNET_log(GNUNET_ERROR_TYPE_INFO, + "Default lookup timeout: %ds\n", default_lookup_timeout_secs); + default_lookup_timeout = GNUNET_TIME_relative_multiply( + GNUNET_TIME_UNIT_SECONDS, + default_lookup_timeout_secs); + } - if (gns_resolver_init(namestore_handle, dht_handle) + if (gns_resolver_init(namestore_handle, dht_handle, zone_hash, + max_parallel_bg_queries) == GNUNET_SYSERR) { GNUNET_log(GNUNET_ERROR_TYPE_ERROR, diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c index ed2f8797c..4e074577f 100644 --- a/src/gns/gnunet-service-gns_resolver.c +++ b/src/gns/gnunet-service-gns_resolver.c @@ -59,6 +59,16 @@ static struct GNUNET_DHT_Handle *dht_handle; */ static struct GNUNET_CONTAINER_Heap *dht_lookup_heap; +/** + * Maximum amount of parallel queries in background + */ +static unsigned long long max_allowed_background_queries; + +/** + * Our local zone + */ +static struct GNUNET_CRYPTO_ShortHashCode local_zone; + /** * Namestore calls this function if we have record for this name. * (or with rd_count=0 to indicate no matches) @@ -321,7 +331,7 @@ process_zone_to_name_discover(void *cls, xquery = htonl(GNUNET_GNS_RECORD_PSEU); gph->get_handle = GNUNET_DHT_get_start(dht_handle, - DHT_OPERATION_TIMEOUT, + GNUNET_TIME_UNIT_FOREVER_REL, GNUNET_BLOCK_TYPE_GNS_NAMERECORD, &lookup_key, DHT_GNS_REPLICATION_LEVEL, @@ -378,12 +388,17 @@ static void process_discovered_authority(char* name, */ int gns_resolver_init(struct GNUNET_NAMESTORE_Handle *nh, - struct GNUNET_DHT_Handle *dh) + struct GNUNET_DHT_Handle *dh, + struct GNUNET_CRYPTO_ShortHashCode lz, + unsigned long long max_bg_queries) { namestore_handle = nh; dht_handle = dh; + local_zone = lz; dht_lookup_heap = GNUNET_CONTAINER_heap_create(GNUNET_CONTAINER_HEAP_ORDER_MIN); + max_allowed_background_queries = max_bg_queries; + if ((namestore_handle != NULL) && (dht_handle != NULL)) { return GNUNET_OK; @@ -391,6 +406,41 @@ gns_resolver_init(struct GNUNET_NAMESTORE_Handle *nh, return GNUNET_SYSERR; } +/** + * Cleanup background lookups + */ +static int +cleanup_pending_background_queries(void* cls, + struct GNUNET_CONTAINER_HeapNode *node, + void *element, + GNUNET_CONTAINER_HeapCostType cost) +{ + struct ResolverHandle *rh = (struct ResolverHandle *)element; + + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Terminating background lookup for %s\n", + rh->name); + GNUNET_DHT_get_stop(rh->get_handle); + rh->proc(rh->proc_cls, rh, 0, NULL); + + return GNUNET_YES; +} + + +/** + * Shutdown resolver + */ +void +gns_resolver_cleanup() +{ + if (0 != GNUNET_CONTAINER_heap_get_size(dht_lookup_heap)) + { + GNUNET_CONTAINER_heap_iterate (dht_lookup_heap, + &cleanup_pending_background_queries, + NULL); + } +} + /** * Helper function to free resolver handle @@ -446,6 +496,14 @@ on_namestore_record_put_result(void *cls, "Error putting records into namestore: %s\n", emsg); } +static void +handle_lookup_timeout(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct ResolverHandle *rh = cls; + + if (rh->timeout_cont) + rh->timeout_cont(rh->timeout_cont_cls, tc); +} /** * Processor for background lookups in the DHT @@ -486,6 +544,10 @@ dht_lookup_timeout(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) strcpy(new_name, rh->name); memcpy(new_name+strlen(new_name), GNUNET_GNS_TLD, strlen(GNUNET_GNS_TLD)); + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Starting background lookup for %s type %d\n", + new_name, rlh->record_type); + gns_resolver_lookup_record(rh->authority, rlh->record_type, new_name, @@ -649,11 +711,27 @@ resolve_record_dht(struct ResolverHandle *rh) "starting dht lookup for %s with key: %s\n", rh->name, (char*)&lookup_key_string); - rh->timeout_task = GNUNET_SCHEDULER_NO_TASK; + //rh->timeout_task = GNUNET_SCHEDULER_NO_TASK; rh->dht_heap_node = NULL; if (rh->timeout.rel_value != GNUNET_TIME_UNIT_FOREVER_REL.rel_value) { + /** + * Update timeout if necessary + */ + if (rh->timeout_task == GNUNET_SCHEDULER_NO_TASK) + { + + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Adjusting timeout\n"); + /* + * Set timeout for authority lookup phase to 1/2 + */ + rh->timeout_task = GNUNET_SCHEDULER_add_delayed( + GNUNET_TIME_relative_divide(rh->timeout, 2), + &handle_lookup_timeout, + rh); + } //rh->timeout_task = GNUNET_SCHEDULER_add_delayed (DHT_LOOKUP_TIMEOUT, // &dht_lookup_timeout, // rh); @@ -662,7 +740,7 @@ resolve_record_dht(struct ResolverHandle *rh) } else { - if (GNUNET_GNS_MAX_PARALLEL_LOOKUPS > + if (max_allowed_background_queries <= GNUNET_CONTAINER_heap_get_size (dht_lookup_heap)) { rh_heap_root = GNUNET_CONTAINER_heap_remove_root (dht_lookup_heap); @@ -680,7 +758,7 @@ resolve_record_dht(struct ResolverHandle *rh) xquery = htonl(rlh->record_type); rh->get_handle = GNUNET_DHT_get_start(dht_handle, - DHT_OPERATION_TIMEOUT, + GNUNET_TIME_UNIT_FOREVER_REL, GNUNET_BLOCK_TYPE_GNS_NAMERECORD, &lookup_key, DHT_GNS_REPLICATION_LEVEL, @@ -827,14 +905,7 @@ resolve_record_ns(struct ResolverHandle *rh) rh); } -static void -handle_lookup_timeout(void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) -{ - struct ResolverHandle *rh = cls; - if (rh->timeout_cont) - rh->timeout_cont(rh->timeout_cont_cls, tc); -} /** * Handle timeout for DHT requests @@ -854,6 +925,8 @@ dht_authority_lookup_timeout(void *cls, "dht lookup for query %s timed out.\n", rh->name); + rh->status |= TIMED_OUT; + if (strcmp(rh->name, "") == 0) { @@ -868,11 +941,15 @@ dht_authority_lookup_timeout(void *cls, /** * Start resolution in bg */ - strcpy(new_name, rh->name); - strcpy(new_name+strlen(new_name), "."); - memcpy(new_name+strlen(new_name), rh->authority_name, - strlen(rh->authority_name)); - memcpy(new_name+strlen(new_name), GNUNET_GNS_TLD, strlen(GNUNET_GNS_TLD)); + GNUNET_snprintf(new_name, MAX_DNS_NAME_LENGTH, + "%s.%s", rh->name, GNUNET_GNS_TLD); + //strcpy(new_name, rh->name); + //strcpy(new_name+strlen(new_name), "."); + //memcpy(new_name+strlen(new_name), GNUNET_GNS_TLD, strlen(GNUNET_GNS_TLD)); + + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Starting background query for %s type %d\n", + new_name, rlh->record_type); gns_resolver_lookup_record(rh->authority, rlh->record_type, @@ -882,8 +959,16 @@ dht_authority_lookup_timeout(void *cls, &background_lookup_result_processor, NULL); + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Terminating auth lookup\n"); + GNUNET_DHT_get_stop (rh->get_handle); + rh->timeout_task = GNUNET_SCHEDULER_NO_TASK; + + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "Terminating auth lookup\n"); + rh->proc(rh->proc_cls, rh, 0, NULL); } @@ -1228,10 +1313,12 @@ handle_record_ns(void* cls, struct ResolverHandle *rh, rlh = (struct RecordLookupHandle*) cls; if (rd_count == 0) { + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Resolution status: %d!\n", rh->status); + /* ns entry expired and not ours. try dht */ if (rh->status & (EXPIRED | !EXISTS) && GNUNET_CRYPTO_short_hash_cmp(&rh->authority_chain_head->zone, - &rh->authority_chain_tail->zone)) + &local_zone)) { rh->proc = &handle_record_dht; resolve_record_dht(rh); @@ -1353,6 +1440,8 @@ handle_delegation_dht(void* cls, struct ResolverHandle *rh, struct RecordLookupHandle* rlh; rlh = (struct RecordLookupHandle*) cls; + + if (strcmp(rh->name, "") == 0) { if ((rlh->record_type == GNUNET_GNS_RECORD_PKEY)) @@ -1427,7 +1516,7 @@ resolve_delegation_dht(struct ResolverHandle *rh) } else { - if (GNUNET_GNS_MAX_PARALLEL_LOOKUPS > + if (max_allowed_background_queries <= GNUNET_CONTAINER_heap_get_size (dht_lookup_heap)) { /* terminate oldest lookup */ @@ -1447,7 +1536,7 @@ resolve_delegation_dht(struct ResolverHandle *rh) xquery = htonl(GNUNET_GNS_RECORD_PKEY); rh->get_handle = GNUNET_DHT_get_start(dht_handle, - DHT_OPERATION_TIMEOUT, + GNUNET_TIME_UNIT_FOREVER_REL, GNUNET_BLOCK_TYPE_GNS_NAMERECORD, &lookup_key, DHT_GNS_REPLICATION_LEVEL, @@ -1610,11 +1699,13 @@ process_delegation_result_ns(void* cls, GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "Adding %s back to %s\n", rh->authority_name, rh->name); - memset(new_name, 0, strlen(rh->name) + strlen(rh->authority_name) + 2); + //memset(new_name, 0, strlen(rh->name) + strlen(rh->authority_name) + 2); strcpy(new_name, rh->name); - strcpy(new_name+strlen(new_name)+1, "."); - strcpy(new_name+strlen(new_name)+2, rh->authority_name); + strcpy(new_name+strlen(new_name), "."); + strcpy(new_name+strlen(new_name), rh->authority_name); strcpy(rh->name, new_name); + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, + "%s restored\n", rh->name); } rh->proc(rh->proc_cls, rh, 0, NULL); return; @@ -1747,14 +1838,19 @@ gns_resolver_lookup_record(struct GNUNET_CRYPTO_ShortHashCode zone, rh->timeout = timeout; if (timeout.rel_value != GNUNET_TIME_UNIT_FOREVER_REL.rel_value) { - rh->timeout_task = GNUNET_SCHEDULER_add_delayed(timeout, + /* + * Set timeout for authority lookup phase to 1/2 + */ + rh->timeout_task = GNUNET_SCHEDULER_add_delayed( + GNUNET_TIME_relative_divide(timeout, 2), &handle_lookup_timeout, rh); - rh->timeout_cont = &handle_auth_discovery_timeout; + rh->timeout_cont = &dht_authority_lookup_timeout; rh->timeout_cont_cls = rh; } else { + GNUNET_log(GNUNET_ERROR_TYPE_DEBUG, "No timeout for query!\n"); rh->timeout_task = GNUNET_SCHEDULER_NO_TASK; } @@ -1891,7 +1987,7 @@ process_zone_to_name_shorten(void *cls, free_resolver_handle(rh); } else if (GNUNET_CRYPTO_short_hash_cmp(&rh->authority_chain_head->zone, - &rh->authority_chain_tail->zone)) + &local_zone)) { /* our zone, just append .gnunet */ answer_len = strlen(rh->name) + strlen(GNUNET_GNS_TLD) + 2; @@ -1967,7 +2063,7 @@ handle_delegation_ns_shorten(void* cls, "PKEY resolved as far as possible in ns up to %s!\n", rh->name); if (GNUNET_CRYPTO_short_hash_cmp(&rh->authority_chain_head->zone, - &rh->authority_chain_tail->zone) == 0) + &local_zone) == 0) { /** * This is our zone append .gnunet unless name is empty diff --git a/src/gns/gnunet-service-gns_resolver.h b/src/gns/gnunet-service-gns_resolver.h index e973b0c52..073c19192 100644 --- a/src/gns/gnunet-service-gns_resolver.h +++ b/src/gns/gnunet-service-gns_resolver.h @@ -88,7 +88,8 @@ typedef void (*ResolutionResultProcessor) (void *cls, enum ResolutionStatus { EXISTS = 1, - EXPIRED = 2 + EXPIRED = 2, + TIMED_OUT = 4 }; /** @@ -244,11 +245,21 @@ struct GetPseuAuthorityHandle * * @param nh handle to the namestore * @param dh handle to the dht + * @param local_zone the local zone + * @param max_bg_queries maximum amount of background queries * @returns GNUNET_OK on success */ int gns_resolver_init(struct GNUNET_NAMESTORE_Handle *nh, - struct GNUNET_DHT_Handle *dh); + struct GNUNET_DHT_Handle *dh, + struct GNUNET_CRYPTO_ShortHashCode local_zone, + unsigned long long max_bg_queries); + +/** + * Cleanup resolver: Terminate pending lookups + */ +void +gns_resolver_cleanup(void); /** * Lookup of a record in a specific zone diff --git a/src/gns/test_gns_simple_lookup.c b/src/gns/test_gns_simple_lookup.c index e93de1a66..9cfde438d 100644 --- a/src/gns/test_gns_simple_lookup.c +++ b/src/gns/test_gns_simple_lookup.c @@ -18,23 +18,9 @@ Boston, MA 02111-1307, USA. */ /** - * @file gns/test_gns_twopeer.c - * @brief base testcase for testing DHT service with - * two running peers. + * @file gns/test_gns_simple_lookup.c + * @brief base testcase for testing a local GNS record lookup * - * This testcase starts peers using the GNUNET_TESTING_daemons_start - * function call. On peer start, connects to the peers DHT service - * by calling GNUNET_DHT_connected. Once notified about all peers - * being started (by the peers_started_callback function), calls - * GNUNET_TESTING_connect_topology, which connects the peers in a - * "straight line" topology. On notification that all peers have - * been properly connected, calls the do_get function which initiates - * a GNUNET_DHT_get from the *second* peer. Once the GNUNET_DHT_get - * function starts, runs the do_put function to insert data at the first peer. - * If the GET is successful, schedules finish_testing - * to stop the test and shut down peers. If GET is unsuccessful - * after GET_TIMEOUT seconds, prints an error message and shuts down - * the peers. */ #include "platform.h" #include "gnunet_testing_lib.h" diff --git a/src/gns/test_gns_simple_lookup.conf b/src/gns/test_gns_simple_lookup.conf index bdb233b3d..c2fede933 100644 --- a/src/gns/test_gns_simple_lookup.conf +++ b/src/gns/test_gns_simple_lookup.conf @@ -80,7 +80,8 @@ UNIXPATH = /tmp/gnunet-service-gns.sock HOME = $SERVICEHOME CONFIG = $DEFAULTCONFIG AUTO_IMPORT_PKEY = YES - +MAX_PARALLEL_BACKGROUND_QUERIES = 10 +DEFAULT_LOOKUP_TIMEOUT = 5 [nse] AUTOSTART = NO -- 2.25.1