From: Christian Grothoff Date: Sun, 29 Apr 2018 08:56:15 +0000 (+0200) Subject: add option to disable namecache, add velocity calculation and delay correction to... X-Git-Tag: v0.11.0pre66~93^2~13 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=ce2864cfaa27e55096b480bf35db5f8cee2a5e7e;p=oweals%2Fgnunet.git add option to disable namecache, add velocity calculation and delay correction to zonemaster, fix some ftbfs from recent NAMESTORE API change --- diff --git a/contrib/timeout_watchdog.c b/contrib/timeout_watchdog.c index 7a0376b9f..fc61a7cc7 100644 --- a/contrib/timeout_watchdog.c +++ b/contrib/timeout_watchdog.c @@ -33,12 +33,14 @@ static pid_t child; + static void sigchld_handler (int val) { int status = 0; int ret = 0; + (void) val; waitpid (child, &status, 0); if (WIFEXITED (status) != 0) { @@ -53,6 +55,7 @@ sigchld_handler (int val) exit (ret); } + static void sigint_handler (int val) { @@ -60,8 +63,10 @@ sigint_handler (int val) exit (val); } + int -main (int argc, char *argv[]) +main (int argc, + char *argv[]) { int timeout = 0; pid_t gpid = 0; diff --git a/src/credential/gnunet-service-credential.c b/src/credential/gnunet-service-credential.c index a2c339363..399371a2e 100644 --- a/src/credential/gnunet-service-credential.c +++ b/src/credential/gnunet-service-credential.c @@ -961,9 +961,11 @@ collect_next (void *cls) struct VerifyRequestHandle *vrh = cls; vrh->collect_next_task = NULL; GNUNET_assert (NULL != vrh->cred_collection_iter); - GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter); + GNUNET_NAMESTORE_zone_iterator_next (vrh->cred_collection_iter, + 1); } + static void handle_cred_collection_cb (void *cls, const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, diff --git a/src/gns/gns.conf.in b/src/gns/gns.conf.in index d48a213e9..2e49a4c60 100644 --- a/src/gns/gns.conf.in +++ b/src/gns/gns.conf.in @@ -22,9 +22,6 @@ MAX_PARALLEL_BACKGROUND_QUERIES = 1000 # called via NSS or other mechanisms). INTERCEPT_DNS = YES -# Using caching (start with namestore), or always ask DHT? -USE_CACHE = YES - # PREFIX = valgrind --leak-check=full --track-origins=yes # Zones diff --git a/src/gns/gnunet-service-gns.c b/src/gns/gnunet-service-gns.c index 90cd47e1d..c376ddfcc 100644 --- a/src/gns/gnunet-service-gns.c +++ b/src/gns/gnunet-service-gns.c @@ -229,14 +229,14 @@ GNS_get_tld (const char *name) /** * Task run during shutdown. * - * @param cls unused - * @param tc unused + * @param cls unused, NULL */ static void shutdown_task (void *cls) { struct GNS_TopLevelDomain *tld; + (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Shutting down!\n"); GNS_interceptor_done (); @@ -283,6 +283,7 @@ client_disconnect_cb (void *cls, struct ClientLookupHandle *clh; struct GnsClient *gc = app_ctx; + (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Client %p disconnected\n", client); @@ -314,6 +315,8 @@ client_connect_cb (void *cls, struct GNUNET_MQ_Handle *mq) { struct GnsClient *gc; + + (void) cls; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Client %p connected\n", client); @@ -383,6 +386,7 @@ check_lookup (void *cls, size_t msg_size; const char* name; + (void) cls; msg_size = ntohs (l_msg->header.size); if (msg_size < sizeof (struct LookupMessage)) { @@ -479,6 +483,8 @@ read_service_conf (void *cls, struct GNUNET_CRYPTO_EcdsaPublicKey pk; struct GNS_TopLevelDomain *tld; + (void) cls; + (void) section; if (option[0] != '.') return; if (GNUNET_OK != diff --git a/src/gns/gnunet-service-gns_resolver.c b/src/gns/gnunet-service-gns_resolver.c index 745a2f3bd..92e03bc69 100644 --- a/src/gns/gnunet-service-gns_resolver.c +++ b/src/gns/gnunet-service-gns_resolver.c @@ -501,7 +501,7 @@ static struct CacheOps *co_tail; /** * Use namecache */ -static int use_cache; +static int disable_cache; /** * Global configuration. @@ -2478,7 +2478,7 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh) GNUNET_GNSRECORD_query_from_public_key (&ac->authority_info.gns_authority, ac->label, &query); - if (GNUNET_YES == use_cache) + if (GNUNET_YES != disable_cache) { rh->namecache_qe = GNUNET_NAMECACHE_lookup_block (namecache_handle, @@ -2489,7 +2489,8 @@ recursive_gns_resolution_namecache (struct GNS_ResolverHandle *rh) } else { - start_dht_request (rh, &query); + start_dht_request (rh, + &query); } } @@ -2816,13 +2817,11 @@ GNS_resolver_init (struct GNUNET_NAMECACHE_Handle *nc, dht_lookup_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); max_allowed_background_queries = max_bg_queries; - if (GNUNET_SYSERR == (use_cache = - GNUNET_CONFIGURATION_get_value_yesno (c, - "gns", - "USE_CACHE"))) - use_cache = GNUNET_YES; - if (GNUNET_NO == use_cache) - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + disable_cache = GNUNET_CONFIGURATION_get_value_yesno (cfg, + "namecache", + "DISABLE"); + if (GNUNET_YES == disable_cache) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, "Namecache disabled\n"); vpn_handle = GNUNET_VPN_connect (cfg); } diff --git a/src/identity-provider/gnunet-service-identity-provider.c b/src/identity-provider/gnunet-service-identity-provider.c index 572202c3f..6afb7bb05 100644 --- a/src/identity-provider/gnunet-service-identity-provider.c +++ b/src/identity-provider/gnunet-service-identity-provider.c @@ -2190,12 +2190,15 @@ attr_iter_cb (void *cls, if (rd_count != 1) { - GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); + GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it, + 1); return; } - if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type) { - GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); + if (GNUNET_GNSRECORD_TYPE_ID_ATTR != rd->record_type) + { + GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it, + 1); return; } attr_ver = ntohl(*((uint32_t*)rd->data)); @@ -2209,8 +2212,10 @@ attr_iter_cb (void *cls, rd->data_size-sizeof (uint32_t), key, (void**)&attr_ser); - if (GNUNET_SYSERR == msg_extra_len) { - GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); + if (GNUNET_SYSERR == msg_extra_len) + { + GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it, + 1); return; } @@ -2253,13 +2258,15 @@ iterate_after_abe_bootstrap (void *cls, ai); } -void + +static void iterate_next_after_abe_bootstrap (void *cls, struct GNUNET_ABE_AbeMasterKey *abe_key) { struct AttributeIterator *ai = cls; ai->abe_key = abe_key; - GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it); + GNUNET_NAMESTORE_zone_iterator_next (ai->ns_it, + 1); } diff --git a/src/identity-provider/plugin_rest_openid_connect.c b/src/identity-provider/plugin_rest_openid_connect.c index 7e8054e60..125bb98f6 100644 --- a/src/identity-provider/plugin_rest_openid_connect.c +++ b/src/identity-provider/plugin_rest_openid_connect.c @@ -1076,16 +1076,19 @@ namestore_iteration_callback ( } } - GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it); + GNUNET_NAMESTORE_zone_iterator_next (handle->namestore_handle_it, + 1); } + /** * Iteration over all results finished, build final * response. * * @param cls the `struct RequestHandle` */ -static void namestore_iteration_finished (void *cls) +static void +namestore_iteration_finished (void *cls) { struct RequestHandle *handle = cls; struct GNUNET_HashCode cache_key; diff --git a/src/namecache/namecache.conf.in b/src/namecache/namecache.conf.in index 236cafecd..becd34187 100644 --- a/src/namecache/namecache.conf.in +++ b/src/namecache/namecache.conf.in @@ -11,6 +11,11 @@ ACCEPT_FROM = 127.0.0.1; ACCEPT_FROM6 = ::1; DATABASE = sqlite +# Disables use of caching by GNS. Useful for systems that +# publish very large zones and are CPU bound, if they do not +# also do a large number of lookups. +DISABLE = NO + [namecache-sqlite] FILENAME = $GNUNET_DATA_HOME/namecache/sqlite.db diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c index 06a50132b..b9061ed9f 100644 --- a/src/namestore/gnunet-service-namestore.c +++ b/src/namestore/gnunet-service-namestore.c @@ -257,6 +257,12 @@ static struct GNUNET_NotificationContext *monitor_nc; */ static int cache_keys; +/** + * Use the namecache? Doing so creates additional cryptographic + * operations whenever we touch a record. + */ +static int disable_namecache; + /** * Task run during shutdown. @@ -281,8 +287,11 @@ cleanup_task (void *cls) cop); GNUNET_free (cop); } - GNUNET_NAMECACHE_disconnect (namecache); - namecache = NULL; + if (NULL != namecache) + { + GNUNET_NAMECACHE_disconnect (namecache); + namecache = NULL; + } GNUNET_break (NULL == GNUNET_PLUGIN_unload (db_lib_name, GSN_database)); GNUNET_free (db_lib_name); @@ -715,6 +724,14 @@ refresh_block (struct NamestoreClient *nc, rid); return; /* no data, no need to update cache */ } + if (GNUNET_YES == disable_namecache) + { + GNUNET_STATISTICS_update (statistics, + "Namecache updates skipped (NC disabled)", + 1, + GNUNET_NO); + return; + } exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count, res); if (cache_keys) @@ -1692,13 +1709,20 @@ run (void *cls, (void) service; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting namestore service\n"); - GSN_cfg = cfg; - monitor_nc = GNUNET_notification_context_create (1); - namecache = GNUNET_NAMECACHE_connect (cfg); - /* Loading database plugin */ cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg, "namestore", "CACHE_KEYS"); + disable_namecache = GNUNET_CONFIGURATION_get_value_yesno (cfg, + "namecache", + "DISABLE"); + GSN_cfg = cfg; + monitor_nc = GNUNET_notification_context_create (1); + if (GNUNET_NO == disable_namecache) + { + namecache = GNUNET_NAMECACHE_connect (cfg); + GNUNET_assert (NULL != namecache); + } + /* Loading database plugin */ if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "namestore", diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index 503262487..55779e06a 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c @@ -951,10 +951,10 @@ store_completed_cb (void *cls, delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Processed 1000 records in %s\n", - GNUNET_STRINGS_relative_time_to_string (delta, - GNUNET_YES)); + fprintf (stderr, + "Processed 1000 records in %s\n", + GNUNET_STRINGS_relative_time_to_string (delta, + GNUNET_YES)); } } free_records (req); @@ -1590,10 +1590,10 @@ process_stdin (void *cls) delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Imported 1000 records in %s\n", - GNUNET_STRINGS_relative_time_to_string (delta, - GNUNET_YES)); + fprintf (stderr, + "Imported 1000 records in %s\n", + GNUNET_STRINGS_relative_time_to_string (delta, + GNUNET_YES)); } queue (hn); } diff --git a/src/namestore/plugin_namestore_flat.c b/src/namestore/plugin_namestore_flat.c index e9ed1cc4f..f061ab7d1 100644 --- a/src/namestore/plugin_namestore_flat.c +++ b/src/namestore/plugin_namestore_flat.c @@ -583,7 +583,7 @@ iterate_zones (void *cls, (void) key; if (0 == ic->limit) return GNUNET_NO; - if ( (NULL != it->zone) && + if ( (NULL != ic->zone) && (0 != memcmp (entry->private_key, ic->zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) ) diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index 518d5f572..25baf4396 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c @@ -271,6 +271,14 @@ static void publish_zone_dht_start (void *cls); +/** + * How often do we measure the delta between desired zone + * iteration speed and actual speed, and tell statistics + * service about it? + */ +#define DELTA_INTERVAL 100 + + /** * Continuation called from DHT once the PUT operation is done. * @@ -283,7 +291,11 @@ dht_put_continuation (void *cls, int success) { struct MonitorActivity *ma = cls; + static unsigned long long put_cnt; + static struct GNUNET_TIME_Absolute last_put_100; + static struct GNUNET_TIME_Relative sub_delta; struct GNUNET_TIME_Relative next_put_interval; + struct GNUNET_TIME_Relative delay; num_public_records++; if (NULL == ma) @@ -300,20 +312,91 @@ dht_put_continuation (void *cls, LATE_ITERATION_SPEEDUP_FACTOR); } else + { next_put_interval = put_interval; + } next_put_interval = GNUNET_TIME_relative_min (next_put_interval, MAXIMUM_ZONE_ITERATION_INTERVAL); GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "PUT complete, next PUT in %s!\n", GNUNET_STRINGS_relative_time_to_string (next_put_interval, GNUNET_YES)); - - GNUNET_STATISTICS_set (statistics, - "Current zone iteration interval (ms)", - next_put_interval.rel_value_us / 1000LL, - GNUNET_NO); + /* compute velocities and delay corrections to apply */ + if (0 == put_cnt) + last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */ + put_cnt++; + if (0 == put_cnt % DELTA_INTERVAL) + { + struct GNUNET_TIME_Relative delta; + unsigned long long pct = 0; + + /* How fast were we really? */ + delta = GNUNET_TIME_absolute_get_duration (last_put_100); + delta.rel_value_us /= DELTA_INTERVAL; + last_put_100 = GNUNET_TIME_absolute_get (); + /* Tell statistics actual vs. desired speed */ + GNUNET_STATISTICS_set (statistics, + "Target zone iteration velocity (μs)", + next_put_interval.rel_value_us, + GNUNET_NO); + GNUNET_STATISTICS_set (statistics, + "Current zone iteration velocity (μs)", + delta.rel_value_us, + GNUNET_NO); + /* update "sub_delta" based on difference, taking + previous sub_delta into account! */ + if (next_put_interval.rel_value_us > delta.rel_value_us) + { + /* We were too fast, reduce sub_delta! */ + struct GNUNET_TIME_Relative corr; + + corr = GNUNET_TIME_relative_subtract (next_put_interval, + delta); + if (sub_delta.rel_value_us > delta.rel_value_us) + { + /* Reduce sub_delta by corr */ + sub_delta = GNUNET_TIME_relative_subtract (sub_delta, + corr); + } + else + { + /* We're doing fine with waiting the full time, this + should theoretically only happen if we run at + infinite speed. */ + sub_delta = GNUNET_TIME_UNIT_ZERO; + } + } + else if (next_put_interval.rel_value_us < delta.rel_value_us) + { + /* We were too slow, increase sub_delta! */ + struct GNUNET_TIME_Relative corr; + + corr = GNUNET_TIME_relative_subtract (delta, + next_put_interval); + sub_delta = GNUNET_TIME_relative_add (sub_delta, + corr); + if (sub_delta.rel_value_us > next_put_interval.rel_value_us) + { + /* CPU overload detected, we cannot go at desired speed, + as this would mean using a negative delay. */ + sub_delta = next_put_interval; + /* compute how much faster we would want to be for + the desired velocity */ + if (0 == next_put_interval.rel_value_us) + pct = UINT64_MAX; /* desired speed is infinity ... */ + else + pct = sub_delta.rel_value_us * 100 / next_put_interval.rel_value_us; + } + } + GNUNET_STATISTICS_set (statistics, + "% speed increase needed for target velocity", + pct, + GNUNET_NO); + } /* end of periodic velocity calculations */ + delay = GNUNET_TIME_relative_subtract (next_put_interval, + sub_delta); GNUNET_assert (NULL == zone_publish_task); - zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval, + zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay, &publish_zone_dht_next, NULL); }