From 823215c974ccb1ef3cad9eb1082999cd1b910416 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 26 Apr 2018 16:40:09 +0200 Subject: [PATCH] misc zoneimporter fixes --- src/namecache/plugin_namecache_sqlite.c | 75 +++++---- src/namestore/gnunet-service-namestore.c | 14 +- src/namestore/gnunet-zoneimport.c | 172 +++++++++++---------- src/zonemaster/gnunet-service-zonemaster.c | 29 +++- 4 files changed, 173 insertions(+), 117 deletions(-) diff --git a/src/namecache/plugin_namecache_sqlite.c b/src/namecache/plugin_namecache_sqlite.c index 6f5f2d952..e0f64a6b0 100644 --- a/src/namecache/plugin_namecache_sqlite.c +++ b/src/namecache/plugin_namecache_sqlite.c @@ -136,13 +136,16 @@ create_indices (sqlite3 * dbh) { /* create indices */ if ( (SQLITE_OK != - sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)", + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)", NULL, NULL, NULL)) || (SQLITE_OK != - sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)", + sqlite3_exec (dbh, + "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)", NULL, NULL, NULL)) ) LOG (GNUNET_ERROR_TYPE_ERROR, - "Failed to create indices: %s\n", sqlite3_errmsg (dbh)); + "Failed to create indices: %s\n", + sqlite3_errmsg (dbh)); } @@ -208,28 +211,44 @@ database_setup (struct Plugin *plugin) return GNUNET_SYSERR; } CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL, + sqlite3_exec (plugin->dbh, + "PRAGMA temp_store=MEMORY", + NULL, NULL, ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL, + sqlite3_exec (plugin->dbh, + "PRAGMA synchronous=NORMAL", + NULL, NULL, ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL, + sqlite3_exec (plugin->dbh, + "PRAGMA legacy_file_format=OFF", + NULL, NULL, ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL, - NULL, ENULL)); + sqlite3_exec (plugin->dbh, + "PRAGMA auto_vacuum=INCREMENTAL", + NULL, NULL, + ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL, - NULL, ENULL)); + sqlite3_exec (plugin->dbh, + "PRAGMA encoding=\"UTF-8\"", + NULL, NULL, + ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL, + sqlite3_exec (plugin->dbh, + "PRAGMA locking_mode=EXCLUSIVE", + NULL, NULL, ENULL)); CHECK (SQLITE_OK == - sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL, + sqlite3_exec (plugin->dbh, + "PRAGMA page_size=4092", + NULL, NULL, ENULL)); - CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS)); + CHECK (SQLITE_OK == + sqlite3_busy_timeout (plugin->dbh, + BUSY_TIMEOUT_MS)); /* Create tables */ @@ -237,17 +256,19 @@ database_setup (struct Plugin *plugin) sq_prepare (plugin->dbh, "SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns096blocks'", &stmt)); - if ((sqlite3_step (stmt) == SQLITE_DONE) && - (sqlite3_exec - (plugin->dbh, - "CREATE TABLE ns096blocks (" - " query BLOB NOT NULL," - " block BLOB NOT NULL," - " expiration_time INT8 NOT NULL" - ")", - NULL, NULL, NULL) != SQLITE_OK)) + if ( (sqlite3_step (stmt) == SQLITE_DONE) && + (SQLITE_OK != + sqlite3_exec (plugin->dbh, + "CREATE TABLE ns096blocks (" + " query BLOB NOT NULL," + " block BLOB NOT NULL," + " expiration_time INT8 NOT NULL" + ")", + NULL, NULL, NULL)) ) { - LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec"); + LOG_SQLITE (plugin, + GNUNET_ERROR_TYPE_ERROR, + "sqlite3_exec"); sqlite3_finalize (stmt); return GNUNET_SYSERR; } @@ -411,11 +432,11 @@ namecache_sqlite_cache_block (void *cls, GNUNET_CRYPTO_hash (&block->derived_key, sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey), &query); - fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Caching new version of block %s (expires %s)\n", - GNUNET_h2s (&query), - GNUNET_STRINGS_absolute_time_to_string (expiration)); expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Caching new version of block %s (expires %s)\n", + GNUNET_h2s (&query), + GNUNET_STRINGS_absolute_time_to_string (expiration)); if (block_size > 64 * 65536) { GNUNET_break (0); diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c index 994eae2c8..06a50132b 100644 --- a/src/namestore/gnunet-service-namestore.c +++ b/src/namestore/gnunet-service-namestore.c @@ -708,11 +708,15 @@ refresh_block (struct NamestoreClient *nc, &res); GNUNET_free (nick); } - - exp_time = (0 == res_count) - ? GNUNET_TIME_UNIT_ZERO_ABS - : GNUNET_GNSRECORD_record_get_expiration_time (res_count, - res); + if (0 == res_count) + { + send_store_response (nc, + GNUNET_OK, + rid); + return; /* no data, no need to update cache */ + } + exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count, + res); if (cache_keys) block = GNUNET_GNSRECORD_block_create2 (zone_key, exp_time, diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index 6622a7676..503262487 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c @@ -47,6 +47,16 @@ */ #define MAX_RETRIES 5 +/** + * How many DNS requests do we at most issue in rapid series? + */ +#define MAX_SERIES 10 + +/** + * How long do we wait at least between series of requests? + */ +#define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10) + /** * How many requests do we request from NAMESTORE in one batch * during our initial iteration? @@ -200,10 +210,15 @@ static struct GNUNET_NAMESTORE_Handle *ns; static struct GNUNET_DNSSTUB_Context *ctx; /** - * The number of queries that are outstanding + * The number of DNS queries that are outstanding */ static unsigned int pending; +/** + * The number of NAMESTORE record store operations that are outstanding + */ +static unsigned int pending_rs; + /** * Number of lookups we performed overall. */ @@ -692,7 +707,7 @@ process_record (void *cls, req->hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "DNS returned record for `%s' of type %u while resolving `%s'\n", + "DNS returned record from zone `%s' of type %u while resolving `%s'\n", rec->name, (unsigned int) rec->type, req->hostname); @@ -910,7 +925,11 @@ store_completed_cb (void *cls, struct Request *req = cls; req->qe = NULL; - pending--; + pending_rs--; + if (NULL == t) + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, + &process_queue, + NULL); if (GNUNET_SYSERR == success) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, @@ -967,6 +986,10 @@ process_result (void *cls, req_tail, req); pending--; + if (NULL == t) + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, + &process_queue, + NULL); GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Stub gave up on DNS reply for `%s'\n", req->hostname); @@ -981,12 +1004,17 @@ process_result (void *cls, return; } if (req->id != dns->id) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "DNS ID did not match request, ignoring reply\n"); return; + } GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); GNUNET_DNSSTUB_resolve_cancel (req->rs); req->rs = NULL; + pending--; p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len); if (NULL == p) @@ -998,11 +1026,17 @@ process_result (void *cls, { failures++; insert_sorted (req); - pending--; + if (NULL == t) + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, + &process_queue, + NULL); return; } insert_sorted (req); - pending--; + if (NULL == t) + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, + &process_queue, + NULL); return; } /* import new records */ @@ -1047,6 +1081,7 @@ process_result (void *cls, /* convert linked list into array */ for (rec = req->rec_head; NULL != rec; rec =rec->next) rd[off++] = rec->grd; + pending_rs++; req->qe = GNUNET_NAMESTORE_records_store (ns, &req->zone->key, get_label (req), @@ -1054,68 +1089,12 @@ process_result (void *cls, rd, &store_completed_cb, req); + GNUNET_assert (NULL != req->qe); } insert_sorted (req); } -/** - * Submit a request to DNS unless we need to slow down because - * we are at the rate limit. - * - * @param req request to submit - * @return #GNUNET_OK if request was submitted - * #GNUNET_NO if request was already submitted - * #GNUNET_SYSERR if we are at the rate limit - */ -static int -submit_req (struct Request *req) -{ - static struct GNUNET_TIME_Absolute last_request; - struct GNUNET_TIME_Absolute now; - void *raw; - size_t raw_size; - - if (NULL != req->qe) - return GNUNET_NO; /* namestore op still pending */ - if (NULL != req->rs) - { - GNUNET_break (0); - return GNUNET_NO; /* already submitted */ - } - now = GNUNET_TIME_absolute_get (); - if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) || - (pending >= THRESH) ) - return GNUNET_SYSERR; - GNUNET_CONTAINER_DLL_insert (req_head, - req_tail, - req); - GNUNET_assert (NULL == req->rs); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Requesting resolution for `%s'\n", - req->hostname); - raw = build_dns_query (req, - &raw_size); - if (NULL == raw) - { - GNUNET_break (0); - free_request (req); - return GNUNET_SYSERR; - } - req->rs = GNUNET_DNSSTUB_resolve (ctx, - raw, - raw_size, - &process_result, - req); - GNUNET_assert (NULL != req->rs); - req->issue_num++; - last_request = now; - lookups++; - pending++; - return GNUNET_OK; -} - - /** * Process as many requests as possible from the queue. * @@ -1125,23 +1104,60 @@ static void process_queue (void *cls) { struct Request *req; + unsigned int series; + void *raw; + size_t raw_size; (void) cls; + series = 0; t = NULL; - while (1) + while (pending + pending_rs < THRESH) { req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) break; + if (NULL != req->qe) + return; /* namestore op still pending */ + if (NULL != req->rs) + { + GNUNET_break (0); + return; /* already submitted */ + } if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) break; - if (GNUNET_OK != submit_req (req)) - break; GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap)); req->hn = NULL; + GNUNET_CONTAINER_DLL_insert (req_head, + req_tail, + req); + GNUNET_assert (NULL == req->rs); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Requesting resolution for `%s'\n", + req->hostname); + raw = build_dns_query (req, + &raw_size); + if (NULL == raw) + { + GNUNET_break (0); + free_request (req); + continue; + } + req->rs = GNUNET_DNSSTUB_resolve (ctx, + raw, + raw_size, + &process_result, + req); + GNUNET_assert (NULL != req->rs); + req->issue_num++; + lookups++; + pending++; + series++; + if (series > MAX_SERIES) + break; } - + if (pending + pending_rs >= THRESH) + return; /* wait for replies */ req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) return; @@ -1156,17 +1172,15 @@ process_queue (void *cls) t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL); + return; } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Throttling for 1ms\n"); - if (NULL != t) - GNUNET_SCHEDULER_cancel (t); - t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, - &process_queue, - NULL); - } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Throttling\n"); + if (NULL != t) + GNUNET_SCHEDULER_cancel (t); + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, + &process_queue, + NULL); } @@ -1512,7 +1526,6 @@ iterate_zones (void *cls) GNUNET_assert (NULL != zone_tail); if (zone_tail == last) { - GNUNET_assert (NULL == t); /* Done iterating over relevant zones in NAMESTORE, move rest of hash map to work queue as well. */ GNUNET_CONTAINER_multihashmap_iterate (ns_pending, @@ -1750,12 +1763,13 @@ main (int argc, NULL); GNUNET_free ((void*) argv); fprintf (stderr, - "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n", + "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u/%u pending on shutdown\n", rejects, lookups, records, failures, - pending); + pending, + pending_rs); return 0; } diff --git a/src/zonemaster/gnunet-service-zonemaster.c b/src/zonemaster/gnunet-service-zonemaster.c index f40f25c3f..518d5f572 100644 --- a/src/zonemaster/gnunet-service-zonemaster.c +++ b/src/zonemaster/gnunet-service-zonemaster.c @@ -180,6 +180,13 @@ static struct GNUNET_SCHEDULER_Task *zone_publish_task; */ static int first_zone_iteration; +/** + * Optimize block insertion by caching map of private keys to + * public keys in memory? + */ +static int cache_keys; + + /** * Task run during shutdown. * @@ -388,11 +395,18 @@ perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count, rd_public); - block = GNUNET_GNSRECORD_block_create (key, - expire, - label, - rd_public, - rd_public_count); + if (cache_keys) + block = GNUNET_GNSRECORD_block_create2 (key, + expire, + label, + rd_public, + rd_public_count); + else + block = GNUNET_GNSRECORD_block_create (key, + expire, + label, + rd_public, + rd_public_count); if (NULL == block) { GNUNET_break (0); @@ -585,6 +599,7 @@ put_gns_record (void *cls, static void publish_zone_dht_start (void *cls) { + (void) cls; zone_publish_task = NULL; GNUNET_STATISTICS_update (statistics, "Full zone iterations launched", @@ -736,7 +751,9 @@ run (void *cls, GNUNET_SCHEDULER_shutdown (); return; } - + cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c, + "namestore", + "CACHE_KEYS"); put_interval = INITIAL_PUT_INTERVAL; zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW; if (GNUNET_OK == -- 2.25.1