{
/* create indices */
if ( (SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_query_hash ON ns096blocks (query,expiration_time)",
NULL, NULL, NULL)) ||
(SQLITE_OK !=
- sqlite3_exec (dbh, "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)",
+ sqlite3_exec (dbh,
+ "CREATE INDEX IF NOT EXISTS ir_block_expiration ON ns096blocks (expiration_time)",
NULL, NULL, NULL)) )
LOG (GNUNET_ERROR_TYPE_ERROR,
- "Failed to create indices: %s\n", sqlite3_errmsg (dbh));
+ "Failed to create indices: %s\n",
+ sqlite3_errmsg (dbh));
}
return GNUNET_SYSERR;
}
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA temp_store=MEMORY", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA temp_store=MEMORY",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA synchronous=NORMAL", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA synchronous=NORMAL",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA legacy_file_format=OFF", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA legacy_file_format=OFF",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA auto_vacuum=INCREMENTAL", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA auto_vacuum=INCREMENTAL",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA encoding=\"UTF-8\"", NULL,
- NULL, ENULL));
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA encoding=\"UTF-8\"",
+ NULL, NULL,
+ ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA locking_mode=EXCLUSIVE", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA locking_mode=EXCLUSIVE",
+ NULL, NULL,
ENULL));
CHECK (SQLITE_OK ==
- sqlite3_exec (plugin->dbh, "PRAGMA page_size=4092", NULL, NULL,
+ sqlite3_exec (plugin->dbh,
+ "PRAGMA page_size=4092",
+ NULL, NULL,
ENULL));
- CHECK (SQLITE_OK == sqlite3_busy_timeout (plugin->dbh, BUSY_TIMEOUT_MS));
+ CHECK (SQLITE_OK ==
+ sqlite3_busy_timeout (plugin->dbh,
+ BUSY_TIMEOUT_MS));
/* Create tables */
sq_prepare (plugin->dbh,
"SELECT 1 FROM sqlite_master WHERE tbl_name = 'ns096blocks'",
&stmt));
- if ((sqlite3_step (stmt) == SQLITE_DONE) &&
- (sqlite3_exec
- (plugin->dbh,
- "CREATE TABLE ns096blocks ("
- " query BLOB NOT NULL,"
- " block BLOB NOT NULL,"
- " expiration_time INT8 NOT NULL"
- ")",
- NULL, NULL, NULL) != SQLITE_OK))
+ if ( (sqlite3_step (stmt) == SQLITE_DONE) &&
+ (SQLITE_OK !=
+ sqlite3_exec (plugin->dbh,
+ "CREATE TABLE ns096blocks ("
+ " query BLOB NOT NULL,"
+ " block BLOB NOT NULL,"
+ " expiration_time INT8 NOT NULL"
+ ")",
+ NULL, NULL, NULL)) )
{
- LOG_SQLITE (plugin, GNUNET_ERROR_TYPE_ERROR, "sqlite3_exec");
+ LOG_SQLITE (plugin,
+ GNUNET_ERROR_TYPE_ERROR,
+ "sqlite3_exec");
sqlite3_finalize (stmt);
return GNUNET_SYSERR;
}
GNUNET_CRYPTO_hash (&block->derived_key,
sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey),
&query);
- fprintf (stderr, // GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Caching new version of block %s (expires %s)\n",
- GNUNET_h2s (&query),
- GNUNET_STRINGS_absolute_time_to_string (expiration));
expiration = GNUNET_TIME_absolute_ntoh (block->expiration_time);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Caching new version of block %s (expires %s)\n",
+ GNUNET_h2s (&query),
+ GNUNET_STRINGS_absolute_time_to_string (expiration));
if (block_size > 64 * 65536)
{
GNUNET_break (0);
&res);
GNUNET_free (nick);
}
-
- exp_time = (0 == res_count)
- ? GNUNET_TIME_UNIT_ZERO_ABS
- : GNUNET_GNSRECORD_record_get_expiration_time (res_count,
- res);
+ if (0 == res_count)
+ {
+ send_store_response (nc,
+ GNUNET_OK,
+ rid);
+ return; /* no data, no need to update cache */
+ }
+ exp_time = GNUNET_GNSRECORD_record_get_expiration_time (res_count,
+ res);
if (cache_keys)
block = GNUNET_GNSRECORD_block_create2 (zone_key,
exp_time,
*/
#define MAX_RETRIES 5
+/**
+ * How many DNS requests do we at most issue in rapid series?
+ */
+#define MAX_SERIES 10
+
+/**
+ * How long do we wait at least between series of requests?
+ */
+#define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
+
/**
* How many requests do we request from NAMESTORE in one batch
* during our initial iteration?
static struct GNUNET_DNSSTUB_Context *ctx;
/**
- * The number of queries that are outstanding
+ * The number of DNS queries that are outstanding
*/
static unsigned int pending;
+/**
+ * The number of NAMESTORE record store operations that are outstanding
+ */
+static unsigned int pending_rs;
+
/**
* Number of lookups we performed overall.
*/
req->hostname))
{
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "DNS returned record for `%s' of type %u while resolving `%s'\n",
+ "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
rec->name,
(unsigned int) rec->type,
req->hostname);
struct Request *req = cls;
req->qe = NULL;
- pending--;
+ pending_rs--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
if (GNUNET_SYSERR == success)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
req_tail,
req);
pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
"Stub gave up on DNS reply for `%s'\n",
req->hostname);
return;
}
if (req->id != dns->id)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "DNS ID did not match request, ignoring reply\n");
return;
+ }
GNUNET_CONTAINER_DLL_remove (req_head,
req_tail,
req);
GNUNET_DNSSTUB_resolve_cancel (req->rs);
req->rs = NULL;
+ pending--;
p = GNUNET_DNSPARSER_parse ((const char *) dns,
dns_len);
if (NULL == p)
{
failures++;
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
insert_sorted (req);
- pending--;
+ if (NULL == t)
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
return;
}
/* import new records */
/* convert linked list into array */
for (rec = req->rec_head; NULL != rec; rec =rec->next)
rd[off++] = rec->grd;
+ pending_rs++;
req->qe = GNUNET_NAMESTORE_records_store (ns,
&req->zone->key,
get_label (req),
rd,
&store_completed_cb,
req);
+ GNUNET_assert (NULL != req->qe);
}
insert_sorted (req);
}
-/**
- * Submit a request to DNS unless we need to slow down because
- * we are at the rate limit.
- *
- * @param req request to submit
- * @return #GNUNET_OK if request was submitted
- * #GNUNET_NO if request was already submitted
- * #GNUNET_SYSERR if we are at the rate limit
- */
-static int
-submit_req (struct Request *req)
-{
- static struct GNUNET_TIME_Absolute last_request;
- struct GNUNET_TIME_Absolute now;
- void *raw;
- size_t raw_size;
-
- if (NULL != req->qe)
- return GNUNET_NO; /* namestore op still pending */
- if (NULL != req->rs)
- {
- GNUNET_break (0);
- return GNUNET_NO; /* already submitted */
- }
- now = GNUNET_TIME_absolute_get ();
- if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
- (pending >= THRESH) )
- return GNUNET_SYSERR;
- GNUNET_CONTAINER_DLL_insert (req_head,
- req_tail,
- req);
- GNUNET_assert (NULL == req->rs);
- GNUNET_log (GNUNET_ERROR_TYPE_INFO,
- "Requesting resolution for `%s'\n",
- req->hostname);
- raw = build_dns_query (req,
- &raw_size);
- if (NULL == raw)
- {
- GNUNET_break (0);
- free_request (req);
- return GNUNET_SYSERR;
- }
- req->rs = GNUNET_DNSSTUB_resolve (ctx,
- raw,
- raw_size,
- &process_result,
- req);
- GNUNET_assert (NULL != req->rs);
- req->issue_num++;
- last_request = now;
- lookups++;
- pending++;
- return GNUNET_OK;
-}
-
-
/**
* Process as many requests as possible from the queue.
*
process_queue (void *cls)
{
struct Request *req;
+ unsigned int series;
+ void *raw;
+ size_t raw_size;
(void) cls;
+ series = 0;
t = NULL;
- while (1)
+ while (pending + pending_rs < THRESH)
{
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
break;
+ if (NULL != req->qe)
+ return; /* namestore op still pending */
+ if (NULL != req->rs)
+ {
+ GNUNET_break (0);
+ return; /* already submitted */
+ }
if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
break;
- if (GNUNET_OK != submit_req (req))
- break;
GNUNET_assert (req ==
GNUNET_CONTAINER_heap_remove_root (req_heap));
req->hn = NULL;
+ GNUNET_CONTAINER_DLL_insert (req_head,
+ req_tail,
+ req);
+ GNUNET_assert (NULL == req->rs);
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ "Requesting resolution for `%s'\n",
+ req->hostname);
+ raw = build_dns_query (req,
+ &raw_size);
+ if (NULL == raw)
+ {
+ GNUNET_break (0);
+ free_request (req);
+ continue;
+ }
+ req->rs = GNUNET_DNSSTUB_resolve (ctx,
+ raw,
+ raw_size,
+ &process_result,
+ req);
+ GNUNET_assert (NULL != req->rs);
+ req->issue_num++;
+ lookups++;
+ pending++;
+ series++;
+ if (series > MAX_SERIES)
+ break;
}
-
+ if (pending + pending_rs >= THRESH)
+ return; /* wait for replies */
req = GNUNET_CONTAINER_heap_peek (req_heap);
if (NULL == req)
return;
t = GNUNET_SCHEDULER_add_at (req->expires,
&process_queue,
NULL);
+ return;
}
- else
- {
- GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Throttling for 1ms\n");
- if (NULL != t)
- GNUNET_SCHEDULER_cancel (t);
- t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
- &process_queue,
- NULL);
- }
+ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
+ "Throttling\n");
+ if (NULL != t)
+ GNUNET_SCHEDULER_cancel (t);
+ t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
+ &process_queue,
+ NULL);
}
GNUNET_assert (NULL != zone_tail);
if (zone_tail == last)
{
- GNUNET_assert (NULL == t);
/* Done iterating over relevant zones in NAMESTORE, move
rest of hash map to work queue as well. */
GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
NULL);
GNUNET_free ((void*) argv);
fprintf (stderr,
- "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n",
+ "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u/%u pending on shutdown\n",
rejects,
lookups,
records,
failures,
- pending);
+ pending,
+ pending_rs);
return 0;
}
*/
static int first_zone_iteration;
+/**
+ * Optimize block insertion by caching map of private keys to
+ * public keys in memory?
+ */
+static int cache_keys;
+
+
/**
* Task run during shutdown.
*
expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
rd_public);
- block = GNUNET_GNSRECORD_block_create (key,
- expire,
- label,
- rd_public,
- rd_public_count);
+ if (cache_keys)
+ block = GNUNET_GNSRECORD_block_create2 (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
+ else
+ block = GNUNET_GNSRECORD_block_create (key,
+ expire,
+ label,
+ rd_public,
+ rd_public_count);
if (NULL == block)
{
GNUNET_break (0);
static void
publish_zone_dht_start (void *cls)
{
+ (void) cls;
zone_publish_task = NULL;
GNUNET_STATISTICS_update (statistics,
"Full zone iterations launched",
GNUNET_SCHEDULER_shutdown ();
return;
}
-
+ cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
+ "namestore",
+ "CACHE_KEYS");
put_interval = INITIAL_PUT_INTERVAL;
zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
if (GNUNET_OK ==
projects / oweals / gnunet.git / commitdiff