X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fnamestore%2Fgnunet-zoneimport.c;h=bbbe7d78389b3d67bb3900f45f6e0f937bcbe9c5;hb=1d468ecabd6c2ee5c0eae672292efa0f51bc9e48;hp=c26d33bad937f52967536a69b5e5cb1c686930a8;hpb=72b802f9173bdd0f5d6830c56d7c51c3349073fe;p=oweals%2Fgnunet.git diff --git a/src/namestore/gnunet-zoneimport.c b/src/namestore/gnunet-zoneimport.c index c26d33bad..bbbe7d783 100644 --- a/src/namestore/gnunet-zoneimport.c +++ b/src/namestore/gnunet-zoneimport.c @@ -2,21 +2,21 @@ This file is part of GNUnet Copyright (C) 2018 GNUnet e.V. - GNUnet is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 3, or (at your - option) any later version. + GNUnet is free software: you can redistribute it and/or modify it + under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, + or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - General Public License for more details. + Affero General Public License for more details. - You should have received a copy of the GNU General Public License - along with GNUnet; see the file COPYING. If not, write to the - Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, - Boston, MA 02110-1301, USA. -*/ + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + + SPDX-License-Identifier: AGPL3.0-or-later + */ /** * @file src/namestore/gnunet-zoneimport.c * @brief import a DNS zone for publication in GNS, incremental @@ -28,6 +28,7 @@ #include #include #include +#include #include @@ -47,6 +48,27 @@ */ #define MAX_RETRIES 5 +/** + * How many DNS requests do we at most issue in rapid series? + */ +#define MAX_SERIES 10 + +/** + * How long do we wait at least between series of requests? + */ +#define SERIES_DELAY \ + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10) + +/** + * How long do DNS records have to last at least after being imported? + */ +static struct GNUNET_TIME_Relative minimum_expiration_time; + +/** + * How many requests do we request from NAMESTORE in one batch + * during our initial iteration? + */ +#define NS_BATCH_SIZE 1024 /** * Some zones may include authoritative records for other @@ -56,7 +78,6 @@ */ struct Zone { - /** * Kept in a DLL. */ @@ -76,7 +97,6 @@ struct Zone * Private key of the zone. */ struct GNUNET_CRYPTO_EcdsaPrivateKey key; - }; @@ -99,7 +119,6 @@ struct Record * GNS record. */ struct GNUNET_GNSRECORD_Data grd; - }; @@ -166,6 +185,14 @@ struct Request */ struct GNUNET_TIME_Absolute expires; + /** + * While we are fetching the record, the value is set to the + * starting time of the DNS operation. While doing a + * NAMESTORE store, again set to the start time of the + * NAMESTORE operation. + */ + struct GNUNET_TIME_Absolute op_start_time; + /** * How often did we issue this query? (And failed, reset * to zero once we were successful.) @@ -179,6 +206,14 @@ struct Request }; +/** + * Command-line argument specifying desired size of the hash map with + * all of our pending names. Usually, we use an automatically growing + * map, but this is only OK up to about a million entries. Above that + * number, the user must explicitly specify the size at startup. + */ +static unsigned int map_size = 1024; + /** * Handle to the identity service. */ @@ -189,21 +224,36 @@ static struct GNUNET_IDENTITY_Handle *id; */ static struct GNUNET_NAMESTORE_Handle *ns; +/** + * Handle to the statistics service. + */ +static struct GNUNET_STATISTICS_Handle *stats; + /** * Context for DNS resolution. */ static struct GNUNET_DNSSTUB_Context *ctx; /** - * The number of queries that are outstanding + * The number of DNS queries that are outstanding */ static unsigned int pending; +/** + * The number of NAMESTORE record store operations that are outstanding + */ +static unsigned int pending_rs; + /** * Number of lookups we performed overall. */ static unsigned int lookups; +/** + * Number of records we had cached. + */ +static unsigned int cached; + /** * How many hostnames did we reject (malformed). */ @@ -220,9 +270,9 @@ static unsigned int failures; static unsigned int records; /** - * #GNUNET_YES if we have more work to be read from `stdin`. + * Number of record sets given to namestore. */ -static int stdin_waiting; +static unsigned int record_sets; /** * Heap of all requests to perform, sorted by @@ -245,6 +295,18 @@ static struct Request *req_tail; */ static struct GNUNET_SCHEDULER_Task *t; +/** + * Hash map of requests for which we may still get a response from + * the namestore. Set to NULL once the initial namestore iteration + * is done. + */ +static struct GNUNET_CONTAINER_MultiHashMap *ns_pending; + +/** + * Current zone iteration handle. + */ +static struct GNUNET_NAMESTORE_ZoneIterator *zone_it; + /** * Head of list of zones we are managing. */ @@ -255,6 +317,47 @@ static struct Zone *zone_head; */ static struct Zone *zone_tail; +/** + * After how many more results must #ns_lookup_result_cb() ask + * the namestore for more? + */ +static uint64_t ns_iterator_trigger_next; + +/** + * Number of DNS requests counted in latency total. + */ +static uint64_t total_dns_latency_cnt; + +/** + * Sum of DNS latencies observed. + */ +static struct GNUNET_TIME_Relative total_dns_latency; + +/** + * Number of records processed (DNS lookup, no NAMESTORE) in total. + */ +static uint64_t total_reg_proc_dns; + +/** + * Number of records processed (DNS lookup, with NAMESTORE) in total. + */ +static uint64_t total_reg_proc_dns_ns; + +/** + * Start time of the regular processing. + */ +static struct GNUNET_TIME_Absolute start_time_reg_proc; + +/** + * Last time we worked before going idle. + */ +static struct GNUNET_TIME_Absolute sleep_time_reg_proc; + +/** + * Time we slept just waiting for work. + */ +static struct GNUNET_TIME_Relative idle_time; + /** * Callback for #for_all_records @@ -262,9 +365,8 @@ static struct Zone *zone_tail; * @param cls closure * @param rec a DNS record */ -typedef void -(*RecordProcessor) (void *cls, - const struct GNUNET_DNSPARSER_Record *rec); +typedef void (*RecordProcessor) (void *cls, + const struct GNUNET_DNSPARSER_Record *rec); /** @@ -277,29 +379,26 @@ typedef void */ static void for_all_records (const struct GNUNET_DNSPARSER_Packet *p, - RecordProcessor rp, - void *rp_cls) + RecordProcessor rp, + void *rp_cls) { - for (unsigned int i=0;inum_answers;i++) + for (unsigned int i = 0; i < p->num_answers; i++) { struct GNUNET_DNSPARSER_Record *rs = &p->answers[i]; - rp (rp_cls, - rs); + rp (rp_cls, rs); } - for (unsigned int i=0;inum_authority_records;i++) + for (unsigned int i = 0; i < p->num_authority_records; i++) { struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i]; - rp (rp_cls, - rs); + rp (rp_cls, rs); } - for (unsigned int i=0;inum_additional_records;i++) + for (unsigned int i = 0; i < p->num_additional_records; i++) { struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i]; - rp (rp_cls, - rs); + rp (rp_cls, rs); } } @@ -317,21 +416,18 @@ get_label (struct Request *req) static char label[64]; const char *dot; - dot = strchr (req->hostname, - (unsigned char) '.'); + dot = strchr (req->hostname, (unsigned char) '.'); if (NULL == dot) { GNUNET_break (0); return NULL; } - if (((size_t) (dot - req->hostname)) >= sizeof (label)) + if (((size_t) (dot - req->hostname)) >= sizeof(label)) { GNUNET_break (0); return NULL; } - memcpy (label, - req->hostname, - dot - req->hostname); + GNUNET_memcpy (label, req->hostname, dot - req->hostname); label[dot - req->hostname] = '\0'; return label; } @@ -346,37 +442,34 @@ get_label (struct Request *req) * allocated query buffer */ static void * -build_dns_query (struct Request *req, - size_t *raw_size) +build_dns_query (struct Request *req, size_t *raw_size) { static char raw[512]; char *rawp; struct GNUNET_DNSPARSER_Packet p; struct GNUNET_DNSPARSER_Query q; + int ret; q.name = (char *) req->hostname; q.type = GNUNET_DNSPARSER_TYPE_NS; q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET; - memset (&p, - 0, - sizeof (p)); + memset (&p, 0, sizeof(p)); p.num_queries = 1; p.queries = &q; p.id = req->id; - if (GNUNET_OK != - GNUNET_DNSPARSER_pack (&p, - UINT16_MAX, - &rawp, - raw_size)) + ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size); + if (GNUNET_OK != ret) { + if (GNUNET_NO == ret) + GNUNET_free (rawp); GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to pack query for hostname `%s'\n", req->hostname); rejects++; return NULL; } - if (*raw_size > sizeof (raw)) + if (*raw_size > sizeof(raw)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to pack query for hostname `%s'\n", @@ -386,31 +479,40 @@ build_dns_query (struct Request *req, GNUNET_free (rawp); return NULL; } - memcpy (raw, - rawp, - *raw_size); + GNUNET_memcpy (raw, rawp, *raw_size); GNUNET_free (rawp); return raw; } /** - * Free @a req and data structures reachable from it. + * Free records associated with @a req. * - * @param req request to free + * @param req request to free records of */ static void -free_request (struct Request *req) +free_records (struct Request *req) { struct Record *rec; + /* Free records */ while (NULL != (rec = req->rec_head)) { - GNUNET_CONTAINER_DLL_remove (req->rec_head, - req->rec_tail, - rec); + GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec); GNUNET_free (rec); } +} + + +/** + * Free @a req and data structures reachable from it. + * + * @param req request to free + */ +static void +free_request (struct Request *req) +{ + free_records (req); GNUNET_free (req); } @@ -432,16 +534,14 @@ process_queue (void *cls); static void insert_sorted (struct Request *req) { - req->hn = GNUNET_CONTAINER_heap_insert (req_heap, - req, - req->expires.abs_value_us); + req->hn = + GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us); if (req == GNUNET_CONTAINER_heap_peek (req_heap)) { if (NULL != t) GNUNET_SCHEDULER_cancel (t); - t = GNUNET_SCHEDULER_add_at (req->expires, - &process_queue, - NULL); + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL); } } @@ -457,25 +557,21 @@ insert_sorted (struct Request *req) */ static void add_record (struct Request *req, - uint32_t type, - struct GNUNET_TIME_Absolute expiration_time, - const void *data, - size_t data_len) + uint32_t type, + struct GNUNET_TIME_Absolute expiration_time, + const void *data, + size_t data_len) { struct Record *rec; - rec = GNUNET_malloc (sizeof (struct Record) + data_len); + rec = GNUNET_malloc (sizeof(struct Record) + data_len); rec->grd.data = &rec[1]; rec->grd.expiration_time = expiration_time.abs_value_us; rec->grd.data_size = data_len; rec->grd.record_type = type; rec->grd.flags = GNUNET_GNSRECORD_RF_NONE; - GNUNET_memcpy (&rec[1], - data, - data_len); - GNUNET_CONTAINER_DLL_insert (req->rec_head, - req->rec_tail, - rec); + GNUNET_memcpy (&rec[1], data, data_len); + GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec); } @@ -508,111 +604,104 @@ struct GlueClosure * @param rec record that may contain glue information */ static void -check_for_glue (void *cls, - const struct GNUNET_DNSPARSER_Record *rec) +check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec) { struct GlueClosure *gc = cls; char dst[65536]; size_t dst_len; size_t off; - char ip[INET6_ADDRSTRLEN+1]; - socklen_t ip_size = (socklen_t) sizeof (ip); + char ip[INET6_ADDRSTRLEN + 1]; + socklen_t ip_size = (socklen_t) sizeof(ip); + struct GNUNET_TIME_Absolute expiration_time; + struct GNUNET_TIME_Relative left; - if (0 != strcasecmp (rec->name, - gc->ns)) + if (0 != strcasecmp (rec->name, gc->ns)) return; - dst_len = sizeof (dst); + expiration_time = rec->expiration_time; + left = GNUNET_TIME_absolute_get_remaining (expiration_time); + if (0 == left.rel_value_us) + return; /* ignore expired glue records */ + /* if expiration window is too short, bump it to configured minimum */ + if (left.rel_value_us < minimum_expiration_time.rel_value_us) + expiration_time = + GNUNET_TIME_relative_to_absolute (minimum_expiration_time); + dst_len = sizeof(dst); off = 0; switch (rec->type) { case GNUNET_DNSPARSER_TYPE_A: - if (sizeof (struct in_addr) != rec->data.raw.data_len) + if (sizeof(struct in_addr) != rec->data.raw.data_len) { GNUNET_break (0); return; } - if (NULL == - inet_ntop (AF_INET, - rec->data.raw.data, - ip, - ip_size)) + if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size)) { GNUNET_break (0); return; } - if ( (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - gc->req->hostname)) && - (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - ip)) ) + if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + gc->req->hostname)) && + (GNUNET_OK == + GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip))) { add_record (gc->req, - GNUNET_GNSRECORD_TYPE_GNS2DNS, - rec->expiration_time, - dst, - off); + GNUNET_GNSRECORD_TYPE_GNS2DNS, + expiration_time, + dst, + off); gc->found = GNUNET_YES; } break; + case GNUNET_DNSPARSER_TYPE_AAAA: - if (sizeof (struct in6_addr) != rec->data.raw.data_len) + if (sizeof(struct in6_addr) != rec->data.raw.data_len) { GNUNET_break (0); return; } - if (NULL == - inet_ntop (AF_INET6, - rec->data.raw.data, - ip, - ip_size)) + if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size)) { GNUNET_break (0); return; } - if ( (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - gc->req->hostname)) && - (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - ip)) ) + if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + gc->req->hostname)) && + (GNUNET_OK == + GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip))) { add_record (gc->req, - GNUNET_GNSRECORD_TYPE_GNS2DNS, - rec->expiration_time, - dst, - off); + GNUNET_GNSRECORD_TYPE_GNS2DNS, + expiration_time, + dst, + off); gc->found = GNUNET_YES; } break; + case GNUNET_DNSPARSER_TYPE_CNAME: - if ( (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - gc->req->hostname)) && - (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - rec->data.hostname)) ) + if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + gc->req->hostname)) && + (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + rec->data.hostname))) { add_record (gc->req, - GNUNET_GNSRECORD_TYPE_GNS2DNS, - rec->expiration_time, - dst, - off); + GNUNET_GNSRECORD_TYPE_GNS2DNS, + expiration_time, + dst, + off); gc->found = GNUNET_YES; } break; + default: /* useless, do nothing */ break; @@ -645,100 +734,107 @@ struct ProcessRecordContext * @param rec response */ static void -process_record (void *cls, - const struct GNUNET_DNSPARSER_Record *rec) +process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec) { struct ProcessRecordContext *prc = cls; struct Request *req = prc->req; char dst[65536]; size_t dst_len; size_t off; + struct GNUNET_TIME_Absolute expiration_time; + struct GNUNET_TIME_Relative left; - dst_len = sizeof (dst); + dst_len = sizeof(dst); off = 0; records++; - if (0 != strcasecmp (rec->name, - req->hostname)) + if (0 != strcasecmp (rec->name, req->hostname)) { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "DNS returned record for `%s' of type %u while resolving `%s'\n", - rec->name, - (unsigned int) rec->type, - req->hostname); - return; /* does not match hostname, might be glue, but - not useful for this pass! */ + GNUNET_log ( + GNUNET_ERROR_TYPE_DEBUG, + "DNS returned record from zone `%s' of type %u while resolving `%s'\n", + rec->name, + (unsigned int) rec->type, + req->hostname); + return; /* does not match hostname, might be glue, but + not useful for this pass! */ } - if (0 == - GNUNET_TIME_absolute_get_remaining (rec->expiration_time).rel_value_us) + expiration_time = rec->expiration_time; + left = GNUNET_TIME_absolute_get_remaining (expiration_time); + if (0 == left.rel_value_us) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "DNS returned expired record for `%s'\n", - req->hostname); - return; /* record expired */ + "DNS returned expired record for `%s'\n", + req->hostname); + GNUNET_STATISTICS_update (stats, + "# expired records obtained from DNS", + 1, + GNUNET_NO); + return; /* record expired */ } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "DNS returned record that expires at %s for `%s'\n", + GNUNET_STRINGS_absolute_time_to_string (expiration_time), + req->hostname); + /* if expiration window is too short, bump it to configured minimum */ + if (left.rel_value_us < minimum_expiration_time.rel_value_us) + expiration_time = + GNUNET_TIME_relative_to_absolute (minimum_expiration_time); switch (rec->type) { - case GNUNET_DNSPARSER_TYPE_NS: - { + case GNUNET_DNSPARSER_TYPE_NS: { struct GlueClosure gc; /* check for glue */ gc.req = req; gc.ns = rec->data.hostname; gc.found = GNUNET_NO; - for_all_records (prc->p, - &check_for_glue, - &gc); - if ( (GNUNET_NO == gc.found) && - (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - req->hostname)) && - (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - rec->data.hostname)) ) + for_all_records (prc->p, &check_for_glue, &gc); + if ((GNUNET_NO == gc.found) && + (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + req->hostname)) && + (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + rec->data.hostname))) { - /* FIXME: actually check if this is out-of-bailiwick, - and if not request explicit resolution... */ - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converted OOB (`%s') NS record for `%s'\n", - rec->data.hostname, - rec->name); - add_record (req, - GNUNET_GNSRECORD_TYPE_GNS2DNS, - rec->expiration_time, - dst, - off); + /* FIXME: actually check if this is out-of-bailiwick, + and if not request explicit resolution... */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Converted OOB (`%s') NS record for `%s'\n", + rec->data.hostname, + rec->name); + add_record (req, + GNUNET_GNSRECORD_TYPE_GNS2DNS, + expiration_time, + dst, + off); } else { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converted NS record for `%s' using glue\n", - rec->name); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Converted NS record for `%s' using glue\n", + rec->name); } break; } + case GNUNET_DNSPARSER_TYPE_CNAME: - if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - rec->data.hostname)) + if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + rec->data.hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting CNAME (`%s') record for `%s'\n", - rec->data.hostname, - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting CNAME (`%s') record for `%s'\n", + rec->data.hostname, + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; + case GNUNET_DNSPARSER_TYPE_DNAME: /* No support for DNAME in GNS yet! FIXME: support later! */ GNUNET_log (GNUNET_ERROR_TYPE_WARNING, @@ -746,110 +842,83 @@ process_record (void *cls, rec->name, rec->data.hostname); break; + case GNUNET_DNSPARSER_TYPE_MX: if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_mx (dst, - dst_len, - &off, - rec->data.mx)) + GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting MX (`%s') record for `%s'\n", - rec->data.mx->mxhost, - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting MX (`%s') record for `%s'\n", + rec->data.mx->mxhost, + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; + case GNUNET_DNSPARSER_TYPE_SOA: if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_soa (dst, - dst_len, - &off, - rec->data.soa)) + GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa)) { /* NOTE: GNS does not really use SOAs */ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting SOA record for `%s'\n", - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting SOA record for `%s'\n", + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; + case GNUNET_DNSPARSER_TYPE_SRV: if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_srv (dst, - dst_len, - &off, - rec->data.srv)) + GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting SRV record for `%s'\n", - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting SRV record for `%s'\n", + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; + case GNUNET_DNSPARSER_TYPE_PTR: - if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_name (dst, - dst_len, - &off, - rec->data.hostname)) + if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst, + dst_len, + &off, + rec->data.hostname)) { /* !?: what does a PTR record do in a regular TLD??? */ GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting PTR record for `%s' (weird)\n", - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting PTR record for `%s' (weird)\n", + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; + case GNUNET_DNSPARSER_TYPE_CERT: if (GNUNET_OK == - GNUNET_DNSPARSER_builder_add_cert (dst, - dst_len, - &off, - rec->data.cert)) + GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert)) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting CERT record for `%s'\n", - rec->name); - add_record (req, - rec->type, - rec->expiration_time, - dst, - off); + "Converting CERT record for `%s'\n", + rec->name); + add_record (req, rec->type, expiration_time, dst, off); } break; - /* Rest is 'raw' encoded and just needs to be copied IF - the hostname matches the requested name; otherwise we - simply cannot use it. */ + + /* Rest is 'raw' encoded and just needs to be copied IF + the hostname matches the requested name; otherwise we + simply cannot use it. */ case GNUNET_DNSPARSER_TYPE_A: case GNUNET_DNSPARSER_TYPE_AAAA: case GNUNET_DNSPARSER_TYPE_TXT: default: GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Converting record of type %u for `%s'\n", - (unsigned int) rec->type, - rec->name); + "Converting record of type %u for `%s'\n", + (unsigned int) rec->type, + rec->name); add_record (req, - rec->type, - rec->expiration_time, - rec->data.raw.data, - rec->data.raw.data_len); + rec->type, + expiration_time, + rec->data.raw.data, + rec->data.raw.data_len); break; } } @@ -866,51 +935,88 @@ process_record (void *cls, * @param emsg NULL on success, otherwise an error message */ static void -store_completed_cb (void *cls, - int32_t success, - const char *emsg) +store_completed_cb (void *cls, int32_t success, const char *emsg) { static struct GNUNET_TIME_Absolute last; - static unsigned int pdot; struct Request *req = cls; - struct Record *rec; req->qe = NULL; - pending--; if (GNUNET_SYSERR == success) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Failed to store zone data for `%s': %s\n", - req->hostname, - emsg); + "Failed to store zone data for `%s': %s\n", + req->hostname, + emsg); } else { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Stored records under `%s'\n", - req->hostname); - if (0 == pdot) + "Stored records under `%s' (%d)\n", + req->hostname, + success); + } + total_reg_proc_dns_ns++; /* finished regular processing */ + pending_rs--; + free_records (req); + /* compute NAMESTORE statistics */ + { + static uint64_t total_ns_latency_cnt; + static struct GNUNET_TIME_Relative total_ns_latency; + struct GNUNET_TIME_Relative ns_latency; + + ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); + total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency); + if (0 == total_ns_latency_cnt) last = GNUNET_TIME_absolute_get (); - pdot++; - if (0 == pdot % 1000) + total_ns_latency_cnt++; + if (0 == (total_ns_latency_cnt % 1000)) { struct GNUNET_TIME_Relative delta; delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Processed 1000 records in %s\n", - GNUNET_STRINGS_relative_time_to_string (delta, - GNUNET_YES)); + fprintf (stderr, + "Processed 1000 records in %s\n", + GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES)); + GNUNET_STATISTICS_set (stats, + "# average NAMESTORE PUT latency (μs)", + total_ns_latency.rel_value_us + / total_ns_latency_cnt, + GNUNET_NO); } } - /* Free records */ - while (NULL != (rec = req->rec_head)) + /* compute and publish overall velocity */ + if (0 == (total_reg_proc_dns_ns % 100)) { - GNUNET_CONTAINER_DLL_remove (req->rec_head, - req->rec_tail, - rec); - GNUNET_free (rec); + struct GNUNET_TIME_Relative runtime; + + runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc); + runtime = GNUNET_TIME_relative_subtract (runtime, idle_time); + runtime = + GNUNET_TIME_relative_divide (runtime, + total_reg_proc_dns + total_reg_proc_dns_ns); + GNUNET_STATISTICS_set (stats, + "# Regular processing completed without NAMESTORE", + total_reg_proc_dns, + GNUNET_NO); + GNUNET_STATISTICS_set (stats, + "# Regular processing completed with NAMESTORE PUT", + total_reg_proc_dns_ns, + GNUNET_NO); + GNUNET_STATISTICS_set (stats, + "# average request processing latency (μs)", + runtime.rel_value_us, + GNUNET_NO); + GNUNET_STATISTICS_set (stats, + "# total time spent idle (μs)", + idle_time.rel_value_us, + GNUNET_NO); + } + + if (NULL == t) + { + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_now (&process_queue, NULL); } } @@ -936,159 +1042,138 @@ process_result (void *cls, if (NULL == dns) { /* stub gave up */ - GNUNET_CONTAINER_DLL_remove (req_head, - req_tail, - req); + GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); pending--; + if (NULL == t) + { + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_now (&process_queue, NULL); + } GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Stub gave up on DNS reply for `%s'\n", req->hostname); + GNUNET_STATISTICS_update (stats, "# DNS lookups timed out", 1, GNUNET_NO); if (req->issue_num > MAX_RETRIES) { failures++; free_request (req); + GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO); return; } + total_reg_proc_dns++; req->rs = NULL; insert_sorted (req); return; } if (req->id != dns->id) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "DNS ID did not match request, ignoring reply\n"); + GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO); return; - GNUNET_CONTAINER_DLL_remove (req_head, - req_tail, - req); + } + GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); GNUNET_DNSSTUB_resolve_cancel (req->rs); req->rs = NULL; - p = GNUNET_DNSPARSER_parse ((const char *) dns, - dns_len); + pending--; + p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len); if (NULL == p) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Failed to parse DNS reply for `%s'\n", req->hostname); + GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO); + if (NULL == t) + { + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_now (&process_queue, NULL); + } if (req->issue_num > MAX_RETRIES) { failures++; - insert_sorted (req); - pending--; + free_request (req); + GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO); return; } insert_sorted (req); - pending--; return; } /* import new records */ req->issue_num = 0; /* success, reset counter! */ { - struct ProcessRecordContext prc = { - .req = req, - .p = p - }; - - for_all_records (p, - &process_record, - &prc); + struct ProcessRecordContext prc = { .req = req, .p = p }; + + for_all_records (p, &process_record, &prc); } GNUNET_DNSPARSER_free_packet (p); /* count records found, determine minimum expiration time */ req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; + { + struct GNUNET_TIME_Relative dns_latency; + + dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time); + total_dns_latency = + GNUNET_TIME_relative_add (total_dns_latency, dns_latency); + total_dns_latency_cnt++; + if (0 == (total_dns_latency_cnt % 1000)) + { + GNUNET_STATISTICS_set (stats, + "# average DNS lookup latency (μs)", + total_dns_latency.rel_value_us + / total_dns_latency_cnt, + GNUNET_NO); + } + } rd_count = 0; for (rec = req->rec_head; NULL != rec; rec = rec->next) { struct GNUNET_TIME_Absolute at; at.abs_value_us = rec->grd.expiration_time; - req->expires = GNUNET_TIME_absolute_min (req->expires, - at); + req->expires = GNUNET_TIME_absolute_min (req->expires, at); rd_count++; } GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Obtained %u records for `%s'\n", - rd_count, - req->hostname); + "Obtained %u records for `%s'\n", + rd_count, + req->hostname); /* Instead of going for SOA, simplified for now to look each day in case we got an empty response */ if (0 == rd_count) - req->expires - = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); + { + req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS); + GNUNET_STATISTICS_update (stats, + "# empty DNS replies (usually NXDOMAIN)", + 1, + GNUNET_NO); + } + else + { + record_sets++; + } /* convert records to namestore import format */ { - struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)]; + struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)]; unsigned int off = 0; /* convert linked list into array */ - for (rec = req->rec_head; NULL != rec; rec =rec->next) + for (rec = req->rec_head; NULL != rec; rec = rec->next) rd[off++] = rec->grd; + pending_rs++; + req->op_start_time = GNUNET_TIME_absolute_get (); req->qe = GNUNET_NAMESTORE_records_store (ns, - &req->zone->key, - get_label (req), - rd_count, - rd, - &store_completed_cb, - req); + &req->zone->key, + get_label (req), + rd_count, + rd, + &store_completed_cb, + req); + GNUNET_assert (NULL != req->qe); } insert_sorted (req); } -/** - * Submit a request to DNS unless we need to slow down because - * we are at the rate limit. - * - * @param req request to submit - * @return #GNUNET_OK if request was submitted - * #GNUNET_NO if request was already submitted - * #GNUNET_SYSERR if we are at the rate limit - */ -static int -submit_req (struct Request *req) -{ - static struct GNUNET_TIME_Absolute last_request; - struct GNUNET_TIME_Absolute now; - void *raw; - size_t raw_size; - - if (NULL != req->qe) - return GNUNET_NO; /* namestore op still pending */ - if (NULL != req->rs) - { - GNUNET_break (0); - return GNUNET_NO; /* already submitted */ - } - now = GNUNET_TIME_absolute_get (); - if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) || - (pending >= THRESH) ) - return GNUNET_SYSERR; - GNUNET_CONTAINER_DLL_insert (req_head, - req_tail, - req); - GNUNET_assert (NULL == req->rs); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Requesting resolution for `%s'\n", - req->hostname); - raw = build_dns_query (req, - &raw_size); - if (NULL == raw) - { - GNUNET_break (0); - free_request (req); - return GNUNET_SYSERR; - } - req->rs = GNUNET_DNSSTUB_resolve (ctx, - raw, - raw_size, - &process_result, - req); - GNUNET_assert (NULL != req->rs); - req->issue_num++; - last_request = now; - lookups++; - pending++; - return GNUNET_OK; -} - - /** * Process as many requests as possible from the queue. * @@ -1098,48 +1183,108 @@ static void process_queue (void *cls) { struct Request *req; + unsigned int series; + void *raw; + size_t raw_size; + struct GNUNET_TIME_Relative delay; (void) cls; + delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc); + idle_time = GNUNET_TIME_relative_add (idle_time, delay); + series = 0; t = NULL; - while (1) + while (pending + pending_rs < THRESH) { req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) break; + if (NULL != req->qe) + return; /* namestore op still pending */ + if (NULL != req->rs) + { + GNUNET_break (0); + return; /* already submitted */ + } if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) break; - if (GNUNET_OK != submit_req (req)) - break; - GNUNET_assert (req == - GNUNET_CONTAINER_heap_remove_root (req_heap)); + GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap)); req->hn = NULL; + GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req); + GNUNET_assert (NULL == req->rs); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Requesting resolution for `%s'\n", + req->hostname); + raw = build_dns_query (req, &raw_size); + if (NULL == raw) + { + GNUNET_break (0); + free_request (req); + continue; + } + req->op_start_time = GNUNET_TIME_absolute_get (); + req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req); + GNUNET_assert (NULL != req->rs); + req->issue_num++; + lookups++; + pending++; + series++; + if (series > MAX_SERIES) + break; + } + if (pending + pending_rs >= THRESH) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Stopped processing queue (%u+%u/%u)]\n", + pending, + pending_rs, + THRESH); + return; /* wait for replies */ } - req = GNUNET_CONTAINER_heap_peek (req_heap); if (NULL == req) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Stopped processing queue: empty queue\n"); return; + } if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Waiting until %s for next record (`%s') to expire\n", - GNUNET_STRINGS_absolute_time_to_string (req->expires), - req->hostname); - if (NULL != t) - GNUNET_SCHEDULER_cancel (t); - t = GNUNET_SCHEDULER_add_at (req->expires, - &process_queue, - NULL); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Throttling for 1ms\n"); + "Waiting until %s for next record (`%s') to expire\n", + GNUNET_STRINGS_absolute_time_to_string (req->expires), + req->hostname); if (NULL != t) GNUNET_SCHEDULER_cancel (t); - t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS, - &process_queue, - NULL); + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL); + return; } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n"); + if (NULL != t) + GNUNET_SCHEDULER_cancel (t); + sleep_time_reg_proc = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL); +} + + +/** + * Iterator called during #do_shutdown() to free requests in + * the #ns_pending map. + * + * @param cls NULL + * @param key unused + * @param value the `struct Request` to free + * @return #GNUNET_OK + */ +static int +free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value) +{ + struct Request *req = value; + + (void) cls; + (void) key; + free_request (req); + return GNUNET_OK; } @@ -1167,9 +1312,7 @@ do_shutdown (void *cls) } while (NULL != (req = req_head)) { - GNUNET_CONTAINER_DLL_remove (req_head, - req_tail, - req); + GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req); if (NULL != req->qe) GNUNET_NAMESTORE_cancel (req->qe); free_request (req); @@ -1181,6 +1324,11 @@ do_shutdown (void *cls) GNUNET_NAMESTORE_cancel (req->qe); free_request (req); } + if (NULL != zone_it) + { + GNUNET_NAMESTORE_zone_iteration_stop (zone_it); + zone_it = NULL; + } if (NULL != ns) { GNUNET_NAMESTORE_disconnect (ns); @@ -1196,135 +1344,157 @@ do_shutdown (void *cls) GNUNET_CONTAINER_heap_destroy (req_heap); req_heap = NULL; } + if (NULL != ns_pending) + { + GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL); + GNUNET_CONTAINER_multihashmap_destroy (ns_pending); + ns_pending = NULL; + } while (NULL != (zone = zone_head)) { - GNUNET_CONTAINER_DLL_remove (zone_head, - zone_tail, - zone); + GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone); GNUNET_free (zone->domain); GNUNET_free (zone); } + if (NULL != stats) + { + GNUNET_STATISTICS_destroy (stats, GNUNET_NO); + stats = NULL; + } } /** - * Begin processing hostnames from stdin. + * Iterate over all of the zones we care about and see which records + * we may need to re-fetch when. * * @param cls NULL */ static void -process_stdin (void *cls); - - -/** - * If applicable, continue processing from stdin. - */ -static void -continue_stdin () -{ - if ( (pending < THRESH) && - (stdin_waiting) ) - { - if (NULL != t) - GNUNET_SCHEDULER_cancel (t); - t = GNUNET_SCHEDULER_add_now (&process_stdin, - NULL); - } -} +iterate_zones (void *cls); /** * Function called if #GNUNET_NAMESTORE_records_lookup() failed. - * Continues resolution based on assumption namestore has no data. + * Just logs an error. * - * @param cls a `struct Request` + * @param cls a `struct Zone` */ static void ns_lookup_error_cb (void *cls) { - struct Request *req = cls; + struct Zone *zone = cls; - req->qe = NULL; GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Failed to load data from namestore for `%s'\n", - req->hostname); - insert_sorted (req); - pending--; - continue_stdin (); + "Failed to load data from namestore for zone `%s'\n", + zone->domain); + zone_it = NULL; + ns_iterator_trigger_next = 0; + iterate_zones (NULL); } /** * Process a record that was stored in the namestore. * - * @param cls a `struct Request *` - * @param zone private key of the zone + * @param cls a `struct Zone *` + * @param key private key of the zone * @param label label of the records * @param rd_count number of entries in @a rd array, 0 if label was deleted * @param rd array of records with data to store */ static void ns_lookup_result_cb (void *cls, - const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, - const char *label, - unsigned int rd_count, - const struct GNUNET_GNSRECORD_Data *rd) + const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, + const char *label, + unsigned int rd_count, + const struct GNUNET_GNSRECORD_Data *rd) { - struct Request *req = cls; + struct Zone *zone = cls; + struct Request *req; + struct GNUNET_HashCode hc; + char *fqdn; - req->qe = NULL; - pending--; - GNUNET_break (0 == memcmp (zone, - &req->zone->key, - sizeof (*zone))); - GNUNET_break (0 == strcasecmp (label, - get_label (req))); - for (unsigned int i=0;idomain); + GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc); + GNUNET_free (fqdn); + req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc); + if (NULL == req) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + "Ignoring record `%s' in zone `%s': not on my list!\n", + label, + zone->domain); + return; + } + GNUNET_assert (GNUNET_OK == + GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req)); + GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key)); + GNUNET_break (0 == strcasecmp (label, get_label (req))); + for (unsigned int i = 0; i < rd_count; i++) { struct GNUNET_TIME_Absolute at; - at.abs_value_us = rd->expiration_time; - add_record (req, - rd->record_type, - at, - rd->data, - rd->data_size); + if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) + { + struct GNUNET_TIME_Relative rel; + + rel.rel_value_us = rd->expiration_time; + at = GNUNET_TIME_relative_to_absolute (rel); + } + else + { + at.abs_value_us = rd->expiration_time; + } + add_record (req, rd->record_type, at, rd->data, rd->data_size); } if (0 == rd_count) { GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Empty record set in namestore for `%s'\n", - req->hostname); + "Empty record set in namestore for `%s'\n", + req->hostname); } else { unsigned int pos = 0; + cached++; req->expires = GNUNET_TIME_UNIT_FOREVER_ABS; - for (struct Record *rec = req->rec_head; - NULL != rec; - rec = rec->next) + for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next) { struct GNUNET_TIME_Absolute at; at.abs_value_us = rec->grd.expiration_time; - req->expires = GNUNET_TIME_absolute_min (req->expires, - at); + req->expires = GNUNET_TIME_absolute_min (req->expires, at); pos++; } if (0 == pos) req->expires = GNUNET_TIME_UNIT_ZERO_ABS; GNUNET_log (GNUNET_ERROR_TYPE_INFO, "Hot-start with %u existing records for `%s'\n", - pos, + pos, req->hostname); } + free_records (req); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Adding `%s' to worklist to start at %s\n", - req->hostname, - GNUNET_STRINGS_absolute_time_to_string (req->expires)); + "Adding `%s' to worklist to start at %s\n", + req->hostname, + GNUNET_STRINGS_absolute_time_to_string (req->expires)); insert_sorted (req); - continue_stdin (); } @@ -1340,31 +1510,27 @@ queue (const char *hostname) const char *dot; struct Zone *zone; size_t hlen; + struct GNUNET_HashCode hc; - if (GNUNET_OK != - GNUNET_DNSPARSER_check_name (hostname)) + if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Refusing invalid hostname `%s'\n", hostname); rejects++; - continue_stdin (); return; } - dot = strchr (hostname, - (unsigned char) '.'); + dot = strchr (hostname, (unsigned char) '.'); if (NULL == dot) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Refusing invalid hostname `%s' (lacks '.')\n", hostname); rejects++; - continue_stdin (); return; } for (zone = zone_head; NULL != zone; zone = zone->next) - if (0 == strcmp (zone->domain, - dot + 1)) + if (0 == strcmp (zone->domain, dot + 1)) break; if (NULL == zone) { @@ -1372,27 +1538,121 @@ queue (const char *hostname) GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Domain name `%s' not in ego list!\n", dot + 1); - continue_stdin (); return; } - pending++; hlen = strlen (hostname) + 1; - req = GNUNET_malloc (sizeof (struct Request) + hlen); + req = GNUNET_malloc (sizeof(struct Request) + hlen); req->zone = zone; req->hostname = (char *) &req[1]; - memcpy (req->hostname, - hostname, - hlen); + GNUNET_memcpy (req->hostname, hostname, hlen); req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE, - UINT16_MAX); - req->qe = GNUNET_NAMESTORE_records_lookup (ns, - &req->zone->key, - get_label (req), - &ns_lookup_error_cb, - req, - &ns_lookup_result_cb, - req); + UINT16_MAX); + GNUNET_CRYPTO_hash (req->hostname, hlen, &hc); + if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put ( + ns_pending, + &hc, + req, + GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Duplicate hostname `%s' ignored\n", + hostname); + GNUNET_free (req); + return; + } +} + + +/** + * We have completed the initial iteration over the namestore's database. + * This function is called on each of the remaining records in + * #move_to_queue to #queue() them, as we will simply not find existing + * records for them any longer. + * + * @param cls NULL + * @param key unused + * @param value a `struct Request` + * @return #GNUNET_OK (continue to iterate) + */ +static int +move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value) +{ + struct Request *req = value; + + (void) cls; + (void) key; + insert_sorted (req); + return GNUNET_OK; +} + + +/** + * Iterate over all of the zones we care about and see which records + * we may need to re-fetch when. + * + * @param cls NULL + */ +static void +iterate_zones (void *cls) +{ + static struct Zone *last; + + (void) cls; + if (NULL != zone_it) + { + zone_it = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Finished iteration over zone `%s'!\n", + last->domain); + /* subtract left-overs from previous iteration */ + GNUNET_STATISTICS_update (stats, + "# NAMESTORE records requested from cache", + (long long) (-ns_iterator_trigger_next), + GNUNET_NO); + ns_iterator_trigger_next = 0; + } + GNUNET_assert (NULL != zone_tail); + if (zone_tail == last) + { + /* Done iterating over relevant zones in NAMESTORE, move + rest of hash map to work queue as well. */ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Finished all NAMESTORE iterations!\n"); + GNUNET_STATISTICS_set (stats, + "# Domain names without cached reply", + GNUNET_CONTAINER_multihashmap_size (ns_pending), + GNUNET_NO); + GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL); + GNUNET_CONTAINER_multihashmap_destroy (ns_pending); + ns_pending = NULL; + start_time_reg_proc = GNUNET_TIME_absolute_get (); + total_reg_proc_dns = 0; + total_reg_proc_dns_ns = 0; + return; + } + if (NULL == last) + last = zone_head; + else + last = last->next; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Starting iteration over zone `%s'!\n", + last->domain); + /* subtract left-overs from previous iteration */ + GNUNET_STATISTICS_update (stats, + "# NAMESTORE records requested from cache", + 1, + GNUNET_NO); + ns_iterator_trigger_next = 1; + GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO); + zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns, + &last->key, + &ns_lookup_error_cb, + NULL, + &ns_lookup_result_cb, + last, + &iterate_zones, + NULL); } @@ -1405,7 +1665,7 @@ static void process_stdin (void *cls) { static struct GNUNET_TIME_Absolute last; - static unsigned int pdot; + static uint64_t idot; char hn[256]; (void) cls; @@ -1415,34 +1675,31 @@ process_stdin (void *cls) GNUNET_IDENTITY_disconnect (id); id = NULL; } - if (NULL != - fgets (hn, - sizeof (hn), - stdin)) + while (NULL != fgets (hn, sizeof(hn), stdin)) { - if (strlen(hn) > 0) - hn[strlen(hn)-1] = '\0'; /* eat newline */ - if (0 == pdot) + if (strlen (hn) > 0) + hn[strlen (hn) - 1] = '\0'; /* eat newline */ + if (0 == idot) last = GNUNET_TIME_absolute_get (); - pdot++; - if (0 == pdot % 1000) + idot++; + if (0 == idot % 100000) { struct GNUNET_TIME_Relative delta; delta = GNUNET_TIME_absolute_get_duration (last); last = GNUNET_TIME_absolute_get (); - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - "Imported 1000 records in %s\n", - GNUNET_STRINGS_relative_time_to_string (delta, - GNUNET_YES)); + fprintf (stderr, + "Read 100000 domain names in %s\n", + GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES)); + GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO); } queue (hn); - return; } - stdin_waiting = GNUNET_NO; - fprintf (stderr, "\n"); - t = GNUNET_SCHEDULER_add_now (&process_queue, - NULL); + fprintf (stderr, + "Done reading %llu domain names\n", + (unsigned long long) idot); + GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO); + iterate_zones (NULL); } @@ -1473,7 +1730,7 @@ process_stdin (void *cls) * cleaned up). * * @param cls closure - * @param ego ego handle + * @param ego ego handle, NULL for end of list * @param ctx context for application to store data for this ego * (during the lifetime of this process, initially NULL) * @param name name assigned by the user for this ego, @@ -1482,27 +1739,25 @@ process_stdin (void *cls) */ static void identity_cb (void *cls, - struct GNUNET_IDENTITY_Ego *ego, - void **ctx, - const char *name) + struct GNUNET_IDENTITY_Ego *ego, + void **ctx, + const char *name) { (void) cls; (void) ctx; + if (NULL == ego) { - if (NULL != zone_head) + /* end of iteration */ + if (NULL == zone_head) { - stdin_waiting = GNUNET_YES; - t = GNUNET_SCHEDULER_add_now (&process_stdin, - NULL); - } - else - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - "Specified zone not found\n"); + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n"); GNUNET_SCHEDULER_shutdown (); return; } + /* zone_head non-null, process hostnames from stdin */ + t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL); + return; } if (NULL != name) { @@ -1511,9 +1766,7 @@ identity_cb (void *cls, zone = GNUNET_new (struct Zone); zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego); zone->domain = GNUNET_strdup (name); - GNUNET_CONTAINER_DLL_insert (zone_head, - zone_tail, - zone); + GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone); } } @@ -1536,12 +1789,18 @@ run (void *cls, (void) cls; (void) args; (void) cfgfile; + stats = GNUNET_STATISTICS_create ("zoneimport", cfg); req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); + ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO); + if (NULL == ns_pending) + { + fprintf (stderr, "Failed to allocate memory for main hash map\n"); + return; + } ctx = GNUNET_DNSSTUB_start (256); if (NULL == ctx) { - fprintf (stderr, - "Failed to initialize GNUnet DNS STUB\n"); + fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n"); return; } if (NULL == args[0]) @@ -1550,31 +1809,24 @@ run (void *cls, "You must provide a list of DNS resolvers on the command line\n"); return; } - for (unsigned int i=0;NULL != args[i];i++) + for (unsigned int i = 0; NULL != args[i]; i++) { - if (GNUNET_OK != - GNUNET_DNSSTUB_add_dns_ip (ctx, - args[i])) + if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i])) { - fprintf (stderr, - "Failed to use `%s' for DNS resolver\n", - args[i]); + fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]); return; } } - GNUNET_SCHEDULER_add_shutdown (&do_shutdown, - NULL); + GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL); ns = GNUNET_NAMESTORE_connect (cfg); if (NULL == ns) { GNUNET_SCHEDULER_shutdown (); return; } - id = GNUNET_IDENTITY_connect (cfg, - &identity_cb, - NULL); + id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL); } @@ -1586,32 +1838,46 @@ run (void *cls, * @return 0 on success */ int -main (int argc, - char *const*argv) +main (int argc, char *const *argv) { - struct GNUNET_GETOPT_CommandLineOption options[] = { - GNUNET_GETOPT_OPTION_END - }; - - if (GNUNET_OK != - GNUNET_STRINGS_get_utf8_args (argc, argv, - &argc, &argv)) + struct GNUNET_GETOPT_CommandLineOption options[] = + { GNUNET_GETOPT_option_uint ('s', + "size", + "MAPSIZE", + gettext_noop ( + "size to use for the main hash map"), + &map_size), + GNUNET_GETOPT_option_relative_time ( + 'm', + "minimum-expiration", + "RELATIVETIME", + gettext_noop ("minimum expiration time we assume for imported records"), + &minimum_expiration_time), + GNUNET_GETOPT_OPTION_END }; + int ret; + + if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv)) return 2; - GNUNET_PROGRAM_run (argc, - argv, - "gnunet-zoneimport", - "import DNS zone into namestore", - options, - &run, - NULL); - GNUNET_free ((void*) argv); + if (GNUNET_OK != (ret = GNUNET_PROGRAM_run (argc, + argv, + "gnunet-zoneimport", + "import DNS zone into namestore", + options, + &run, + NULL))) + return ret; + GNUNET_free ((void *) argv); fprintf (stderr, - "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n", - rejects, + "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n" + "Found %u records, %u lookups failed, %u/%u pending on shutdown\n", + rejects, + cached, lookups, + record_sets, records, failures, - pending); + pending, + pending_rs); return 0; }