2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
26 #include <gnunet_util_lib.h>
27 #include <gnunet_dnsstub_lib.h>
28 #include <gnunet_dnsparser_lib.h>
29 #include <gnunet_gnsrecord_lib.h>
30 #include <gnunet_namestore_service.h>
31 #include <gnunet_statistics_service.h>
32 #include <gnunet_identity_service.h>
36 * Maximum number of queries pending at the same time.
41 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
42 * Used as an additional throttle.
44 #define TIME_THRESH 10
47 * How often do we retry a query before giving up for good?
52 * How many DNS requests do we at most issue in rapid series?
57 * How long do we wait at least between series of requests?
59 #define SERIES_DELAY \
60 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
63 * How long do DNS records have to last at least after being imported?
65 static struct GNUNET_TIME_Relative minimum_expiration_time;
68 * How many requests do we request from NAMESTORE in one batch
69 * during our initial iteration?
71 #define NS_BATCH_SIZE 1024
74 * Some zones may include authoritative records for other
75 * zones, such as foo.com.uk or bar.com.fr. As for GNS
76 * each dot represents a zone cut, we then need to create a
77 * zone on-the-fly to capture those records properly.
92 * Domain of the zone (i.e. "fr" or "com.fr")
97 * Private key of the zone.
99 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
104 * Record for the request to be stored by GNS.
121 struct GNUNET_GNSRECORD_Data grd;
126 * Request we should make. We keep this struct in memory per request,
127 * thus optimizing it is crucial for the overall memory consumption of
133 * Requests are kept in a heap while waiting to be resolved.
135 struct GNUNET_CONTAINER_HeapNode *hn;
138 * Active requests are kept in a DLL.
140 struct Request *next;
143 * Active requests are kept in a DLL.
145 struct Request *prev;
148 * Head of records that should be published in GNS for
151 struct Record *rec_head;
154 * Tail of records that should be published in GNS for
157 struct Record *rec_tail;
160 * Socket used to make the request, NULL if not active.
162 struct GNUNET_DNSSTUB_RequestSocket *rs;
165 * Hostname we are resolving, allocated at the end of
166 * this struct (optimizing memory consumption by reducing
167 * total number of allocations).
172 * Namestore operation pending for this record.
174 struct GNUNET_NAMESTORE_QueueEntry *qe;
177 * Zone responsible for this request.
179 const struct Zone *zone;
182 * At what time does the (earliest) of the returned records
183 * for this name expire? At this point, we need to re-fetch
186 struct GNUNET_TIME_Absolute expires;
189 * While we are fetching the record, the value is set to the
190 * starting time of the DNS operation. While doing a
191 * NAMESTORE store, again set to the start time of the
192 * NAMESTORE operation.
194 struct GNUNET_TIME_Absolute op_start_time;
197 * How often did we issue this query? (And failed, reset
198 * to zero once we were successful.)
200 unsigned int issue_num;
203 * random 16-bit DNS query identifier.
210 * Command-line argument specifying desired size of the hash map with
211 * all of our pending names. Usually, we use an automatically growing
212 * map, but this is only OK up to about a million entries. Above that
213 * number, the user must explicitly specify the size at startup.
215 static unsigned int map_size = 1024;
218 * Handle to the identity service.
220 static struct GNUNET_IDENTITY_Handle *id;
225 static struct GNUNET_NAMESTORE_Handle *ns;
228 * Handle to the statistics service.
230 static struct GNUNET_STATISTICS_Handle *stats;
233 * Context for DNS resolution.
235 static struct GNUNET_DNSSTUB_Context *ctx;
238 * The number of DNS queries that are outstanding
240 static unsigned int pending;
243 * The number of NAMESTORE record store operations that are outstanding
245 static unsigned int pending_rs;
248 * Number of lookups we performed overall.
250 static unsigned int lookups;
253 * Number of records we had cached.
255 static unsigned int cached;
258 * How many hostnames did we reject (malformed).
260 static unsigned int rejects;
263 * Number of lookups that failed.
265 static unsigned int failures;
268 * Number of records we found.
270 static unsigned int records;
273 * Number of record sets given to namestore.
275 static unsigned int record_sets;
278 * Heap of all requests to perform, sorted by
279 * the time we should next do the request (i.e. by expires).
281 static struct GNUNET_CONTAINER_Heap *req_heap;
284 * Active requests are kept in a DLL.
286 static struct Request *req_head;
289 * Active requests are kept in a DLL.
291 static struct Request *req_tail;
296 static struct GNUNET_SCHEDULER_Task *t;
299 * Hash map of requests for which we may still get a response from
300 * the namestore. Set to NULL once the initial namestore iteration
303 static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
306 * Current zone iteration handle.
308 static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
311 * Head of list of zones we are managing.
313 static struct Zone *zone_head;
316 * Tail of list of zones we are managing.
318 static struct Zone *zone_tail;
321 * After how many more results must #ns_lookup_result_cb() ask
322 * the namestore for more?
324 static uint64_t ns_iterator_trigger_next;
327 * Number of DNS requests counted in latency total.
329 static uint64_t total_dns_latency_cnt;
332 * Sum of DNS latencies observed.
334 static struct GNUNET_TIME_Relative total_dns_latency;
337 * Number of records processed (DNS lookup, no NAMESTORE) in total.
339 static uint64_t total_reg_proc_dns;
342 * Number of records processed (DNS lookup, with NAMESTORE) in total.
344 static uint64_t total_reg_proc_dns_ns;
347 * Start time of the regular processing.
349 static struct GNUNET_TIME_Absolute start_time_reg_proc;
352 * Last time we worked before going idle.
354 static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
357 * Time we slept just waiting for work.
359 static struct GNUNET_TIME_Relative idle_time;
363 * Callback for #for_all_records
366 * @param rec a DNS record
368 typedef void (*RecordProcessor) (void *cls,
369 const struct GNUNET_DNSPARSER_Record *rec);
373 * Call @a rp for each record in @a p, regardless of
374 * what response section it is in.
376 * @param p packet from DNS
377 * @param rp function to call
378 * @param rp_cls closure for @a rp
381 for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
385 for (unsigned int i = 0; i < p->num_answers; i++)
387 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
391 for (unsigned int i = 0; i < p->num_authority_records; i++)
393 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
397 for (unsigned int i = 0; i < p->num_additional_records; i++)
399 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
407 * Return just the label of the hostname in @a req.
409 * @param req request to process hostname of
410 * @return statically allocated pointer to the label,
411 * overwritten upon the next request!
414 get_label (struct Request *req)
416 static char label[64];
419 dot = strchr (req->hostname, (unsigned char) '.');
425 if (((size_t) (dot - req->hostname)) >= sizeof(label))
430 GNUNET_memcpy (label, req->hostname, dot - req->hostname);
431 label[dot - req->hostname] = '\0';
437 * Build DNS query for @a hostname.
439 * @param hostname host to build query for
440 * @param raw_size[out] number of bytes in the query
441 * @return NULL on error, otherwise pointer to statically (!)
442 * allocated query buffer
445 build_dns_query (struct Request *req, size_t *raw_size)
447 static char raw[512];
449 struct GNUNET_DNSPARSER_Packet p;
450 struct GNUNET_DNSPARSER_Query q;
453 q.name = (char *) req->hostname;
454 q.type = GNUNET_DNSPARSER_TYPE_NS;
455 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
457 memset (&p, 0, sizeof(p));
461 ret = GNUNET_DNSPARSER_pack (&p, UINT16_MAX, &rawp, raw_size);
462 if (GNUNET_OK != ret)
464 if (GNUNET_NO == ret)
466 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
467 "Failed to pack query for hostname `%s'\n",
472 if (*raw_size > sizeof(raw))
474 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
475 "Failed to pack query for hostname `%s'\n",
482 GNUNET_memcpy (raw, rawp, *raw_size);
489 * Free records associated with @a req.
491 * @param req request to free records of
494 free_records (struct Request *req)
499 while (NULL != (rec = req->rec_head))
501 GNUNET_CONTAINER_DLL_remove (req->rec_head, req->rec_tail, rec);
508 * Free @a req and data structures reachable from it.
510 * @param req request to free
513 free_request (struct Request *req)
521 * Process as many requests as possible from the queue.
526 process_queue (void *cls);
530 * Insert @a req into DLL sorted by next fetch time.
532 * @param req request to insert into #req_heap
535 insert_sorted (struct Request *req)
538 GNUNET_CONTAINER_heap_insert (req_heap, req, req->expires.abs_value_us);
539 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
542 GNUNET_SCHEDULER_cancel (t);
543 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
544 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
550 * Add record to the GNS record set for @a req.
552 * @param req the request to expand GNS record set for
553 * @param type type to use
554 * @param expiration_time when should @a rec expire
555 * @param data raw data to store
556 * @param data_len number of bytes in @a data
559 add_record (struct Request *req,
561 struct GNUNET_TIME_Absolute expiration_time,
567 rec = GNUNET_malloc (sizeof(struct Record) + data_len);
568 rec->grd.data = &rec[1];
569 rec->grd.expiration_time = expiration_time.abs_value_us;
570 rec->grd.data_size = data_len;
571 rec->grd.record_type = type;
572 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
573 GNUNET_memcpy (&rec[1], data, data_len);
574 GNUNET_CONTAINER_DLL_insert (req->rec_head, req->rec_tail, rec);
579 * Closure for #check_for_glue.
584 * Overall request we are processing.
589 * NS name we are looking for glue for.
594 * Set to #GNUNET_YES if glue was found.
601 * Try to find glue records for a given NS record.
603 * @param cls a `struct GlueClosure *`
604 * @param rec record that may contain glue information
607 check_for_glue (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
609 struct GlueClosure *gc = cls;
613 char ip[INET6_ADDRSTRLEN + 1];
614 socklen_t ip_size = (socklen_t) sizeof(ip);
615 struct GNUNET_TIME_Absolute expiration_time;
616 struct GNUNET_TIME_Relative left;
618 if (0 != strcasecmp (rec->name, gc->ns))
620 expiration_time = rec->expiration_time;
621 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
622 if (0 == left.rel_value_us)
623 return; /* ignore expired glue records */
624 /* if expiration window is too short, bump it to configured minimum */
625 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
627 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
628 dst_len = sizeof(dst);
632 case GNUNET_DNSPARSER_TYPE_A:
633 if (sizeof(struct in_addr) != rec->data.raw.data_len)
638 if (NULL == inet_ntop (AF_INET, rec->data.raw.data, ip, ip_size))
643 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
646 gc->req->hostname)) &&
648 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
651 GNUNET_GNSRECORD_TYPE_GNS2DNS,
655 gc->found = GNUNET_YES;
659 case GNUNET_DNSPARSER_TYPE_AAAA:
660 if (sizeof(struct in6_addr) != rec->data.raw.data_len)
665 if (NULL == inet_ntop (AF_INET6, rec->data.raw.data, ip, ip_size))
670 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
673 gc->req->hostname)) &&
675 GNUNET_DNSPARSER_builder_add_name (dst, dst_len, &off, ip)))
678 GNUNET_GNSRECORD_TYPE_GNS2DNS,
682 gc->found = GNUNET_YES;
686 case GNUNET_DNSPARSER_TYPE_CNAME:
687 if ((GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
690 gc->req->hostname)) &&
691 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
694 rec->data.hostname)))
697 GNUNET_GNSRECORD_TYPE_GNS2DNS,
701 gc->found = GNUNET_YES;
706 /* useless, do nothing */
713 * Closure for #process_record().
715 struct ProcessRecordContext
718 * Answer we got back and are currently parsing, or NULL
721 struct GNUNET_DNSPARSER_Packet *p;
724 * Request we are processing.
731 * We received @a rec for @a req. Remember the answer.
733 * @param cls a `struct ProcessRecordContext`
734 * @param rec response
737 process_record (void *cls, const struct GNUNET_DNSPARSER_Record *rec)
739 struct ProcessRecordContext *prc = cls;
740 struct Request *req = prc->req;
744 struct GNUNET_TIME_Absolute expiration_time;
745 struct GNUNET_TIME_Relative left;
747 dst_len = sizeof(dst);
750 if (0 != strcasecmp (rec->name, req->hostname))
753 GNUNET_ERROR_TYPE_DEBUG,
754 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
756 (unsigned int) rec->type,
758 return; /* does not match hostname, might be glue, but
759 not useful for this pass! */
761 expiration_time = rec->expiration_time;
762 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
763 if (0 == left.rel_value_us)
765 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
766 "DNS returned expired record for `%s'\n",
768 GNUNET_STATISTICS_update (stats,
769 "# expired records obtained from DNS",
772 return; /* record expired */
775 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
776 "DNS returned record that expires at %s for `%s'\n",
777 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
779 /* if expiration window is too short, bump it to configured minimum */
780 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
782 GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
785 case GNUNET_DNSPARSER_TYPE_NS: {
786 struct GlueClosure gc;
790 gc.ns = rec->data.hostname;
791 gc.found = GNUNET_NO;
792 for_all_records (prc->p, &check_for_glue, &gc);
793 if ((GNUNET_NO == gc.found) &&
794 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
798 (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
801 rec->data.hostname)))
803 /* FIXME: actually check if this is out-of-bailiwick,
804 and if not request explicit resolution... */
805 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
806 "Converted OOB (`%s') NS record for `%s'\n",
810 GNUNET_GNSRECORD_TYPE_GNS2DNS,
817 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
818 "Converted NS record for `%s' using glue\n",
824 case GNUNET_DNSPARSER_TYPE_CNAME:
825 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
830 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
831 "Converting CNAME (`%s') record for `%s'\n",
834 add_record (req, rec->type, expiration_time, dst, off);
838 case GNUNET_DNSPARSER_TYPE_DNAME:
839 /* No support for DNAME in GNS yet! FIXME: support later! */
840 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
841 "FIXME: not supported: %s DNAME %s\n",
846 case GNUNET_DNSPARSER_TYPE_MX:
848 GNUNET_DNSPARSER_builder_add_mx (dst, dst_len, &off, rec->data.mx))
850 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
851 "Converting MX (`%s') record for `%s'\n",
852 rec->data.mx->mxhost,
854 add_record (req, rec->type, expiration_time, dst, off);
858 case GNUNET_DNSPARSER_TYPE_SOA:
860 GNUNET_DNSPARSER_builder_add_soa (dst, dst_len, &off, rec->data.soa))
862 /* NOTE: GNS does not really use SOAs */
863 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
864 "Converting SOA record for `%s'\n",
866 add_record (req, rec->type, expiration_time, dst, off);
870 case GNUNET_DNSPARSER_TYPE_SRV:
872 GNUNET_DNSPARSER_builder_add_srv (dst, dst_len, &off, rec->data.srv))
874 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
875 "Converting SRV record for `%s'\n",
877 add_record (req, rec->type, expiration_time, dst, off);
881 case GNUNET_DNSPARSER_TYPE_PTR:
882 if (GNUNET_OK == GNUNET_DNSPARSER_builder_add_name (dst,
887 /* !?: what does a PTR record do in a regular TLD??? */
888 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
889 "Converting PTR record for `%s' (weird)\n",
891 add_record (req, rec->type, expiration_time, dst, off);
895 case GNUNET_DNSPARSER_TYPE_CERT:
897 GNUNET_DNSPARSER_builder_add_cert (dst, dst_len, &off, rec->data.cert))
899 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
900 "Converting CERT record for `%s'\n",
902 add_record (req, rec->type, expiration_time, dst, off);
906 /* Rest is 'raw' encoded and just needs to be copied IF
907 the hostname matches the requested name; otherwise we
908 simply cannot use it. */
909 case GNUNET_DNSPARSER_TYPE_A:
910 case GNUNET_DNSPARSER_TYPE_AAAA:
911 case GNUNET_DNSPARSER_TYPE_TXT:
913 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
914 "Converting record of type %u for `%s'\n",
915 (unsigned int) rec->type,
921 rec->data.raw.data_len);
928 * Continuation called to notify client about result of the
931 * @param cls closure with our `struct Request`
932 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
933 * #GNUNET_NO if content was already there or not found
934 * #GNUNET_YES (or other positive value) on success
935 * @param emsg NULL on success, otherwise an error message
938 store_completed_cb (void *cls, int32_t success, const char *emsg)
940 static struct GNUNET_TIME_Absolute last;
941 struct Request *req = cls;
944 if (GNUNET_SYSERR == success)
946 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
947 "Failed to store zone data for `%s': %s\n",
953 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
954 "Stored records under `%s' (%d)\n",
958 total_reg_proc_dns_ns++; /* finished regular processing */
961 /* compute NAMESTORE statistics */
963 static uint64_t total_ns_latency_cnt;
964 static struct GNUNET_TIME_Relative total_ns_latency;
965 struct GNUNET_TIME_Relative ns_latency;
967 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
968 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency, ns_latency);
969 if (0 == total_ns_latency_cnt)
970 last = GNUNET_TIME_absolute_get ();
971 total_ns_latency_cnt++;
972 if (0 == (total_ns_latency_cnt % 1000))
974 struct GNUNET_TIME_Relative delta;
976 delta = GNUNET_TIME_absolute_get_duration (last);
977 last = GNUNET_TIME_absolute_get ();
979 "Processed 1000 records in %s\n",
980 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
981 GNUNET_STATISTICS_set (stats,
982 "# average NAMESTORE PUT latency (μs)",
983 total_ns_latency.rel_value_us
984 / total_ns_latency_cnt,
988 /* compute and publish overall velocity */
989 if (0 == (total_reg_proc_dns_ns % 100))
991 struct GNUNET_TIME_Relative runtime;
993 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
994 runtime = GNUNET_TIME_relative_subtract (runtime, idle_time);
996 GNUNET_TIME_relative_divide (runtime,
997 total_reg_proc_dns + total_reg_proc_dns_ns);
998 GNUNET_STATISTICS_set (stats,
999 "# Regular processing completed without NAMESTORE",
1002 GNUNET_STATISTICS_set (stats,
1003 "# Regular processing completed with NAMESTORE PUT",
1004 total_reg_proc_dns_ns,
1006 GNUNET_STATISTICS_set (stats,
1007 "# average request processing latency (μs)",
1008 runtime.rel_value_us,
1010 GNUNET_STATISTICS_set (stats,
1011 "# total time spent idle (μs)",
1012 idle_time.rel_value_us,
1018 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1019 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1025 * Function called with the result of a DNS resolution.
1027 * @param cls closure with the `struct Request`
1028 * @param dns dns response, never NULL
1029 * @param dns_len number of bytes in @a dns
1032 process_result (void *cls,
1033 const struct GNUNET_TUN_DnsHeader *dns,
1036 struct Request *req = cls;
1038 struct GNUNET_DNSPARSER_Packet *p;
1039 unsigned int rd_count;
1041 GNUNET_assert (NULL == req->hn);
1045 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1049 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1050 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1052 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1053 "Stub gave up on DNS reply for `%s'\n",
1055 GNUNET_STATISTICS_update (stats, "# DNS lookups timed out", 1, GNUNET_NO);
1056 if (req->issue_num > MAX_RETRIES)
1060 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1063 total_reg_proc_dns++;
1065 insert_sorted (req);
1068 if (req->id != dns->id)
1070 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1071 "DNS ID did not match request, ignoring reply\n");
1072 GNUNET_STATISTICS_update (stats, "# DNS ID mismatches", 1, GNUNET_NO);
1075 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1076 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1079 p = GNUNET_DNSPARSER_parse ((const char *) dns, dns_len);
1082 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1083 "Failed to parse DNS reply for `%s'\n",
1085 GNUNET_STATISTICS_update (stats, "# DNS parser errors", 1, GNUNET_NO);
1088 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1089 t = GNUNET_SCHEDULER_add_now (&process_queue, NULL);
1091 if (req->issue_num > MAX_RETRIES)
1095 GNUNET_STATISTICS_update (stats, "# requests given up on", 1, GNUNET_NO);
1098 insert_sorted (req);
1101 /* import new records */
1102 req->issue_num = 0; /* success, reset counter! */
1104 struct ProcessRecordContext prc = { .req = req, .p = p };
1106 for_all_records (p, &process_record, &prc);
1108 GNUNET_DNSPARSER_free_packet (p);
1109 /* count records found, determine minimum expiration time */
1110 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1112 struct GNUNET_TIME_Relative dns_latency;
1114 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1116 GNUNET_TIME_relative_add (total_dns_latency, dns_latency);
1117 total_dns_latency_cnt++;
1118 if (0 == (total_dns_latency_cnt % 1000))
1120 GNUNET_STATISTICS_set (stats,
1121 "# average DNS lookup latency (μs)",
1122 total_dns_latency.rel_value_us
1123 / total_dns_latency_cnt,
1128 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1130 struct GNUNET_TIME_Absolute at;
1132 at.abs_value_us = rec->grd.expiration_time;
1133 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1136 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1137 "Obtained %u records for `%s'\n",
1140 /* Instead of going for SOA, simplified for now to look each
1141 day in case we got an empty response */
1144 req->expires = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1145 GNUNET_STATISTICS_update (stats,
1146 "# empty DNS replies (usually NXDOMAIN)",
1154 /* convert records to namestore import format */
1156 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL (rd_count)];
1157 unsigned int off = 0;
1159 /* convert linked list into array */
1160 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1161 rd[off++] = rec->grd;
1163 req->op_start_time = GNUNET_TIME_absolute_get ();
1164 req->qe = GNUNET_NAMESTORE_records_store (ns,
1169 &store_completed_cb,
1171 GNUNET_assert (NULL != req->qe);
1173 insert_sorted (req);
1178 * Process as many requests as possible from the queue.
1183 process_queue (void *cls)
1185 struct Request *req;
1186 unsigned int series;
1189 struct GNUNET_TIME_Relative delay;
1192 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1193 idle_time = GNUNET_TIME_relative_add (idle_time, delay);
1196 while (pending + pending_rs < THRESH)
1198 req = GNUNET_CONTAINER_heap_peek (req_heap);
1201 if (NULL != req->qe)
1202 return; /* namestore op still pending */
1203 if (NULL != req->rs)
1206 return; /* already submitted */
1208 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1210 GNUNET_assert (req == GNUNET_CONTAINER_heap_remove_root (req_heap));
1212 GNUNET_CONTAINER_DLL_insert (req_head, req_tail, req);
1213 GNUNET_assert (NULL == req->rs);
1214 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1215 "Requesting resolution for `%s'\n",
1217 raw = build_dns_query (req, &raw_size);
1224 req->op_start_time = GNUNET_TIME_absolute_get ();
1225 req->rs = GNUNET_DNSSTUB_resolve (ctx, raw, raw_size, &process_result, req);
1226 GNUNET_assert (NULL != req->rs);
1231 if (series > MAX_SERIES)
1234 if (pending + pending_rs >= THRESH)
1236 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1237 "Stopped processing queue (%u+%u/%u)]\n",
1241 return; /* wait for replies */
1243 req = GNUNET_CONTAINER_heap_peek (req_heap);
1246 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1247 "Stopped processing queue: empty queue\n");
1250 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1252 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1253 "Waiting until %s for next record (`%s') to expire\n",
1254 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1257 GNUNET_SCHEDULER_cancel (t);
1258 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1259 t = GNUNET_SCHEDULER_add_at (req->expires, &process_queue, NULL);
1262 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Throttling\n");
1264 GNUNET_SCHEDULER_cancel (t);
1265 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1266 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY, &process_queue, NULL);
1271 * Iterator called during #do_shutdown() to free requests in
1272 * the #ns_pending map.
1276 * @param value the `struct Request` to free
1277 * @return #GNUNET_OK
1280 free_request_it (void *cls, const struct GNUNET_HashCode *key, void *value)
1282 struct Request *req = value;
1292 * Clean up and terminate the process.
1297 do_shutdown (void *cls)
1299 struct Request *req;
1305 GNUNET_IDENTITY_disconnect (id);
1310 GNUNET_SCHEDULER_cancel (t);
1313 while (NULL != (req = req_head))
1315 GNUNET_CONTAINER_DLL_remove (req_head, req_tail, req);
1316 if (NULL != req->qe)
1317 GNUNET_NAMESTORE_cancel (req->qe);
1320 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1323 if (NULL != req->qe)
1324 GNUNET_NAMESTORE_cancel (req->qe);
1327 if (NULL != zone_it)
1329 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1334 GNUNET_NAMESTORE_disconnect (ns);
1339 GNUNET_DNSSTUB_stop (ctx);
1342 if (NULL != req_heap)
1344 GNUNET_CONTAINER_heap_destroy (req_heap);
1347 if (NULL != ns_pending)
1349 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &free_request_it, NULL);
1350 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1353 while (NULL != (zone = zone_head))
1355 GNUNET_CONTAINER_DLL_remove (zone_head, zone_tail, zone);
1356 GNUNET_free (zone->domain);
1361 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1368 * Iterate over all of the zones we care about and see which records
1369 * we may need to re-fetch when.
1374 iterate_zones (void *cls);
1378 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1379 * Just logs an error.
1381 * @param cls a `struct Zone`
1384 ns_lookup_error_cb (void *cls)
1386 struct Zone *zone = cls;
1388 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1389 "Failed to load data from namestore for zone `%s'\n",
1392 ns_iterator_trigger_next = 0;
1393 iterate_zones (NULL);
1398 * Process a record that was stored in the namestore.
1400 * @param cls a `struct Zone *`
1401 * @param key private key of the zone
1402 * @param label label of the records
1403 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1404 * @param rd array of records with data to store
1407 ns_lookup_result_cb (void *cls,
1408 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1410 unsigned int rd_count,
1411 const struct GNUNET_GNSRECORD_Data *rd)
1413 struct Zone *zone = cls;
1414 struct Request *req;
1415 struct GNUNET_HashCode hc;
1418 ns_iterator_trigger_next--;
1419 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1420 "Obtained NAMESTORE reply, %llu left in round\n",
1421 (unsigned long long) ns_iterator_trigger_next);
1422 if (0 == ns_iterator_trigger_next)
1424 ns_iterator_trigger_next = NS_BATCH_SIZE;
1425 GNUNET_STATISTICS_update (stats,
1426 "# NAMESTORE records requested from cache",
1427 ns_iterator_trigger_next,
1429 GNUNET_NAMESTORE_zone_iterator_next (zone_it, ns_iterator_trigger_next);
1431 GNUNET_asprintf (&fqdn, "%s.%s", label, zone->domain);
1432 GNUNET_CRYPTO_hash (fqdn, strlen (fqdn) + 1, &hc);
1434 req = GNUNET_CONTAINER_multihashmap_get (ns_pending, &hc);
1437 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1438 "Ignoring record `%s' in zone `%s': not on my list!\n",
1443 GNUNET_assert (GNUNET_OK ==
1444 GNUNET_CONTAINER_multihashmap_remove (ns_pending, &hc, req));
1445 GNUNET_break (0 == GNUNET_memcmp (key, &req->zone->key));
1446 GNUNET_break (0 == strcasecmp (label, get_label (req)));
1447 for (unsigned int i = 0; i < rd_count; i++)
1449 struct GNUNET_TIME_Absolute at;
1451 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1453 struct GNUNET_TIME_Relative rel;
1455 rel.rel_value_us = rd->expiration_time;
1456 at = GNUNET_TIME_relative_to_absolute (rel);
1460 at.abs_value_us = rd->expiration_time;
1462 add_record (req, rd->record_type, at, rd->data, rd->data_size);
1466 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1467 "Empty record set in namestore for `%s'\n",
1472 unsigned int pos = 0;
1475 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1476 for (struct Record *rec = req->rec_head; NULL != rec; rec = rec->next)
1478 struct GNUNET_TIME_Absolute at;
1480 at.abs_value_us = rec->grd.expiration_time;
1481 req->expires = GNUNET_TIME_absolute_min (req->expires, at);
1485 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1486 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1487 "Hot-start with %u existing records for `%s'\n",
1493 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1494 "Adding `%s' to worklist to start at %s\n",
1496 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1497 insert_sorted (req);
1502 * Add @a hostname to the list of requests to be made.
1504 * @param hostname name to resolve
1507 queue (const char *hostname)
1509 struct Request *req;
1513 struct GNUNET_HashCode hc;
1515 if (GNUNET_OK != GNUNET_DNSPARSER_check_name (hostname))
1517 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1518 "Refusing invalid hostname `%s'\n",
1523 dot = strchr (hostname, (unsigned char) '.');
1526 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1527 "Refusing invalid hostname `%s' (lacks '.')\n",
1532 for (zone = zone_head; NULL != zone; zone = zone->next)
1533 if (0 == strcmp (zone->domain, dot + 1))
1538 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1539 "Domain name `%s' not in ego list!\n",
1544 hlen = strlen (hostname) + 1;
1545 req = GNUNET_malloc (sizeof(struct Request) + hlen);
1547 req->hostname = (char *) &req[1];
1548 GNUNET_memcpy (req->hostname, hostname, hlen);
1549 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1551 GNUNET_CRYPTO_hash (req->hostname, hlen, &hc);
1552 if (GNUNET_OK != GNUNET_CONTAINER_multihashmap_put (
1556 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1558 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1559 "Duplicate hostname `%s' ignored\n",
1568 * We have completed the initial iteration over the namestore's database.
1569 * This function is called on each of the remaining records in
1570 * #move_to_queue to #queue() them, as we will simply not find existing
1571 * records for them any longer.
1575 * @param value a `struct Request`
1576 * @return #GNUNET_OK (continue to iterate)
1579 move_to_queue (void *cls, const struct GNUNET_HashCode *key, void *value)
1581 struct Request *req = value;
1585 insert_sorted (req);
1591 * Iterate over all of the zones we care about and see which records
1592 * we may need to re-fetch when.
1597 iterate_zones (void *cls)
1599 static struct Zone *last;
1602 if (NULL != zone_it)
1605 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1606 "Finished iteration over zone `%s'!\n",
1608 /* subtract left-overs from previous iteration */
1609 GNUNET_STATISTICS_update (stats,
1610 "# NAMESTORE records requested from cache",
1611 (long long) (-ns_iterator_trigger_next),
1613 ns_iterator_trigger_next = 0;
1615 GNUNET_assert (NULL != zone_tail);
1616 if (zone_tail == last)
1618 /* Done iterating over relevant zones in NAMESTORE, move
1619 rest of hash map to work queue as well. */
1620 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1621 "Finished all NAMESTORE iterations!\n");
1622 GNUNET_STATISTICS_set (stats,
1623 "# Domain names without cached reply",
1624 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1626 GNUNET_CONTAINER_multihashmap_iterate (ns_pending, &move_to_queue, NULL);
1627 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1629 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1630 total_reg_proc_dns = 0;
1631 total_reg_proc_dns_ns = 0;
1638 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1639 "Starting iteration over zone `%s'!\n",
1641 /* subtract left-overs from previous iteration */
1642 GNUNET_STATISTICS_update (stats,
1643 "# NAMESTORE records requested from cache",
1646 ns_iterator_trigger_next = 1;
1647 GNUNET_STATISTICS_update (stats, "# zones iterated", 1, GNUNET_NO);
1648 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1650 &ns_lookup_error_cb,
1652 &ns_lookup_result_cb,
1660 * Begin processing hostnames from stdin.
1665 process_stdin (void *cls)
1667 static struct GNUNET_TIME_Absolute last;
1668 static uint64_t idot;
1675 GNUNET_IDENTITY_disconnect (id);
1678 while (NULL != fgets (hn, sizeof(hn), stdin))
1680 if (strlen (hn) > 0)
1681 hn[strlen (hn) - 1] = '\0'; /* eat newline */
1683 last = GNUNET_TIME_absolute_get ();
1685 if (0 == idot % 100000)
1687 struct GNUNET_TIME_Relative delta;
1689 delta = GNUNET_TIME_absolute_get_duration (last);
1690 last = GNUNET_TIME_absolute_get ();
1692 "Read 100000 domain names in %s\n",
1693 GNUNET_STRINGS_relative_time_to_string (delta, GNUNET_YES));
1694 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1699 "Done reading %llu domain names\n",
1700 (unsigned long long) idot);
1701 GNUNET_STATISTICS_set (stats, "# domain names provided", idot, GNUNET_NO);
1702 iterate_zones (NULL);
1707 * Method called to inform about the egos of this peer.
1709 * When used with #GNUNET_IDENTITY_connect, this function is
1710 * initially called for all egos and then again whenever a
1711 * ego's name changes or if it is deleted. At the end of
1712 * the initial pass over all egos, the function is once called
1713 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1714 * be invoked in the future or that there was an error.
1716 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
1717 * this function is only called ONCE, and 'NULL' being passed in
1718 * @a ego does indicate an error (i.e. name is taken or no default
1719 * value is known). If @a ego is non-NULL and if '*ctx'
1720 * is set in those callbacks, the value WILL be passed to a subsequent
1721 * call to the identity callback of #GNUNET_IDENTITY_connect (if
1722 * that one was not NULL).
1724 * When an identity is renamed, this function is called with the
1725 * (known) @a ego but the NEW @a name.
1727 * When an identity is deleted, this function is called with the
1728 * (known) ego and "NULL" for the @a name. In this case,
1729 * the @a ego is henceforth invalid (and the @a ctx should also be
1732 * @param cls closure
1733 * @param ego ego handle, NULL for end of list
1734 * @param ctx context for application to store data for this ego
1735 * (during the lifetime of this process, initially NULL)
1736 * @param name name assigned by the user for this ego,
1737 * NULL if the user just deleted the ego and it
1738 * must thus no longer be used
1741 identity_cb (void *cls,
1742 struct GNUNET_IDENTITY_Ego *ego,
1751 /* end of iteration */
1752 if (NULL == zone_head)
1754 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "No zone found\n");
1755 GNUNET_SCHEDULER_shutdown ();
1758 /* zone_head non-null, process hostnames from stdin */
1759 t = GNUNET_SCHEDULER_add_now (&process_stdin, NULL);
1766 zone = GNUNET_new (struct Zone);
1767 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1768 zone->domain = GNUNET_strdup (name);
1769 GNUNET_CONTAINER_DLL_insert (zone_head, zone_tail, zone);
1775 * Process requests from the queue, then if the queue is
1776 * not empty, try again.
1779 * @param args remaining command-line arguments
1780 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1781 * @param cfg configuration
1786 const char *cfgfile,
1787 const struct GNUNET_CONFIGURATION_Handle *cfg)
1792 stats = GNUNET_STATISTICS_create ("zoneimport", cfg);
1793 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1794 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size, GNUNET_NO);
1795 if (NULL == ns_pending)
1797 fprintf (stderr, "Failed to allocate memory for main hash map\n");
1800 ctx = GNUNET_DNSSTUB_start (256);
1803 fprintf (stderr, "Failed to initialize GNUnet DNS STUB\n");
1806 if (NULL == args[0])
1809 "You must provide a list of DNS resolvers on the command line\n");
1812 for (unsigned int i = 0; NULL != args[i]; i++)
1814 if (GNUNET_OK != GNUNET_DNSSTUB_add_dns_ip (ctx, args[i]))
1816 fprintf (stderr, "Failed to use `%s' for DNS resolver\n", args[i]);
1822 GNUNET_SCHEDULER_add_shutdown (&do_shutdown, NULL);
1823 ns = GNUNET_NAMESTORE_connect (cfg);
1826 GNUNET_SCHEDULER_shutdown ();
1829 id = GNUNET_IDENTITY_connect (cfg, &identity_cb, NULL);
1834 * Call with IP address of resolver to query.
1836 * @param argc should be 2
1837 * @param argv[1] should contain IP address
1838 * @return 0 on success
1841 main (int argc, char *const *argv)
1843 struct GNUNET_GETOPT_CommandLineOption options[] =
1844 { GNUNET_GETOPT_option_uint ('s',
1848 "size to use for the main hash map"),
1850 GNUNET_GETOPT_option_relative_time (
1852 "minimum-expiration",
1854 gettext_noop ("minimum expiration time we assume for imported records"),
1855 &minimum_expiration_time),
1856 GNUNET_GETOPT_OPTION_END };
1859 if (GNUNET_OK != GNUNET_STRINGS_get_utf8_args (argc, argv, &argc, &argv))
1861 if (GNUNET_OK != (ret = GNUNET_PROGRAM_run (argc,
1863 "gnunet-zoneimport",
1864 "import DNS zone into namestore",
1869 GNUNET_free ((void *) argv);
1871 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
1872 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
1885 /* end of gnunet-zoneimport.c */