2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
26 #include <gnunet_util_lib.h>
27 #include <gnunet_dnsstub_lib.h>
28 #include <gnunet_dnsparser_lib.h>
29 #include <gnunet_gnsrecord_lib.h>
30 #include <gnunet_namestore_service.h>
31 #include <gnunet_statistics_service.h>
32 #include <gnunet_identity_service.h>
36 * Maximum number of queries pending at the same time.
41 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
42 * Used as an additional throttle.
44 #define TIME_THRESH 10
47 * How often do we retry a query before giving up for good?
52 * How many DNS requests do we at most issue in rapid series?
57 * How long do we wait at least between series of requests?
59 #define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
62 * How long do DNS records have to last at least after being imported?
64 static struct GNUNET_TIME_Relative minimum_expiration_time;
67 * How many requests do we request from NAMESTORE in one batch
68 * during our initial iteration?
70 #define NS_BATCH_SIZE 1024
73 * Some zones may include authoritative records for other
74 * zones, such as foo.com.uk or bar.com.fr. As for GNS
75 * each dot represents a zone cut, we then need to create a
76 * zone on-the-fly to capture those records properly.
92 * Domain of the zone (i.e. "fr" or "com.fr")
97 * Private key of the zone.
99 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
105 * Record for the request to be stored by GNS.
122 struct GNUNET_GNSRECORD_Data grd;
128 * Request we should make. We keep this struct in memory per request,
129 * thus optimizing it is crucial for the overall memory consumption of
135 * Requests are kept in a heap while waiting to be resolved.
137 struct GNUNET_CONTAINER_HeapNode *hn;
140 * Active requests are kept in a DLL.
142 struct Request *next;
145 * Active requests are kept in a DLL.
147 struct Request *prev;
150 * Head of records that should be published in GNS for
153 struct Record *rec_head;
156 * Tail of records that should be published in GNS for
159 struct Record *rec_tail;
162 * Socket used to make the request, NULL if not active.
164 struct GNUNET_DNSSTUB_RequestSocket *rs;
167 * Hostname we are resolving, allocated at the end of
168 * this struct (optimizing memory consumption by reducing
169 * total number of allocations).
174 * Namestore operation pending for this record.
176 struct GNUNET_NAMESTORE_QueueEntry *qe;
179 * Zone responsible for this request.
181 const struct Zone *zone;
184 * At what time does the (earliest) of the returned records
185 * for this name expire? At this point, we need to re-fetch
188 struct GNUNET_TIME_Absolute expires;
191 * While we are fetching the record, the value is set to the
192 * starting time of the DNS operation. While doing a
193 * NAMESTORE store, again set to the start time of the
194 * NAMESTORE operation.
196 struct GNUNET_TIME_Absolute op_start_time;
199 * How often did we issue this query? (And failed, reset
200 * to zero once we were successful.)
202 unsigned int issue_num;
205 * random 16-bit DNS query identifier.
212 * Command-line argument specifying desired size of the hash map with
213 * all of our pending names. Usually, we use an automatically growing
214 * map, but this is only OK up to about a million entries. Above that
215 * number, the user must explicitly specify the size at startup.
217 static unsigned int map_size = 1024;
220 * Handle to the identity service.
222 static struct GNUNET_IDENTITY_Handle *id;
227 static struct GNUNET_NAMESTORE_Handle *ns;
230 * Handle to the statistics service.
232 static struct GNUNET_STATISTICS_Handle *stats;
235 * Context for DNS resolution.
237 static struct GNUNET_DNSSTUB_Context *ctx;
240 * The number of DNS queries that are outstanding
242 static unsigned int pending;
245 * The number of NAMESTORE record store operations that are outstanding
247 static unsigned int pending_rs;
250 * Number of lookups we performed overall.
252 static unsigned int lookups;
255 * Number of records we had cached.
257 static unsigned int cached;
260 * How many hostnames did we reject (malformed).
262 static unsigned int rejects;
265 * Number of lookups that failed.
267 static unsigned int failures;
270 * Number of records we found.
272 static unsigned int records;
275 * Number of record sets given to namestore.
277 static unsigned int record_sets;
280 * Heap of all requests to perform, sorted by
281 * the time we should next do the request (i.e. by expires).
283 static struct GNUNET_CONTAINER_Heap *req_heap;
286 * Active requests are kept in a DLL.
288 static struct Request *req_head;
291 * Active requests are kept in a DLL.
293 static struct Request *req_tail;
298 static struct GNUNET_SCHEDULER_Task *t;
301 * Hash map of requests for which we may still get a response from
302 * the namestore. Set to NULL once the initial namestore iteration
305 static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
308 * Current zone iteration handle.
310 static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
313 * Head of list of zones we are managing.
315 static struct Zone *zone_head;
318 * Tail of list of zones we are managing.
320 static struct Zone *zone_tail;
323 * After how many more results must #ns_lookup_result_cb() ask
324 * the namestore for more?
326 static uint64_t ns_iterator_trigger_next;
329 * Number of DNS requests counted in latency total.
331 static uint64_t total_dns_latency_cnt;
334 * Sum of DNS latencies observed.
336 static struct GNUNET_TIME_Relative total_dns_latency;
339 * Number of records processed (DNS lookup, no NAMESTORE) in total.
341 static uint64_t total_reg_proc_dns;
344 * Number of records processed (DNS lookup, with NAMESTORE) in total.
346 static uint64_t total_reg_proc_dns_ns;
349 * Start time of the regular processing.
351 static struct GNUNET_TIME_Absolute start_time_reg_proc;
354 * Last time we worked before going idle.
356 static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
359 * Time we slept just waiting for work.
361 static struct GNUNET_TIME_Relative idle_time;
365 * Callback for #for_all_records
368 * @param rec a DNS record
371 (*RecordProcessor) (void *cls,
372 const struct GNUNET_DNSPARSER_Record *rec);
376 * Call @a rp for each record in @a p, regardless of
377 * what response section it is in.
379 * @param p packet from DNS
380 * @param rp function to call
381 * @param rp_cls closure for @a rp
384 for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
388 for (unsigned int i=0;i<p->num_answers;i++)
390 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
395 for (unsigned int i=0;i<p->num_authority_records;i++)
397 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
402 for (unsigned int i=0;i<p->num_additional_records;i++)
404 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
413 * Return just the label of the hostname in @a req.
415 * @param req request to process hostname of
416 * @return statically allocated pointer to the label,
417 * overwritten upon the next request!
420 get_label (struct Request *req)
422 static char label[64];
425 dot = strchr (req->hostname,
426 (unsigned char) '.');
432 if (((size_t) (dot - req->hostname)) >= sizeof (label))
437 GNUNET_memcpy (label,
439 dot - req->hostname);
440 label[dot - req->hostname] = '\0';
446 * Build DNS query for @a hostname.
448 * @param hostname host to build query for
449 * @param raw_size[out] number of bytes in the query
450 * @return NULL on error, otherwise pointer to statically (!)
451 * allocated query buffer
454 build_dns_query (struct Request *req,
457 static char raw[512];
459 struct GNUNET_DNSPARSER_Packet p;
460 struct GNUNET_DNSPARSER_Query q;
462 q.name = (char *) req->hostname;
463 q.type = GNUNET_DNSPARSER_TYPE_NS;
464 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
473 GNUNET_DNSPARSER_pack (&p,
478 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
479 "Failed to pack query for hostname `%s'\n",
484 if (*raw_size > sizeof (raw))
486 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
487 "Failed to pack query for hostname `%s'\n",
504 * Free records associated with @a req.
506 * @param req request to free records of
509 free_records (struct Request *req)
514 while (NULL != (rec = req->rec_head))
516 GNUNET_CONTAINER_DLL_remove (req->rec_head,
525 * Free @a req and data structures reachable from it.
527 * @param req request to free
530 free_request (struct Request *req)
538 * Process as many requests as possible from the queue.
543 process_queue (void *cls);
547 * Insert @a req into DLL sorted by next fetch time.
549 * @param req request to insert into #req_heap
552 insert_sorted (struct Request *req)
554 req->hn = GNUNET_CONTAINER_heap_insert (req_heap,
556 req->expires.abs_value_us);
557 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
560 GNUNET_SCHEDULER_cancel (t);
561 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
562 t = GNUNET_SCHEDULER_add_at (req->expires,
570 * Add record to the GNS record set for @a req.
572 * @param req the request to expand GNS record set for
573 * @param type type to use
574 * @param expiration_time when should @a rec expire
575 * @param data raw data to store
576 * @param data_len number of bytes in @a data
579 add_record (struct Request *req,
581 struct GNUNET_TIME_Absolute expiration_time,
587 rec = GNUNET_malloc (sizeof (struct Record) + data_len);
588 rec->grd.data = &rec[1];
589 rec->grd.expiration_time = expiration_time.abs_value_us;
590 rec->grd.data_size = data_len;
591 rec->grd.record_type = type;
592 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
593 GNUNET_memcpy (&rec[1],
596 GNUNET_CONTAINER_DLL_insert (req->rec_head,
603 * Closure for #check_for_glue.
608 * Overall request we are processing.
613 * NS name we are looking for glue for.
618 * Set to #GNUNET_YES if glue was found.
625 * Try to find glue records for a given NS record.
627 * @param cls a `struct GlueClosure *`
628 * @param rec record that may contain glue information
631 check_for_glue (void *cls,
632 const struct GNUNET_DNSPARSER_Record *rec)
634 struct GlueClosure *gc = cls;
638 char ip[INET6_ADDRSTRLEN+1];
639 socklen_t ip_size = (socklen_t) sizeof (ip);
640 struct GNUNET_TIME_Absolute expiration_time;
641 struct GNUNET_TIME_Relative left;
643 if (0 != strcasecmp (rec->name,
646 expiration_time = rec->expiration_time;
647 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
648 if (0 == left.rel_value_us)
649 return; /* ignore expired glue records */
650 /* if expiration window is too short, bump it to configured minimum */
651 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
652 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
653 dst_len = sizeof (dst);
657 case GNUNET_DNSPARSER_TYPE_A:
658 if (sizeof (struct in_addr) != rec->data.raw.data_len)
673 GNUNET_DNSPARSER_builder_add_name (dst,
676 gc->req->hostname)) &&
678 GNUNET_DNSPARSER_builder_add_name (dst,
684 GNUNET_GNSRECORD_TYPE_GNS2DNS,
688 gc->found = GNUNET_YES;
691 case GNUNET_DNSPARSER_TYPE_AAAA:
692 if (sizeof (struct in6_addr) != rec->data.raw.data_len)
707 GNUNET_DNSPARSER_builder_add_name (dst,
710 gc->req->hostname)) &&
712 GNUNET_DNSPARSER_builder_add_name (dst,
718 GNUNET_GNSRECORD_TYPE_GNS2DNS,
722 gc->found = GNUNET_YES;
725 case GNUNET_DNSPARSER_TYPE_CNAME:
727 GNUNET_DNSPARSER_builder_add_name (dst,
730 gc->req->hostname)) &&
732 GNUNET_DNSPARSER_builder_add_name (dst,
735 rec->data.hostname)) )
738 GNUNET_GNSRECORD_TYPE_GNS2DNS,
742 gc->found = GNUNET_YES;
746 /* useless, do nothing */
753 * Closure for #process_record().
755 struct ProcessRecordContext
758 * Answer we got back and are currently parsing, or NULL
761 struct GNUNET_DNSPARSER_Packet *p;
764 * Request we are processing.
771 * We received @a rec for @a req. Remember the answer.
773 * @param cls a `struct ProcessRecordContext`
774 * @param rec response
777 process_record (void *cls,
778 const struct GNUNET_DNSPARSER_Record *rec)
780 struct ProcessRecordContext *prc = cls;
781 struct Request *req = prc->req;
785 struct GNUNET_TIME_Absolute expiration_time;
786 struct GNUNET_TIME_Relative left;
788 dst_len = sizeof (dst);
791 if (0 != strcasecmp (rec->name,
794 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
795 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
797 (unsigned int) rec->type,
799 return; /* does not match hostname, might be glue, but
800 not useful for this pass! */
802 expiration_time = rec->expiration_time;
803 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
804 if (0 == left.rel_value_us)
806 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
807 "DNS returned expired record for `%s'\n",
809 GNUNET_STATISTICS_update (stats,
810 "# expired records obtained from DNS",
813 return; /* record expired */
816 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
817 "DNS returned record that expires at %s for `%s'\n",
818 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
820 /* if expiration window is too short, bump it to configured minimum */
821 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
822 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
825 case GNUNET_DNSPARSER_TYPE_NS:
827 struct GlueClosure gc;
831 gc.ns = rec->data.hostname;
832 gc.found = GNUNET_NO;
833 for_all_records (prc->p,
836 if ( (GNUNET_NO == gc.found) &&
838 GNUNET_DNSPARSER_builder_add_name (dst,
843 GNUNET_DNSPARSER_builder_add_name (dst,
846 rec->data.hostname)) )
848 /* FIXME: actually check if this is out-of-bailiwick,
849 and if not request explicit resolution... */
850 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
851 "Converted OOB (`%s') NS record for `%s'\n",
855 GNUNET_GNSRECORD_TYPE_GNS2DNS,
862 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
863 "Converted NS record for `%s' using glue\n",
868 case GNUNET_DNSPARSER_TYPE_CNAME:
870 GNUNET_DNSPARSER_builder_add_name (dst,
875 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
876 "Converting CNAME (`%s') record for `%s'\n",
886 case GNUNET_DNSPARSER_TYPE_DNAME:
887 /* No support for DNAME in GNS yet! FIXME: support later! */
888 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
889 "FIXME: not supported: %s DNAME %s\n",
893 case GNUNET_DNSPARSER_TYPE_MX:
895 GNUNET_DNSPARSER_builder_add_mx (dst,
900 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
901 "Converting MX (`%s') record for `%s'\n",
902 rec->data.mx->mxhost,
911 case GNUNET_DNSPARSER_TYPE_SOA:
913 GNUNET_DNSPARSER_builder_add_soa (dst,
918 /* NOTE: GNS does not really use SOAs */
919 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
920 "Converting SOA record for `%s'\n",
929 case GNUNET_DNSPARSER_TYPE_SRV:
931 GNUNET_DNSPARSER_builder_add_srv (dst,
936 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
937 "Converting SRV record for `%s'\n",
946 case GNUNET_DNSPARSER_TYPE_PTR:
948 GNUNET_DNSPARSER_builder_add_name (dst,
953 /* !?: what does a PTR record do in a regular TLD??? */
954 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
955 "Converting PTR record for `%s' (weird)\n",
964 case GNUNET_DNSPARSER_TYPE_CERT:
966 GNUNET_DNSPARSER_builder_add_cert (dst,
971 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
972 "Converting CERT record for `%s'\n",
981 /* Rest is 'raw' encoded and just needs to be copied IF
982 the hostname matches the requested name; otherwise we
983 simply cannot use it. */
984 case GNUNET_DNSPARSER_TYPE_A:
985 case GNUNET_DNSPARSER_TYPE_AAAA:
986 case GNUNET_DNSPARSER_TYPE_TXT:
988 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
989 "Converting record of type %u for `%s'\n",
990 (unsigned int) rec->type,
996 rec->data.raw.data_len);
1003 * Continuation called to notify client about result of the
1006 * @param cls closure with our `struct Request`
1007 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
1008 * #GNUNET_NO if content was already there or not found
1009 * #GNUNET_YES (or other positive value) on success
1010 * @param emsg NULL on success, otherwise an error message
1013 store_completed_cb (void *cls,
1017 static struct GNUNET_TIME_Absolute last;
1018 struct Request *req = cls;
1021 if (GNUNET_SYSERR == success)
1023 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1024 "Failed to store zone data for `%s': %s\n",
1030 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1031 "Stored records under `%s' (%d)\n",
1035 total_reg_proc_dns_ns++; /* finished regular processing */
1038 /* compute NAMESTORE statistics */
1040 static uint64_t total_ns_latency_cnt;
1041 static struct GNUNET_TIME_Relative total_ns_latency;
1042 struct GNUNET_TIME_Relative ns_latency;
1044 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1045 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1047 if (0 == total_ns_latency_cnt)
1048 last = GNUNET_TIME_absolute_get ();
1049 total_ns_latency_cnt++;
1050 if (0 == (total_ns_latency_cnt % 1000))
1052 struct GNUNET_TIME_Relative delta;
1054 delta = GNUNET_TIME_absolute_get_duration (last);
1055 last = GNUNET_TIME_absolute_get ();
1057 "Processed 1000 records in %s\n",
1058 GNUNET_STRINGS_relative_time_to_string (delta,
1060 GNUNET_STATISTICS_set (stats,
1061 "# average NAMESTORE PUT latency (μs)",
1062 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1066 /* compute and publish overall velocity */
1067 if (0 == (total_reg_proc_dns_ns % 100) )
1069 struct GNUNET_TIME_Relative runtime;
1071 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1072 runtime = GNUNET_TIME_relative_subtract (runtime,
1074 runtime = GNUNET_TIME_relative_divide (runtime,
1075 total_reg_proc_dns + total_reg_proc_dns_ns);
1076 GNUNET_STATISTICS_set (stats,
1077 "# Regular processing completed without NAMESTORE",
1080 GNUNET_STATISTICS_set (stats,
1081 "# Regular processing completed with NAMESTORE PUT",
1082 total_reg_proc_dns_ns,
1084 GNUNET_STATISTICS_set (stats,
1085 "# average request processing latency (μs)",
1086 runtime.rel_value_us,
1088 GNUNET_STATISTICS_set (stats,
1089 "# total time spent idle (μs)",
1090 idle_time.rel_value_us,
1096 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1097 t = GNUNET_SCHEDULER_add_now (&process_queue,
1104 * Function called with the result of a DNS resolution.
1106 * @param cls closure with the `struct Request`
1107 * @param dns dns response, never NULL
1108 * @param dns_len number of bytes in @a dns
1111 process_result (void *cls,
1112 const struct GNUNET_TUN_DnsHeader *dns,
1115 struct Request *req = cls;
1117 struct GNUNET_DNSPARSER_Packet *p;
1118 unsigned int rd_count;
1120 GNUNET_assert (NULL == req->hn);
1124 GNUNET_CONTAINER_DLL_remove (req_head,
1130 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1131 t = GNUNET_SCHEDULER_add_now (&process_queue,
1134 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1135 "Stub gave up on DNS reply for `%s'\n",
1137 GNUNET_STATISTICS_update (stats,
1138 "# DNS lookups timed out",
1141 if (req->issue_num > MAX_RETRIES)
1145 GNUNET_STATISTICS_update (stats,
1146 "# requests given up on",
1151 total_reg_proc_dns++;
1153 insert_sorted (req);
1156 if (req->id != dns->id)
1158 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1159 "DNS ID did not match request, ignoring reply\n");
1160 GNUNET_STATISTICS_update (stats,
1161 "# DNS ID missmatches",
1166 GNUNET_CONTAINER_DLL_remove (req_head,
1169 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1172 p = GNUNET_DNSPARSER_parse ((const char *) dns,
1176 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1177 "Failed to parse DNS reply for `%s'\n",
1179 GNUNET_STATISTICS_update (stats,
1180 "# DNS parser errors",
1185 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1186 t = GNUNET_SCHEDULER_add_now (&process_queue,
1189 if (req->issue_num > MAX_RETRIES)
1193 GNUNET_STATISTICS_update (stats,
1194 "# requests given up on",
1199 insert_sorted (req);
1202 /* import new records */
1203 req->issue_num = 0; /* success, reset counter! */
1205 struct ProcessRecordContext prc = {
1214 GNUNET_DNSPARSER_free_packet (p);
1215 /* count records found, determine minimum expiration time */
1216 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1218 struct GNUNET_TIME_Relative dns_latency;
1220 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1221 total_dns_latency = GNUNET_TIME_relative_add (total_dns_latency,
1223 total_dns_latency_cnt++;
1224 if (0 == (total_dns_latency_cnt % 1000))
1226 GNUNET_STATISTICS_set (stats,
1227 "# average DNS lookup latency (μs)",
1228 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1233 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1235 struct GNUNET_TIME_Absolute at;
1237 at.abs_value_us = rec->grd.expiration_time;
1238 req->expires = GNUNET_TIME_absolute_min (req->expires,
1242 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1243 "Obtained %u records for `%s'\n",
1246 /* Instead of going for SOA, simplified for now to look each
1247 day in case we got an empty response */
1251 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1252 GNUNET_STATISTICS_update (stats,
1253 "# empty DNS replies (usually NXDOMAIN)",
1261 /* convert records to namestore import format */
1263 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
1264 unsigned int off = 0;
1266 /* convert linked list into array */
1267 for (rec = req->rec_head; NULL != rec; rec =rec->next)
1268 rd[off++] = rec->grd;
1270 req->op_start_time = GNUNET_TIME_absolute_get ();
1271 req->qe = GNUNET_NAMESTORE_records_store (ns,
1276 &store_completed_cb,
1278 GNUNET_assert (NULL != req->qe);
1280 insert_sorted (req);
1285 * Process as many requests as possible from the queue.
1290 process_queue (void *cls)
1292 struct Request *req;
1293 unsigned int series;
1296 struct GNUNET_TIME_Relative delay;
1299 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1300 idle_time = GNUNET_TIME_relative_add (idle_time,
1304 while (pending + pending_rs < THRESH)
1306 req = GNUNET_CONTAINER_heap_peek (req_heap);
1309 if (NULL != req->qe)
1310 return; /* namestore op still pending */
1311 if (NULL != req->rs)
1314 return; /* already submitted */
1316 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1318 GNUNET_assert (req ==
1319 GNUNET_CONTAINER_heap_remove_root (req_heap));
1321 GNUNET_CONTAINER_DLL_insert (req_head,
1324 GNUNET_assert (NULL == req->rs);
1325 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1326 "Requesting resolution for `%s'\n",
1328 raw = build_dns_query (req,
1336 req->op_start_time = GNUNET_TIME_absolute_get ();
1337 req->rs = GNUNET_DNSSTUB_resolve (ctx,
1342 GNUNET_assert (NULL != req->rs);
1347 if (series > MAX_SERIES)
1350 if (pending + pending_rs >= THRESH)
1352 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1353 "Stopped processing queue (%u+%u/%u)]\n",
1357 return; /* wait for replies */
1359 req = GNUNET_CONTAINER_heap_peek (req_heap);
1362 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1363 "Stopped processing queue: empty queue\n");
1366 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1368 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1369 "Waiting until %s for next record (`%s') to expire\n",
1370 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1373 GNUNET_SCHEDULER_cancel (t);
1374 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1375 t = GNUNET_SCHEDULER_add_at (req->expires,
1380 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1383 GNUNET_SCHEDULER_cancel (t);
1384 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1385 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1392 * Iterator called during #do_shutdown() to free requests in
1393 * the #ns_pending map.
1397 * @param value the `struct Request` to free
1398 * @return #GNUNET_OK
1401 free_request_it (void *cls,
1402 const struct GNUNET_HashCode *key,
1405 struct Request *req = value;
1415 * Clean up and terminate the process.
1420 do_shutdown (void *cls)
1422 struct Request *req;
1428 GNUNET_IDENTITY_disconnect (id);
1433 GNUNET_SCHEDULER_cancel (t);
1436 while (NULL != (req = req_head))
1438 GNUNET_CONTAINER_DLL_remove (req_head,
1441 if (NULL != req->qe)
1442 GNUNET_NAMESTORE_cancel (req->qe);
1445 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1448 if (NULL != req->qe)
1449 GNUNET_NAMESTORE_cancel (req->qe);
1452 if (NULL != zone_it)
1454 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1459 GNUNET_NAMESTORE_disconnect (ns);
1464 GNUNET_DNSSTUB_stop (ctx);
1467 if (NULL != req_heap)
1469 GNUNET_CONTAINER_heap_destroy (req_heap);
1472 if (NULL != ns_pending)
1474 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1477 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1480 while (NULL != (zone = zone_head))
1482 GNUNET_CONTAINER_DLL_remove (zone_head,
1485 GNUNET_free (zone->domain);
1490 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1497 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1498 * Just logs an error.
1500 * @param cls a `struct Zone`
1503 ns_lookup_error_cb (void *cls)
1505 struct Zone *zone = cls;
1507 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1508 "Failed to load data from namestore for zone `%s'\n",
1514 * Process a record that was stored in the namestore.
1516 * @param cls a `struct Zone *`
1517 * @param key private key of the zone
1518 * @param label label of the records
1519 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1520 * @param rd array of records with data to store
1523 ns_lookup_result_cb (void *cls,
1524 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1526 unsigned int rd_count,
1527 const struct GNUNET_GNSRECORD_Data *rd)
1529 struct Zone *zone = cls;
1530 struct Request *req;
1531 struct GNUNET_HashCode hc;
1534 ns_iterator_trigger_next--;
1535 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1536 "Obtained NAMESTORE reply, %llu left in round\n",
1537 (unsigned long long) ns_iterator_trigger_next);
1538 if (0 == ns_iterator_trigger_next)
1540 ns_iterator_trigger_next = NS_BATCH_SIZE;
1541 GNUNET_STATISTICS_update (stats,
1542 "# NAMESTORE records requested from cache",
1543 ns_iterator_trigger_next,
1545 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1546 ns_iterator_trigger_next);
1548 GNUNET_asprintf (&fqdn,
1552 GNUNET_CRYPTO_hash (fqdn,
1556 req = GNUNET_CONTAINER_multihashmap_get (ns_pending,
1560 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1561 "Ignoring record `%s' in zone `%s': not on my list!\n",
1566 GNUNET_assert (GNUNET_OK ==
1567 GNUNET_CONTAINER_multihashmap_remove (ns_pending,
1570 GNUNET_break (0 == memcmp (key,
1573 GNUNET_break (0 == strcasecmp (label,
1575 for (unsigned int i=0;i<rd_count;i++)
1577 struct GNUNET_TIME_Absolute at;
1579 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1581 struct GNUNET_TIME_Relative rel;
1583 rel.rel_value_us = rd->expiration_time;
1584 at = GNUNET_TIME_relative_to_absolute (rel);
1588 at.abs_value_us = rd->expiration_time;
1598 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1599 "Empty record set in namestore for `%s'\n",
1604 unsigned int pos = 0;
1607 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1608 for (struct Record *rec = req->rec_head;
1612 struct GNUNET_TIME_Absolute at;
1614 at.abs_value_us = rec->grd.expiration_time;
1615 req->expires = GNUNET_TIME_absolute_min (req->expires,
1620 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1621 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1622 "Hot-start with %u existing records for `%s'\n",
1628 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1629 "Adding `%s' to worklist to start at %s\n",
1631 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1632 insert_sorted (req);
1637 * Add @a hostname to the list of requests to be made.
1639 * @param hostname name to resolve
1642 queue (const char *hostname)
1644 struct Request *req;
1648 struct GNUNET_HashCode hc;
1651 GNUNET_DNSPARSER_check_name (hostname))
1653 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1654 "Refusing invalid hostname `%s'\n",
1659 dot = strchr (hostname,
1660 (unsigned char) '.');
1663 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1664 "Refusing invalid hostname `%s' (lacks '.')\n",
1669 for (zone = zone_head; NULL != zone; zone = zone->next)
1670 if (0 == strcmp (zone->domain,
1676 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1677 "Domain name `%s' not in ego list!\n",
1682 hlen = strlen (hostname) + 1;
1683 req = GNUNET_malloc (sizeof (struct Request) + hlen);
1685 req->hostname = (char *) &req[1];
1686 GNUNET_memcpy (req->hostname,
1689 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1691 GNUNET_CRYPTO_hash (req->hostname,
1695 GNUNET_CONTAINER_multihashmap_put (ns_pending,
1698 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1700 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1701 "Duplicate hostname `%s' ignored\n",
1710 * We have completed the initial iteration over the namestore's database.
1711 * This function is called on each of the remaining records in
1712 * #move_to_queue to #queue() them, as we will simply not find existing
1713 * records for them any longer.
1717 * @param value a `struct Request`
1718 * @return #GNUNET_OK (continue to iterate)
1721 move_to_queue (void *cls,
1722 const struct GNUNET_HashCode *key,
1725 struct Request *req = value;
1729 insert_sorted (req);
1735 * Iterate over all of the zones we care about and see which records
1736 * we may need to re-fetch when.
1741 iterate_zones (void *cls)
1743 static struct Zone *last;
1746 if (NULL != zone_it)
1749 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1750 "Finished iteration over zone `%s'!\n",
1752 /* subtract left-overs from previous iteration */
1753 GNUNET_STATISTICS_update (stats,
1754 "# NAMESTORE records requested from cache",
1755 (long long) (- ns_iterator_trigger_next),
1757 ns_iterator_trigger_next = 0;
1759 GNUNET_assert (NULL != zone_tail);
1760 if (zone_tail == last)
1762 /* Done iterating over relevant zones in NAMESTORE, move
1763 rest of hash map to work queue as well. */
1764 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1765 "Finished all NAMESTORE iterations!\n");
1766 GNUNET_STATISTICS_set (stats,
1767 "# Domain names without cached reply",
1768 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1770 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1773 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1775 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1776 total_reg_proc_dns = 0;
1777 total_reg_proc_dns_ns = 0;
1784 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1785 "Starting iteration over zone `%s'!\n",
1787 /* subtract left-overs from previous iteration */
1788 GNUNET_STATISTICS_update (stats,
1789 "# NAMESTORE records requested from cache",
1792 ns_iterator_trigger_next = 1;
1793 GNUNET_STATISTICS_update (stats,
1797 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1799 &ns_lookup_error_cb,
1801 &ns_lookup_result_cb,
1810 * Begin processing hostnames from stdin.
1815 process_stdin (void *cls)
1817 static struct GNUNET_TIME_Absolute last;
1818 static uint64_t idot;
1825 GNUNET_IDENTITY_disconnect (id);
1834 hn[strlen(hn)-1] = '\0'; /* eat newline */
1836 last = GNUNET_TIME_absolute_get ();
1838 if (0 == idot % 100000)
1840 struct GNUNET_TIME_Relative delta;
1842 delta = GNUNET_TIME_absolute_get_duration (last);
1843 last = GNUNET_TIME_absolute_get ();
1845 "Read 10000 domain names in %s\n",
1846 GNUNET_STRINGS_relative_time_to_string (delta,
1848 GNUNET_STATISTICS_set (stats,
1849 "# domain names provided",
1856 "Done reading %llu domain names\n",
1857 (unsigned long long) idot);
1858 GNUNET_STATISTICS_set (stats,
1859 "# domain names provided",
1862 iterate_zones (NULL);
1867 * Method called to inform about the egos of this peer.
1869 * When used with #GNUNET_IDENTITY_connect, this function is
1870 * initially called for all egos and then again whenever a
1871 * ego's name changes or if it is deleted. At the end of
1872 * the initial pass over all egos, the function is once called
1873 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1874 * be invoked in the future or that there was an error.
1876 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
1877 * this function is only called ONCE, and 'NULL' being passed in
1878 * @a ego does indicate an error (i.e. name is taken or no default
1879 * value is known). If @a ego is non-NULL and if '*ctx'
1880 * is set in those callbacks, the value WILL be passed to a subsequent
1881 * call to the identity callback of #GNUNET_IDENTITY_connect (if
1882 * that one was not NULL).
1884 * When an identity is renamed, this function is called with the
1885 * (known) @a ego but the NEW @a name.
1887 * When an identity is deleted, this function is called with the
1888 * (known) ego and "NULL" for the @a name. In this case,
1889 * the @a ego is henceforth invalid (and the @a ctx should also be
1892 * @param cls closure
1893 * @param ego ego handle
1894 * @param ctx context for application to store data for this ego
1895 * (during the lifetime of this process, initially NULL)
1896 * @param name name assigned by the user for this ego,
1897 * NULL if the user just deleted the ego and it
1898 * must thus no longer be used
1901 identity_cb (void *cls,
1902 struct GNUNET_IDENTITY_Ego *ego,
1910 if (NULL != zone_head)
1912 t = GNUNET_SCHEDULER_add_now (&process_stdin,
1917 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1919 GNUNET_SCHEDULER_shutdown ();
1927 zone = GNUNET_new (struct Zone);
1928 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1929 zone->domain = GNUNET_strdup (name);
1930 GNUNET_CONTAINER_DLL_insert (zone_head,
1938 * Process requests from the queue, then if the queue is
1939 * not empty, try again.
1942 * @param args remaining command-line arguments
1943 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1944 * @param cfg configuration
1949 const char *cfgfile,
1950 const struct GNUNET_CONFIGURATION_Handle *cfg)
1955 stats = GNUNET_STATISTICS_create ("zoneimport",
1957 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1958 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1960 if (NULL == ns_pending)
1963 "Failed to allocate memory for main hash map\n");
1966 ctx = GNUNET_DNSSTUB_start (256);
1970 "Failed to initialize GNUnet DNS STUB\n");
1973 if (NULL == args[0])
1976 "You must provide a list of DNS resolvers on the command line\n");
1979 for (unsigned int i=0;NULL != args[i];i++)
1982 GNUNET_DNSSTUB_add_dns_ip (ctx,
1986 "Failed to use `%s' for DNS resolver\n",
1993 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
1995 ns = GNUNET_NAMESTORE_connect (cfg);
1998 GNUNET_SCHEDULER_shutdown ();
2001 id = GNUNET_IDENTITY_connect (cfg,
2008 * Call with IP address of resolver to query.
2010 * @param argc should be 2
2011 * @param argv[1] should contain IP address
2012 * @return 0 on success
2018 struct GNUNET_GETOPT_CommandLineOption options[] = {
2019 GNUNET_GETOPT_option_uint ('s',
2022 gettext_noop ("size to use for the main hash map"),
2024 GNUNET_GETOPT_option_relative_time ('m',
2025 "minimum-expiration",
2027 gettext_noop ("minimum expiration time we assume for imported records"),
2028 &minimum_expiration_time),
2029 GNUNET_GETOPT_OPTION_END
2033 GNUNET_STRINGS_get_utf8_args (argc, argv,
2036 GNUNET_PROGRAM_run (argc,
2038 "gnunet-zoneimport",
2039 "import DNS zone into namestore",
2043 GNUNET_free ((void*) argv);
2045 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
2046 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
2058 /* end of gnunet-zoneimport.c */