2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
26 #include <gnunet_util_lib.h>
27 #include <gnunet_dnsstub_lib.h>
28 #include <gnunet_dnsparser_lib.h>
29 #include <gnunet_gnsrecord_lib.h>
30 #include <gnunet_namestore_service.h>
31 #include <gnunet_statistics_service.h>
32 #include <gnunet_identity_service.h>
36 * Maximum number of queries pending at the same time.
41 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
42 * Used as an additional throttle.
44 #define TIME_THRESH 10
47 * How often do we retry a query before giving up for good?
52 * How many DNS requests do we at most issue in rapid series?
57 * How long do we wait at least between series of requests?
59 #define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
62 * How long do DNS records have to last at least after being imported?
64 static struct GNUNET_TIME_Relative minimum_expiration_time;
67 * How many requests do we request from NAMESTORE in one batch
68 * during our initial iteration?
70 #define NS_BATCH_SIZE 1024
73 * Some zones may include authoritative records for other
74 * zones, such as foo.com.uk or bar.com.fr. As for GNS
75 * each dot represents a zone cut, we then need to create a
76 * zone on-the-fly to capture those records properly.
92 * Domain of the zone (i.e. "fr" or "com.fr")
97 * Private key of the zone.
99 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
105 * Record for the request to be stored by GNS.
122 struct GNUNET_GNSRECORD_Data grd;
128 * Request we should make. We keep this struct in memory per request,
129 * thus optimizing it is crucial for the overall memory consumption of
135 * Requests are kept in a heap while waiting to be resolved.
137 struct GNUNET_CONTAINER_HeapNode *hn;
140 * Active requests are kept in a DLL.
142 struct Request *next;
145 * Active requests are kept in a DLL.
147 struct Request *prev;
150 * Head of records that should be published in GNS for
153 struct Record *rec_head;
156 * Tail of records that should be published in GNS for
159 struct Record *rec_tail;
162 * Socket used to make the request, NULL if not active.
164 struct GNUNET_DNSSTUB_RequestSocket *rs;
167 * Hostname we are resolving, allocated at the end of
168 * this struct (optimizing memory consumption by reducing
169 * total number of allocations).
174 * Namestore operation pending for this record.
176 struct GNUNET_NAMESTORE_QueueEntry *qe;
179 * Zone responsible for this request.
181 const struct Zone *zone;
184 * At what time does the (earliest) of the returned records
185 * for this name expire? At this point, we need to re-fetch
188 struct GNUNET_TIME_Absolute expires;
191 * While we are fetching the record, the value is set to the
192 * starting time of the DNS operation. While doing a
193 * NAMESTORE store, again set to the start time of the
194 * NAMESTORE operation.
196 struct GNUNET_TIME_Absolute op_start_time;
199 * How often did we issue this query? (And failed, reset
200 * to zero once we were successful.)
202 unsigned int issue_num;
205 * random 16-bit DNS query identifier.
212 * Command-line argument specifying desired size of the hash map with
213 * all of our pending names. Usually, we use an automatically growing
214 * map, but this is only OK up to about a million entries. Above that
215 * number, the user must explicitly specify the size at startup.
217 static unsigned int map_size = 1024;
220 * Handle to the identity service.
222 static struct GNUNET_IDENTITY_Handle *id;
227 static struct GNUNET_NAMESTORE_Handle *ns;
230 * Handle to the statistics service.
232 static struct GNUNET_STATISTICS_Handle *stats;
235 * Context for DNS resolution.
237 static struct GNUNET_DNSSTUB_Context *ctx;
240 * The number of DNS queries that are outstanding
242 static unsigned int pending;
245 * The number of NAMESTORE record store operations that are outstanding
247 static unsigned int pending_rs;
250 * Number of lookups we performed overall.
252 static unsigned int lookups;
255 * Number of records we had cached.
257 static unsigned int cached;
260 * How many hostnames did we reject (malformed).
262 static unsigned int rejects;
265 * Number of lookups that failed.
267 static unsigned int failures;
270 * Number of records we found.
272 static unsigned int records;
275 * Number of record sets given to namestore.
277 static unsigned int record_sets;
280 * Heap of all requests to perform, sorted by
281 * the time we should next do the request (i.e. by expires).
283 static struct GNUNET_CONTAINER_Heap *req_heap;
286 * Active requests are kept in a DLL.
288 static struct Request *req_head;
291 * Active requests are kept in a DLL.
293 static struct Request *req_tail;
298 static struct GNUNET_SCHEDULER_Task *t;
301 * Hash map of requests for which we may still get a response from
302 * the namestore. Set to NULL once the initial namestore iteration
305 static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
308 * Current zone iteration handle.
310 static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
313 * Head of list of zones we are managing.
315 static struct Zone *zone_head;
318 * Tail of list of zones we are managing.
320 static struct Zone *zone_tail;
323 * After how many more results must #ns_lookup_result_cb() ask
324 * the namestore for more?
326 static uint64_t ns_iterator_trigger_next;
329 * Number of DNS requests counted in latency total.
331 static uint64_t total_dns_latency_cnt;
334 * Sum of DNS latencies observed.
336 static struct GNUNET_TIME_Relative total_dns_latency;
339 * Number of records processed (DNS lookup, no NAMESTORE) in total.
341 static uint64_t total_reg_proc_dns;
344 * Number of records processed (DNS lookup, with NAMESTORE) in total.
346 static uint64_t total_reg_proc_dns_ns;
349 * Start time of the regular processing.
351 static struct GNUNET_TIME_Absolute start_time_reg_proc;
354 * Last time we worked before going idle.
356 static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
359 * Time we slept just waiting for work.
361 static struct GNUNET_TIME_Relative idle_time;
365 * Callback for #for_all_records
368 * @param rec a DNS record
371 (*RecordProcessor) (void *cls,
372 const struct GNUNET_DNSPARSER_Record *rec);
376 * Call @a rp for each record in @a p, regardless of
377 * what response section it is in.
379 * @param p packet from DNS
380 * @param rp function to call
381 * @param rp_cls closure for @a rp
384 for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
388 for (unsigned int i=0;i<p->num_answers;i++)
390 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
395 for (unsigned int i=0;i<p->num_authority_records;i++)
397 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
402 for (unsigned int i=0;i<p->num_additional_records;i++)
404 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
413 * Return just the label of the hostname in @a req.
415 * @param req request to process hostname of
416 * @return statically allocated pointer to the label,
417 * overwritten upon the next request!
420 get_label (struct Request *req)
422 static char label[64];
425 dot = strchr (req->hostname,
426 (unsigned char) '.');
432 if (((size_t) (dot - req->hostname)) >= sizeof (label))
437 GNUNET_memcpy (label,
439 dot - req->hostname);
440 label[dot - req->hostname] = '\0';
446 * Build DNS query for @a hostname.
448 * @param hostname host to build query for
449 * @param raw_size[out] number of bytes in the query
450 * @return NULL on error, otherwise pointer to statically (!)
451 * allocated query buffer
454 build_dns_query (struct Request *req,
457 static char raw[512];
459 struct GNUNET_DNSPARSER_Packet p;
460 struct GNUNET_DNSPARSER_Query q;
463 q.name = (char *) req->hostname;
464 q.type = GNUNET_DNSPARSER_TYPE_NS;
465 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
473 ret = GNUNET_DNSPARSER_pack (&p,
477 if (GNUNET_OK != ret)
479 if (GNUNET_NO == ret)
481 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
482 "Failed to pack query for hostname `%s'\n",
487 if (*raw_size > sizeof (raw))
489 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
490 "Failed to pack query for hostname `%s'\n",
507 * Free records associated with @a req.
509 * @param req request to free records of
512 free_records (struct Request *req)
517 while (NULL != (rec = req->rec_head))
519 GNUNET_CONTAINER_DLL_remove (req->rec_head,
528 * Free @a req and data structures reachable from it.
530 * @param req request to free
533 free_request (struct Request *req)
541 * Process as many requests as possible from the queue.
546 process_queue (void *cls);
550 * Insert @a req into DLL sorted by next fetch time.
552 * @param req request to insert into #req_heap
555 insert_sorted (struct Request *req)
557 req->hn = GNUNET_CONTAINER_heap_insert (req_heap,
559 req->expires.abs_value_us);
560 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
563 GNUNET_SCHEDULER_cancel (t);
564 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
565 t = GNUNET_SCHEDULER_add_at (req->expires,
573 * Add record to the GNS record set for @a req.
575 * @param req the request to expand GNS record set for
576 * @param type type to use
577 * @param expiration_time when should @a rec expire
578 * @param data raw data to store
579 * @param data_len number of bytes in @a data
582 add_record (struct Request *req,
584 struct GNUNET_TIME_Absolute expiration_time,
590 rec = GNUNET_malloc (sizeof (struct Record) + data_len);
591 rec->grd.data = &rec[1];
592 rec->grd.expiration_time = expiration_time.abs_value_us;
593 rec->grd.data_size = data_len;
594 rec->grd.record_type = type;
595 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
596 GNUNET_memcpy (&rec[1],
599 GNUNET_CONTAINER_DLL_insert (req->rec_head,
606 * Closure for #check_for_glue.
611 * Overall request we are processing.
616 * NS name we are looking for glue for.
621 * Set to #GNUNET_YES if glue was found.
628 * Try to find glue records for a given NS record.
630 * @param cls a `struct GlueClosure *`
631 * @param rec record that may contain glue information
634 check_for_glue (void *cls,
635 const struct GNUNET_DNSPARSER_Record *rec)
637 struct GlueClosure *gc = cls;
641 char ip[INET6_ADDRSTRLEN+1];
642 socklen_t ip_size = (socklen_t) sizeof (ip);
643 struct GNUNET_TIME_Absolute expiration_time;
644 struct GNUNET_TIME_Relative left;
646 if (0 != strcasecmp (rec->name,
649 expiration_time = rec->expiration_time;
650 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
651 if (0 == left.rel_value_us)
652 return; /* ignore expired glue records */
653 /* if expiration window is too short, bump it to configured minimum */
654 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
655 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
656 dst_len = sizeof (dst);
660 case GNUNET_DNSPARSER_TYPE_A:
661 if (sizeof (struct in_addr) != rec->data.raw.data_len)
676 GNUNET_DNSPARSER_builder_add_name (dst,
679 gc->req->hostname)) &&
681 GNUNET_DNSPARSER_builder_add_name (dst,
687 GNUNET_GNSRECORD_TYPE_GNS2DNS,
691 gc->found = GNUNET_YES;
694 case GNUNET_DNSPARSER_TYPE_AAAA:
695 if (sizeof (struct in6_addr) != rec->data.raw.data_len)
710 GNUNET_DNSPARSER_builder_add_name (dst,
713 gc->req->hostname)) &&
715 GNUNET_DNSPARSER_builder_add_name (dst,
721 GNUNET_GNSRECORD_TYPE_GNS2DNS,
725 gc->found = GNUNET_YES;
728 case GNUNET_DNSPARSER_TYPE_CNAME:
730 GNUNET_DNSPARSER_builder_add_name (dst,
733 gc->req->hostname)) &&
735 GNUNET_DNSPARSER_builder_add_name (dst,
738 rec->data.hostname)) )
741 GNUNET_GNSRECORD_TYPE_GNS2DNS,
745 gc->found = GNUNET_YES;
749 /* useless, do nothing */
756 * Closure for #process_record().
758 struct ProcessRecordContext
761 * Answer we got back and are currently parsing, or NULL
764 struct GNUNET_DNSPARSER_Packet *p;
767 * Request we are processing.
774 * We received @a rec for @a req. Remember the answer.
776 * @param cls a `struct ProcessRecordContext`
777 * @param rec response
780 process_record (void *cls,
781 const struct GNUNET_DNSPARSER_Record *rec)
783 struct ProcessRecordContext *prc = cls;
784 struct Request *req = prc->req;
788 struct GNUNET_TIME_Absolute expiration_time;
789 struct GNUNET_TIME_Relative left;
791 dst_len = sizeof (dst);
794 if (0 != strcasecmp (rec->name,
797 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
798 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
800 (unsigned int) rec->type,
802 return; /* does not match hostname, might be glue, but
803 not useful for this pass! */
805 expiration_time = rec->expiration_time;
806 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
807 if (0 == left.rel_value_us)
809 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
810 "DNS returned expired record for `%s'\n",
812 GNUNET_STATISTICS_update (stats,
813 "# expired records obtained from DNS",
816 return; /* record expired */
819 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
820 "DNS returned record that expires at %s for `%s'\n",
821 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
823 /* if expiration window is too short, bump it to configured minimum */
824 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
825 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
828 case GNUNET_DNSPARSER_TYPE_NS:
830 struct GlueClosure gc;
834 gc.ns = rec->data.hostname;
835 gc.found = GNUNET_NO;
836 for_all_records (prc->p,
839 if ( (GNUNET_NO == gc.found) &&
841 GNUNET_DNSPARSER_builder_add_name (dst,
846 GNUNET_DNSPARSER_builder_add_name (dst,
849 rec->data.hostname)) )
851 /* FIXME: actually check if this is out-of-bailiwick,
852 and if not request explicit resolution... */
853 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
854 "Converted OOB (`%s') NS record for `%s'\n",
858 GNUNET_GNSRECORD_TYPE_GNS2DNS,
865 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
866 "Converted NS record for `%s' using glue\n",
871 case GNUNET_DNSPARSER_TYPE_CNAME:
873 GNUNET_DNSPARSER_builder_add_name (dst,
878 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
879 "Converting CNAME (`%s') record for `%s'\n",
889 case GNUNET_DNSPARSER_TYPE_DNAME:
890 /* No support for DNAME in GNS yet! FIXME: support later! */
891 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
892 "FIXME: not supported: %s DNAME %s\n",
896 case GNUNET_DNSPARSER_TYPE_MX:
898 GNUNET_DNSPARSER_builder_add_mx (dst,
903 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
904 "Converting MX (`%s') record for `%s'\n",
905 rec->data.mx->mxhost,
914 case GNUNET_DNSPARSER_TYPE_SOA:
916 GNUNET_DNSPARSER_builder_add_soa (dst,
921 /* NOTE: GNS does not really use SOAs */
922 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
923 "Converting SOA record for `%s'\n",
932 case GNUNET_DNSPARSER_TYPE_SRV:
934 GNUNET_DNSPARSER_builder_add_srv (dst,
939 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
940 "Converting SRV record for `%s'\n",
949 case GNUNET_DNSPARSER_TYPE_PTR:
951 GNUNET_DNSPARSER_builder_add_name (dst,
956 /* !?: what does a PTR record do in a regular TLD??? */
957 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
958 "Converting PTR record for `%s' (weird)\n",
967 case GNUNET_DNSPARSER_TYPE_CERT:
969 GNUNET_DNSPARSER_builder_add_cert (dst,
974 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
975 "Converting CERT record for `%s'\n",
984 /* Rest is 'raw' encoded and just needs to be copied IF
985 the hostname matches the requested name; otherwise we
986 simply cannot use it. */
987 case GNUNET_DNSPARSER_TYPE_A:
988 case GNUNET_DNSPARSER_TYPE_AAAA:
989 case GNUNET_DNSPARSER_TYPE_TXT:
991 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
992 "Converting record of type %u for `%s'\n",
993 (unsigned int) rec->type,
999 rec->data.raw.data_len);
1006 * Continuation called to notify client about result of the
1009 * @param cls closure with our `struct Request`
1010 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
1011 * #GNUNET_NO if content was already there or not found
1012 * #GNUNET_YES (or other positive value) on success
1013 * @param emsg NULL on success, otherwise an error message
1016 store_completed_cb (void *cls,
1020 static struct GNUNET_TIME_Absolute last;
1021 struct Request *req = cls;
1024 if (GNUNET_SYSERR == success)
1026 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1027 "Failed to store zone data for `%s': %s\n",
1033 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1034 "Stored records under `%s' (%d)\n",
1038 total_reg_proc_dns_ns++; /* finished regular processing */
1041 /* compute NAMESTORE statistics */
1043 static uint64_t total_ns_latency_cnt;
1044 static struct GNUNET_TIME_Relative total_ns_latency;
1045 struct GNUNET_TIME_Relative ns_latency;
1047 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1048 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1050 if (0 == total_ns_latency_cnt)
1051 last = GNUNET_TIME_absolute_get ();
1052 total_ns_latency_cnt++;
1053 if (0 == (total_ns_latency_cnt % 1000))
1055 struct GNUNET_TIME_Relative delta;
1057 delta = GNUNET_TIME_absolute_get_duration (last);
1058 last = GNUNET_TIME_absolute_get ();
1060 "Processed 1000 records in %s\n",
1061 GNUNET_STRINGS_relative_time_to_string (delta,
1063 GNUNET_STATISTICS_set (stats,
1064 "# average NAMESTORE PUT latency (μs)",
1065 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1069 /* compute and publish overall velocity */
1070 if (0 == (total_reg_proc_dns_ns % 100) )
1072 struct GNUNET_TIME_Relative runtime;
1074 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1075 runtime = GNUNET_TIME_relative_subtract (runtime,
1077 runtime = GNUNET_TIME_relative_divide (runtime,
1078 total_reg_proc_dns + total_reg_proc_dns_ns);
1079 GNUNET_STATISTICS_set (stats,
1080 "# Regular processing completed without NAMESTORE",
1083 GNUNET_STATISTICS_set (stats,
1084 "# Regular processing completed with NAMESTORE PUT",
1085 total_reg_proc_dns_ns,
1087 GNUNET_STATISTICS_set (stats,
1088 "# average request processing latency (μs)",
1089 runtime.rel_value_us,
1091 GNUNET_STATISTICS_set (stats,
1092 "# total time spent idle (μs)",
1093 idle_time.rel_value_us,
1099 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1100 t = GNUNET_SCHEDULER_add_now (&process_queue,
1107 * Function called with the result of a DNS resolution.
1109 * @param cls closure with the `struct Request`
1110 * @param dns dns response, never NULL
1111 * @param dns_len number of bytes in @a dns
1114 process_result (void *cls,
1115 const struct GNUNET_TUN_DnsHeader *dns,
1118 struct Request *req = cls;
1120 struct GNUNET_DNSPARSER_Packet *p;
1121 unsigned int rd_count;
1123 GNUNET_assert (NULL == req->hn);
1127 GNUNET_CONTAINER_DLL_remove (req_head,
1133 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1134 t = GNUNET_SCHEDULER_add_now (&process_queue,
1137 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1138 "Stub gave up on DNS reply for `%s'\n",
1140 GNUNET_STATISTICS_update (stats,
1141 "# DNS lookups timed out",
1144 if (req->issue_num > MAX_RETRIES)
1148 GNUNET_STATISTICS_update (stats,
1149 "# requests given up on",
1154 total_reg_proc_dns++;
1156 insert_sorted (req);
1159 if (req->id != dns->id)
1161 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1162 "DNS ID did not match request, ignoring reply\n");
1163 GNUNET_STATISTICS_update (stats,
1164 "# DNS ID mismatches",
1169 GNUNET_CONTAINER_DLL_remove (req_head,
1172 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1175 p = GNUNET_DNSPARSER_parse ((const char *) dns,
1179 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1180 "Failed to parse DNS reply for `%s'\n",
1182 GNUNET_STATISTICS_update (stats,
1183 "# DNS parser errors",
1188 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1189 t = GNUNET_SCHEDULER_add_now (&process_queue,
1192 if (req->issue_num > MAX_RETRIES)
1196 GNUNET_STATISTICS_update (stats,
1197 "# requests given up on",
1202 insert_sorted (req);
1205 /* import new records */
1206 req->issue_num = 0; /* success, reset counter! */
1208 struct ProcessRecordContext prc = {
1217 GNUNET_DNSPARSER_free_packet (p);
1218 /* count records found, determine minimum expiration time */
1219 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1221 struct GNUNET_TIME_Relative dns_latency;
1223 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1224 total_dns_latency = GNUNET_TIME_relative_add (total_dns_latency,
1226 total_dns_latency_cnt++;
1227 if (0 == (total_dns_latency_cnt % 1000))
1229 GNUNET_STATISTICS_set (stats,
1230 "# average DNS lookup latency (μs)",
1231 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1236 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1238 struct GNUNET_TIME_Absolute at;
1240 at.abs_value_us = rec->grd.expiration_time;
1241 req->expires = GNUNET_TIME_absolute_min (req->expires,
1245 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1246 "Obtained %u records for `%s'\n",
1249 /* Instead of going for SOA, simplified for now to look each
1250 day in case we got an empty response */
1254 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1255 GNUNET_STATISTICS_update (stats,
1256 "# empty DNS replies (usually NXDOMAIN)",
1264 /* convert records to namestore import format */
1266 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
1267 unsigned int off = 0;
1269 /* convert linked list into array */
1270 for (rec = req->rec_head; NULL != rec; rec =rec->next)
1271 rd[off++] = rec->grd;
1273 req->op_start_time = GNUNET_TIME_absolute_get ();
1274 req->qe = GNUNET_NAMESTORE_records_store (ns,
1279 &store_completed_cb,
1281 GNUNET_assert (NULL != req->qe);
1283 insert_sorted (req);
1288 * Process as many requests as possible from the queue.
1293 process_queue (void *cls)
1295 struct Request *req;
1296 unsigned int series;
1299 struct GNUNET_TIME_Relative delay;
1302 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1303 idle_time = GNUNET_TIME_relative_add (idle_time,
1307 while (pending + pending_rs < THRESH)
1309 req = GNUNET_CONTAINER_heap_peek (req_heap);
1312 if (NULL != req->qe)
1313 return; /* namestore op still pending */
1314 if (NULL != req->rs)
1317 return; /* already submitted */
1319 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1321 GNUNET_assert (req ==
1322 GNUNET_CONTAINER_heap_remove_root (req_heap));
1324 GNUNET_CONTAINER_DLL_insert (req_head,
1327 GNUNET_assert (NULL == req->rs);
1328 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1329 "Requesting resolution for `%s'\n",
1331 raw = build_dns_query (req,
1339 req->op_start_time = GNUNET_TIME_absolute_get ();
1340 req->rs = GNUNET_DNSSTUB_resolve (ctx,
1345 GNUNET_assert (NULL != req->rs);
1350 if (series > MAX_SERIES)
1353 if (pending + pending_rs >= THRESH)
1355 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1356 "Stopped processing queue (%u+%u/%u)]\n",
1360 return; /* wait for replies */
1362 req = GNUNET_CONTAINER_heap_peek (req_heap);
1365 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1366 "Stopped processing queue: empty queue\n");
1369 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1371 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1372 "Waiting until %s for next record (`%s') to expire\n",
1373 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1376 GNUNET_SCHEDULER_cancel (t);
1377 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1378 t = GNUNET_SCHEDULER_add_at (req->expires,
1383 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1386 GNUNET_SCHEDULER_cancel (t);
1387 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1388 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1395 * Iterator called during #do_shutdown() to free requests in
1396 * the #ns_pending map.
1400 * @param value the `struct Request` to free
1401 * @return #GNUNET_OK
1404 free_request_it (void *cls,
1405 const struct GNUNET_HashCode *key,
1408 struct Request *req = value;
1418 * Clean up and terminate the process.
1423 do_shutdown (void *cls)
1425 struct Request *req;
1431 GNUNET_IDENTITY_disconnect (id);
1436 GNUNET_SCHEDULER_cancel (t);
1439 while (NULL != (req = req_head))
1441 GNUNET_CONTAINER_DLL_remove (req_head,
1444 if (NULL != req->qe)
1445 GNUNET_NAMESTORE_cancel (req->qe);
1448 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1451 if (NULL != req->qe)
1452 GNUNET_NAMESTORE_cancel (req->qe);
1455 if (NULL != zone_it)
1457 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1462 GNUNET_NAMESTORE_disconnect (ns);
1467 GNUNET_DNSSTUB_stop (ctx);
1470 if (NULL != req_heap)
1472 GNUNET_CONTAINER_heap_destroy (req_heap);
1475 if (NULL != ns_pending)
1477 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1480 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1483 while (NULL != (zone = zone_head))
1485 GNUNET_CONTAINER_DLL_remove (zone_head,
1488 GNUNET_free (zone->domain);
1493 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1500 * Iterate over all of the zones we care about and see which records
1501 * we may need to re-fetch when.
1506 iterate_zones (void *cls);
1510 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1511 * Just logs an error.
1513 * @param cls a `struct Zone`
1516 ns_lookup_error_cb (void *cls)
1518 struct Zone *zone = cls;
1520 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1521 "Failed to load data from namestore for zone `%s'\n",
1524 ns_iterator_trigger_next = 0;
1525 iterate_zones (NULL);
1530 * Process a record that was stored in the namestore.
1532 * @param cls a `struct Zone *`
1533 * @param key private key of the zone
1534 * @param label label of the records
1535 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1536 * @param rd array of records with data to store
1539 ns_lookup_result_cb (void *cls,
1540 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1542 unsigned int rd_count,
1543 const struct GNUNET_GNSRECORD_Data *rd)
1545 struct Zone *zone = cls;
1546 struct Request *req;
1547 struct GNUNET_HashCode hc;
1550 ns_iterator_trigger_next--;
1551 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1552 "Obtained NAMESTORE reply, %llu left in round\n",
1553 (unsigned long long) ns_iterator_trigger_next);
1554 if (0 == ns_iterator_trigger_next)
1556 ns_iterator_trigger_next = NS_BATCH_SIZE;
1557 GNUNET_STATISTICS_update (stats,
1558 "# NAMESTORE records requested from cache",
1559 ns_iterator_trigger_next,
1561 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1562 ns_iterator_trigger_next);
1564 GNUNET_asprintf (&fqdn,
1568 GNUNET_CRYPTO_hash (fqdn,
1572 req = GNUNET_CONTAINER_multihashmap_get (ns_pending,
1576 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1577 "Ignoring record `%s' in zone `%s': not on my list!\n",
1582 GNUNET_assert (GNUNET_OK ==
1583 GNUNET_CONTAINER_multihashmap_remove (ns_pending,
1586 GNUNET_break (0 == memcmp (key,
1589 GNUNET_break (0 == strcasecmp (label,
1591 for (unsigned int i=0;i<rd_count;i++)
1593 struct GNUNET_TIME_Absolute at;
1595 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1597 struct GNUNET_TIME_Relative rel;
1599 rel.rel_value_us = rd->expiration_time;
1600 at = GNUNET_TIME_relative_to_absolute (rel);
1604 at.abs_value_us = rd->expiration_time;
1614 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1615 "Empty record set in namestore for `%s'\n",
1620 unsigned int pos = 0;
1623 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1624 for (struct Record *rec = req->rec_head;
1628 struct GNUNET_TIME_Absolute at;
1630 at.abs_value_us = rec->grd.expiration_time;
1631 req->expires = GNUNET_TIME_absolute_min (req->expires,
1636 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1637 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1638 "Hot-start with %u existing records for `%s'\n",
1644 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1645 "Adding `%s' to worklist to start at %s\n",
1647 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1648 insert_sorted (req);
1653 * Add @a hostname to the list of requests to be made.
1655 * @param hostname name to resolve
1658 queue (const char *hostname)
1660 struct Request *req;
1664 struct GNUNET_HashCode hc;
1667 GNUNET_DNSPARSER_check_name (hostname))
1669 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1670 "Refusing invalid hostname `%s'\n",
1675 dot = strchr (hostname,
1676 (unsigned char) '.');
1679 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1680 "Refusing invalid hostname `%s' (lacks '.')\n",
1685 for (zone = zone_head; NULL != zone; zone = zone->next)
1686 if (0 == strcmp (zone->domain,
1692 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1693 "Domain name `%s' not in ego list!\n",
1698 hlen = strlen (hostname) + 1;
1699 req = GNUNET_malloc (sizeof (struct Request) + hlen);
1701 req->hostname = (char *) &req[1];
1702 GNUNET_memcpy (req->hostname,
1705 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1707 GNUNET_CRYPTO_hash (req->hostname,
1711 GNUNET_CONTAINER_multihashmap_put (ns_pending,
1714 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1716 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1717 "Duplicate hostname `%s' ignored\n",
1726 * We have completed the initial iteration over the namestore's database.
1727 * This function is called on each of the remaining records in
1728 * #move_to_queue to #queue() them, as we will simply not find existing
1729 * records for them any longer.
1733 * @param value a `struct Request`
1734 * @return #GNUNET_OK (continue to iterate)
1737 move_to_queue (void *cls,
1738 const struct GNUNET_HashCode *key,
1741 struct Request *req = value;
1745 insert_sorted (req);
1751 * Iterate over all of the zones we care about and see which records
1752 * we may need to re-fetch when.
1757 iterate_zones (void *cls)
1759 static struct Zone *last;
1762 if (NULL != zone_it)
1765 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1766 "Finished iteration over zone `%s'!\n",
1768 /* subtract left-overs from previous iteration */
1769 GNUNET_STATISTICS_update (stats,
1770 "# NAMESTORE records requested from cache",
1771 (long long) (- ns_iterator_trigger_next),
1773 ns_iterator_trigger_next = 0;
1775 GNUNET_assert (NULL != zone_tail);
1776 if (zone_tail == last)
1778 /* Done iterating over relevant zones in NAMESTORE, move
1779 rest of hash map to work queue as well. */
1780 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1781 "Finished all NAMESTORE iterations!\n");
1782 GNUNET_STATISTICS_set (stats,
1783 "# Domain names without cached reply",
1784 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1786 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1789 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1791 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1792 total_reg_proc_dns = 0;
1793 total_reg_proc_dns_ns = 0;
1800 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1801 "Starting iteration over zone `%s'!\n",
1803 /* subtract left-overs from previous iteration */
1804 GNUNET_STATISTICS_update (stats,
1805 "# NAMESTORE records requested from cache",
1808 ns_iterator_trigger_next = 1;
1809 GNUNET_STATISTICS_update (stats,
1813 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1815 &ns_lookup_error_cb,
1817 &ns_lookup_result_cb,
1826 * Begin processing hostnames from stdin.
1831 process_stdin (void *cls)
1833 static struct GNUNET_TIME_Absolute last;
1834 static uint64_t idot;
1841 GNUNET_IDENTITY_disconnect (id);
1850 hn[strlen(hn)-1] = '\0'; /* eat newline */
1852 last = GNUNET_TIME_absolute_get ();
1854 if (0 == idot % 100000)
1856 struct GNUNET_TIME_Relative delta;
1858 delta = GNUNET_TIME_absolute_get_duration (last);
1859 last = GNUNET_TIME_absolute_get ();
1861 "Read 100000 domain names in %s\n",
1862 GNUNET_STRINGS_relative_time_to_string (delta,
1864 GNUNET_STATISTICS_set (stats,
1865 "# domain names provided",
1872 "Done reading %llu domain names\n",
1873 (unsigned long long) idot);
1874 GNUNET_STATISTICS_set (stats,
1875 "# domain names provided",
1878 iterate_zones (NULL);
1883 * Method called to inform about the egos of this peer.
1885 * When used with #GNUNET_IDENTITY_connect, this function is
1886 * initially called for all egos and then again whenever a
1887 * ego's name changes or if it is deleted. At the end of
1888 * the initial pass over all egos, the function is once called
1889 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1890 * be invoked in the future or that there was an error.
1892 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
1893 * this function is only called ONCE, and 'NULL' being passed in
1894 * @a ego does indicate an error (i.e. name is taken or no default
1895 * value is known). If @a ego is non-NULL and if '*ctx'
1896 * is set in those callbacks, the value WILL be passed to a subsequent
1897 * call to the identity callback of #GNUNET_IDENTITY_connect (if
1898 * that one was not NULL).
1900 * When an identity is renamed, this function is called with the
1901 * (known) @a ego but the NEW @a name.
1903 * When an identity is deleted, this function is called with the
1904 * (known) ego and "NULL" for the @a name. In this case,
1905 * the @a ego is henceforth invalid (and the @a ctx should also be
1908 * @param cls closure
1909 * @param ego ego handle
1910 * @param ctx context for application to store data for this ego
1911 * (during the lifetime of this process, initially NULL)
1912 * @param name name assigned by the user for this ego,
1913 * NULL if the user just deleted the ego and it
1914 * must thus no longer be used
1917 identity_cb (void *cls,
1918 struct GNUNET_IDENTITY_Ego *ego,
1926 if (NULL != zone_head)
1928 t = GNUNET_SCHEDULER_add_now (&process_stdin,
1933 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1935 GNUNET_SCHEDULER_shutdown ();
1943 zone = GNUNET_new (struct Zone);
1944 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1945 zone->domain = GNUNET_strdup (name);
1946 GNUNET_CONTAINER_DLL_insert (zone_head,
1954 * Process requests from the queue, then if the queue is
1955 * not empty, try again.
1958 * @param args remaining command-line arguments
1959 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1960 * @param cfg configuration
1965 const char *cfgfile,
1966 const struct GNUNET_CONFIGURATION_Handle *cfg)
1971 stats = GNUNET_STATISTICS_create ("zoneimport",
1973 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1974 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1976 if (NULL == ns_pending)
1979 "Failed to allocate memory for main hash map\n");
1982 ctx = GNUNET_DNSSTUB_start (256);
1986 "Failed to initialize GNUnet DNS STUB\n");
1989 if (NULL == args[0])
1992 "You must provide a list of DNS resolvers on the command line\n");
1995 for (unsigned int i=0;NULL != args[i];i++)
1998 GNUNET_DNSSTUB_add_dns_ip (ctx,
2002 "Failed to use `%s' for DNS resolver\n",
2009 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
2011 ns = GNUNET_NAMESTORE_connect (cfg);
2014 GNUNET_SCHEDULER_shutdown ();
2017 id = GNUNET_IDENTITY_connect (cfg,
2024 * Call with IP address of resolver to query.
2026 * @param argc should be 2
2027 * @param argv[1] should contain IP address
2028 * @return 0 on success
2034 struct GNUNET_GETOPT_CommandLineOption options[] = {
2035 GNUNET_GETOPT_option_uint ('s',
2038 gettext_noop ("size to use for the main hash map"),
2040 GNUNET_GETOPT_option_relative_time ('m',
2041 "minimum-expiration",
2043 gettext_noop ("minimum expiration time we assume for imported records"),
2044 &minimum_expiration_time),
2045 GNUNET_GETOPT_OPTION_END
2050 GNUNET_STRINGS_get_utf8_args (argc, argv,
2054 (ret = GNUNET_PROGRAM_run (argc,
2056 "gnunet-zoneimport",
2057 "import DNS zone into namestore",
2062 GNUNET_free ((void*) argv);
2064 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
2065 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
2077 /* end of gnunet-zoneimport.c */