2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
19 * @file src/namestore/gnunet-zoneimport.c
20 * @brief import a DNS zone for publication in GNS, incremental
21 * @author Christian Grothoff
24 #include <gnunet_util_lib.h>
25 #include <gnunet_dnsstub_lib.h>
26 #include <gnunet_dnsparser_lib.h>
27 #include <gnunet_gnsrecord_lib.h>
28 #include <gnunet_namestore_service.h>
29 #include <gnunet_statistics_service.h>
30 #include <gnunet_identity_service.h>
34 * Maximum number of queries pending at the same time.
39 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
40 * Used as an additional throttle.
42 #define TIME_THRESH 10
45 * How often do we retry a query before giving up for good?
50 * How many DNS requests do we at most issue in rapid series?
55 * How long do we wait at least between series of requests?
57 #define SERIES_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MICROSECONDS, 10)
60 * How long do DNS records have to last at least after being imported?
62 static struct GNUNET_TIME_Relative minimum_expiration_time;
65 * How many requests do we request from NAMESTORE in one batch
66 * during our initial iteration?
68 #define NS_BATCH_SIZE 1024
71 * Some zones may include authoritative records for other
72 * zones, such as foo.com.uk or bar.com.fr. As for GNS
73 * each dot represents a zone cut, we then need to create a
74 * zone on-the-fly to capture those records properly.
90 * Domain of the zone (i.e. "fr" or "com.fr")
95 * Private key of the zone.
97 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
103 * Record for the request to be stored by GNS.
120 struct GNUNET_GNSRECORD_Data grd;
126 * Request we should make. We keep this struct in memory per request,
127 * thus optimizing it is crucial for the overall memory consumption of
133 * Requests are kept in a heap while waiting to be resolved.
135 struct GNUNET_CONTAINER_HeapNode *hn;
138 * Active requests are kept in a DLL.
140 struct Request *next;
143 * Active requests are kept in a DLL.
145 struct Request *prev;
148 * Head of records that should be published in GNS for
151 struct Record *rec_head;
154 * Tail of records that should be published in GNS for
157 struct Record *rec_tail;
160 * Socket used to make the request, NULL if not active.
162 struct GNUNET_DNSSTUB_RequestSocket *rs;
165 * Hostname we are resolving, allocated at the end of
166 * this struct (optimizing memory consumption by reducing
167 * total number of allocations).
172 * Namestore operation pending for this record.
174 struct GNUNET_NAMESTORE_QueueEntry *qe;
177 * Zone responsible for this request.
179 const struct Zone *zone;
182 * At what time does the (earliest) of the returned records
183 * for this name expire? At this point, we need to re-fetch
186 struct GNUNET_TIME_Absolute expires;
189 * While we are fetching the record, the value is set to the
190 * starting time of the DNS operation. While doing a
191 * NAMESTORE store, again set to the start time of the
192 * NAMESTORE operation.
194 struct GNUNET_TIME_Absolute op_start_time;
197 * How often did we issue this query? (And failed, reset
198 * to zero once we were successful.)
200 unsigned int issue_num;
203 * random 16-bit DNS query identifier.
210 * Command-line argument specifying desired size of the hash map with
211 * all of our pending names. Usually, we use an automatically growing
212 * map, but this is only OK up to about a million entries. Above that
213 * number, the user must explicitly specify the size at startup.
215 static unsigned int map_size = 1024;
218 * Handle to the identity service.
220 static struct GNUNET_IDENTITY_Handle *id;
225 static struct GNUNET_NAMESTORE_Handle *ns;
228 * Handle to the statistics service.
230 static struct GNUNET_STATISTICS_Handle *stats;
233 * Context for DNS resolution.
235 static struct GNUNET_DNSSTUB_Context *ctx;
238 * The number of DNS queries that are outstanding
240 static unsigned int pending;
243 * The number of NAMESTORE record store operations that are outstanding
245 static unsigned int pending_rs;
248 * Number of lookups we performed overall.
250 static unsigned int lookups;
253 * Number of records we had cached.
255 static unsigned int cached;
258 * How many hostnames did we reject (malformed).
260 static unsigned int rejects;
263 * Number of lookups that failed.
265 static unsigned int failures;
268 * Number of records we found.
270 static unsigned int records;
273 * Number of record sets given to namestore.
275 static unsigned int record_sets;
278 * Heap of all requests to perform, sorted by
279 * the time we should next do the request (i.e. by expires).
281 static struct GNUNET_CONTAINER_Heap *req_heap;
284 * Active requests are kept in a DLL.
286 static struct Request *req_head;
289 * Active requests are kept in a DLL.
291 static struct Request *req_tail;
296 static struct GNUNET_SCHEDULER_Task *t;
299 * Hash map of requests for which we may still get a response from
300 * the namestore. Set to NULL once the initial namestore iteration
303 static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
306 * Current zone iteration handle.
308 static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
311 * Head of list of zones we are managing.
313 static struct Zone *zone_head;
316 * Tail of list of zones we are managing.
318 static struct Zone *zone_tail;
321 * After how many more results must #ns_lookup_result_cb() ask
322 * the namestore for more?
324 static uint64_t ns_iterator_trigger_next;
327 * Number of DNS requests counted in latency total.
329 static uint64_t total_dns_latency_cnt;
332 * Sum of DNS latencies observed.
334 static struct GNUNET_TIME_Relative total_dns_latency;
337 * Number of records processed (DNS lookup, no NAMESTORE) in total.
339 static uint64_t total_reg_proc_dns;
342 * Number of records processed (DNS lookup, with NAMESTORE) in total.
344 static uint64_t total_reg_proc_dns_ns;
347 * Start time of the regular processing.
349 static struct GNUNET_TIME_Absolute start_time_reg_proc;
352 * Last time we worked before going idle.
354 static struct GNUNET_TIME_Absolute sleep_time_reg_proc;
357 * Time we slept just waiting for work.
359 static struct GNUNET_TIME_Relative idle_time;
363 * Callback for #for_all_records
366 * @param rec a DNS record
369 (*RecordProcessor) (void *cls,
370 const struct GNUNET_DNSPARSER_Record *rec);
374 * Call @a rp for each record in @a p, regardless of
375 * what response section it is in.
377 * @param p packet from DNS
378 * @param rp function to call
379 * @param rp_cls closure for @a rp
382 for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
386 for (unsigned int i=0;i<p->num_answers;i++)
388 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
393 for (unsigned int i=0;i<p->num_authority_records;i++)
395 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
400 for (unsigned int i=0;i<p->num_additional_records;i++)
402 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
411 * Return just the label of the hostname in @a req.
413 * @param req request to process hostname of
414 * @return statically allocated pointer to the label,
415 * overwritten upon the next request!
418 get_label (struct Request *req)
420 static char label[64];
423 dot = strchr (req->hostname,
424 (unsigned char) '.');
430 if (((size_t) (dot - req->hostname)) >= sizeof (label))
435 GNUNET_memcpy (label,
437 dot - req->hostname);
438 label[dot - req->hostname] = '\0';
444 * Build DNS query for @a hostname.
446 * @param hostname host to build query for
447 * @param raw_size[out] number of bytes in the query
448 * @return NULL on error, otherwise pointer to statically (!)
449 * allocated query buffer
452 build_dns_query (struct Request *req,
455 static char raw[512];
457 struct GNUNET_DNSPARSER_Packet p;
458 struct GNUNET_DNSPARSER_Query q;
461 q.name = (char *) req->hostname;
462 q.type = GNUNET_DNSPARSER_TYPE_NS;
463 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
471 ret = GNUNET_DNSPARSER_pack (&p,
475 if (GNUNET_OK != ret)
477 if (GNUNET_NO == ret)
479 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
480 "Failed to pack query for hostname `%s'\n",
485 if (*raw_size > sizeof (raw))
487 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
488 "Failed to pack query for hostname `%s'\n",
505 * Free records associated with @a req.
507 * @param req request to free records of
510 free_records (struct Request *req)
515 while (NULL != (rec = req->rec_head))
517 GNUNET_CONTAINER_DLL_remove (req->rec_head,
526 * Free @a req and data structures reachable from it.
528 * @param req request to free
531 free_request (struct Request *req)
539 * Process as many requests as possible from the queue.
544 process_queue (void *cls);
548 * Insert @a req into DLL sorted by next fetch time.
550 * @param req request to insert into #req_heap
553 insert_sorted (struct Request *req)
555 req->hn = GNUNET_CONTAINER_heap_insert (req_heap,
557 req->expires.abs_value_us);
558 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
561 GNUNET_SCHEDULER_cancel (t);
562 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
563 t = GNUNET_SCHEDULER_add_at (req->expires,
571 * Add record to the GNS record set for @a req.
573 * @param req the request to expand GNS record set for
574 * @param type type to use
575 * @param expiration_time when should @a rec expire
576 * @param data raw data to store
577 * @param data_len number of bytes in @a data
580 add_record (struct Request *req,
582 struct GNUNET_TIME_Absolute expiration_time,
588 rec = GNUNET_malloc (sizeof (struct Record) + data_len);
589 rec->grd.data = &rec[1];
590 rec->grd.expiration_time = expiration_time.abs_value_us;
591 rec->grd.data_size = data_len;
592 rec->grd.record_type = type;
593 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
594 GNUNET_memcpy (&rec[1],
597 GNUNET_CONTAINER_DLL_insert (req->rec_head,
604 * Closure for #check_for_glue.
609 * Overall request we are processing.
614 * NS name we are looking for glue for.
619 * Set to #GNUNET_YES if glue was found.
626 * Try to find glue records for a given NS record.
628 * @param cls a `struct GlueClosure *`
629 * @param rec record that may contain glue information
632 check_for_glue (void *cls,
633 const struct GNUNET_DNSPARSER_Record *rec)
635 struct GlueClosure *gc = cls;
639 char ip[INET6_ADDRSTRLEN+1];
640 socklen_t ip_size = (socklen_t) sizeof (ip);
641 struct GNUNET_TIME_Absolute expiration_time;
642 struct GNUNET_TIME_Relative left;
644 if (0 != strcasecmp (rec->name,
647 expiration_time = rec->expiration_time;
648 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
649 if (0 == left.rel_value_us)
650 return; /* ignore expired glue records */
651 /* if expiration window is too short, bump it to configured minimum */
652 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
653 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
654 dst_len = sizeof (dst);
658 case GNUNET_DNSPARSER_TYPE_A:
659 if (sizeof (struct in_addr) != rec->data.raw.data_len)
674 GNUNET_DNSPARSER_builder_add_name (dst,
677 gc->req->hostname)) &&
679 GNUNET_DNSPARSER_builder_add_name (dst,
685 GNUNET_GNSRECORD_TYPE_GNS2DNS,
689 gc->found = GNUNET_YES;
692 case GNUNET_DNSPARSER_TYPE_AAAA:
693 if (sizeof (struct in6_addr) != rec->data.raw.data_len)
708 GNUNET_DNSPARSER_builder_add_name (dst,
711 gc->req->hostname)) &&
713 GNUNET_DNSPARSER_builder_add_name (dst,
719 GNUNET_GNSRECORD_TYPE_GNS2DNS,
723 gc->found = GNUNET_YES;
726 case GNUNET_DNSPARSER_TYPE_CNAME:
728 GNUNET_DNSPARSER_builder_add_name (dst,
731 gc->req->hostname)) &&
733 GNUNET_DNSPARSER_builder_add_name (dst,
736 rec->data.hostname)) )
739 GNUNET_GNSRECORD_TYPE_GNS2DNS,
743 gc->found = GNUNET_YES;
747 /* useless, do nothing */
754 * Closure for #process_record().
756 struct ProcessRecordContext
759 * Answer we got back and are currently parsing, or NULL
762 struct GNUNET_DNSPARSER_Packet *p;
765 * Request we are processing.
772 * We received @a rec for @a req. Remember the answer.
774 * @param cls a `struct ProcessRecordContext`
775 * @param rec response
778 process_record (void *cls,
779 const struct GNUNET_DNSPARSER_Record *rec)
781 struct ProcessRecordContext *prc = cls;
782 struct Request *req = prc->req;
786 struct GNUNET_TIME_Absolute expiration_time;
787 struct GNUNET_TIME_Relative left;
789 dst_len = sizeof (dst);
792 if (0 != strcasecmp (rec->name,
795 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
796 "DNS returned record from zone `%s' of type %u while resolving `%s'\n",
798 (unsigned int) rec->type,
800 return; /* does not match hostname, might be glue, but
801 not useful for this pass! */
803 expiration_time = rec->expiration_time;
804 left = GNUNET_TIME_absolute_get_remaining (expiration_time);
805 if (0 == left.rel_value_us)
807 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
808 "DNS returned expired record for `%s'\n",
810 GNUNET_STATISTICS_update (stats,
811 "# expired records obtained from DNS",
814 return; /* record expired */
817 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
818 "DNS returned record that expires at %s for `%s'\n",
819 GNUNET_STRINGS_absolute_time_to_string (expiration_time),
821 /* if expiration window is too short, bump it to configured minimum */
822 if (left.rel_value_us < minimum_expiration_time.rel_value_us)
823 expiration_time = GNUNET_TIME_relative_to_absolute (minimum_expiration_time);
826 case GNUNET_DNSPARSER_TYPE_NS:
828 struct GlueClosure gc;
832 gc.ns = rec->data.hostname;
833 gc.found = GNUNET_NO;
834 for_all_records (prc->p,
837 if ( (GNUNET_NO == gc.found) &&
839 GNUNET_DNSPARSER_builder_add_name (dst,
844 GNUNET_DNSPARSER_builder_add_name (dst,
847 rec->data.hostname)) )
849 /* FIXME: actually check if this is out-of-bailiwick,
850 and if not request explicit resolution... */
851 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
852 "Converted OOB (`%s') NS record for `%s'\n",
856 GNUNET_GNSRECORD_TYPE_GNS2DNS,
863 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
864 "Converted NS record for `%s' using glue\n",
869 case GNUNET_DNSPARSER_TYPE_CNAME:
871 GNUNET_DNSPARSER_builder_add_name (dst,
876 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
877 "Converting CNAME (`%s') record for `%s'\n",
887 case GNUNET_DNSPARSER_TYPE_DNAME:
888 /* No support for DNAME in GNS yet! FIXME: support later! */
889 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
890 "FIXME: not supported: %s DNAME %s\n",
894 case GNUNET_DNSPARSER_TYPE_MX:
896 GNUNET_DNSPARSER_builder_add_mx (dst,
901 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
902 "Converting MX (`%s') record for `%s'\n",
903 rec->data.mx->mxhost,
912 case GNUNET_DNSPARSER_TYPE_SOA:
914 GNUNET_DNSPARSER_builder_add_soa (dst,
919 /* NOTE: GNS does not really use SOAs */
920 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
921 "Converting SOA record for `%s'\n",
930 case GNUNET_DNSPARSER_TYPE_SRV:
932 GNUNET_DNSPARSER_builder_add_srv (dst,
937 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
938 "Converting SRV record for `%s'\n",
947 case GNUNET_DNSPARSER_TYPE_PTR:
949 GNUNET_DNSPARSER_builder_add_name (dst,
954 /* !?: what does a PTR record do in a regular TLD??? */
955 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
956 "Converting PTR record for `%s' (weird)\n",
965 case GNUNET_DNSPARSER_TYPE_CERT:
967 GNUNET_DNSPARSER_builder_add_cert (dst,
972 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
973 "Converting CERT record for `%s'\n",
982 /* Rest is 'raw' encoded and just needs to be copied IF
983 the hostname matches the requested name; otherwise we
984 simply cannot use it. */
985 case GNUNET_DNSPARSER_TYPE_A:
986 case GNUNET_DNSPARSER_TYPE_AAAA:
987 case GNUNET_DNSPARSER_TYPE_TXT:
989 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
990 "Converting record of type %u for `%s'\n",
991 (unsigned int) rec->type,
997 rec->data.raw.data_len);
1004 * Continuation called to notify client about result of the
1007 * @param cls closure with our `struct Request`
1008 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
1009 * #GNUNET_NO if content was already there or not found
1010 * #GNUNET_YES (or other positive value) on success
1011 * @param emsg NULL on success, otherwise an error message
1014 store_completed_cb (void *cls,
1018 static struct GNUNET_TIME_Absolute last;
1019 struct Request *req = cls;
1022 if (GNUNET_SYSERR == success)
1024 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1025 "Failed to store zone data for `%s': %s\n",
1031 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1032 "Stored records under `%s' (%d)\n",
1036 total_reg_proc_dns_ns++; /* finished regular processing */
1039 /* compute NAMESTORE statistics */
1041 static uint64_t total_ns_latency_cnt;
1042 static struct GNUNET_TIME_Relative total_ns_latency;
1043 struct GNUNET_TIME_Relative ns_latency;
1045 ns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1046 total_ns_latency = GNUNET_TIME_relative_add (total_ns_latency,
1048 if (0 == total_ns_latency_cnt)
1049 last = GNUNET_TIME_absolute_get ();
1050 total_ns_latency_cnt++;
1051 if (0 == (total_ns_latency_cnt % 1000))
1053 struct GNUNET_TIME_Relative delta;
1055 delta = GNUNET_TIME_absolute_get_duration (last);
1056 last = GNUNET_TIME_absolute_get ();
1058 "Processed 1000 records in %s\n",
1059 GNUNET_STRINGS_relative_time_to_string (delta,
1061 GNUNET_STATISTICS_set (stats,
1062 "# average NAMESTORE PUT latency (μs)",
1063 total_ns_latency.rel_value_us / total_ns_latency_cnt,
1067 /* compute and publish overall velocity */
1068 if (0 == (total_reg_proc_dns_ns % 100) )
1070 struct GNUNET_TIME_Relative runtime;
1072 runtime = GNUNET_TIME_absolute_get_duration (start_time_reg_proc);
1073 runtime = GNUNET_TIME_relative_subtract (runtime,
1075 runtime = GNUNET_TIME_relative_divide (runtime,
1076 total_reg_proc_dns + total_reg_proc_dns_ns);
1077 GNUNET_STATISTICS_set (stats,
1078 "# Regular processing completed without NAMESTORE",
1081 GNUNET_STATISTICS_set (stats,
1082 "# Regular processing completed with NAMESTORE PUT",
1083 total_reg_proc_dns_ns,
1085 GNUNET_STATISTICS_set (stats,
1086 "# average request processing latency (μs)",
1087 runtime.rel_value_us,
1089 GNUNET_STATISTICS_set (stats,
1090 "# total time spent idle (μs)",
1091 idle_time.rel_value_us,
1097 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1098 t = GNUNET_SCHEDULER_add_now (&process_queue,
1105 * Function called with the result of a DNS resolution.
1107 * @param cls closure with the `struct Request`
1108 * @param dns dns response, never NULL
1109 * @param dns_len number of bytes in @a dns
1112 process_result (void *cls,
1113 const struct GNUNET_TUN_DnsHeader *dns,
1116 struct Request *req = cls;
1118 struct GNUNET_DNSPARSER_Packet *p;
1119 unsigned int rd_count;
1121 GNUNET_assert (NULL == req->hn);
1125 GNUNET_CONTAINER_DLL_remove (req_head,
1131 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1132 t = GNUNET_SCHEDULER_add_now (&process_queue,
1135 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1136 "Stub gave up on DNS reply for `%s'\n",
1138 GNUNET_STATISTICS_update (stats,
1139 "# DNS lookups timed out",
1142 if (req->issue_num > MAX_RETRIES)
1146 GNUNET_STATISTICS_update (stats,
1147 "# requests given up on",
1152 total_reg_proc_dns++;
1154 insert_sorted (req);
1157 if (req->id != dns->id)
1159 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1160 "DNS ID did not match request, ignoring reply\n");
1161 GNUNET_STATISTICS_update (stats,
1162 "# DNS ID missmatches",
1167 GNUNET_CONTAINER_DLL_remove (req_head,
1170 GNUNET_DNSSTUB_resolve_cancel (req->rs);
1173 p = GNUNET_DNSPARSER_parse ((const char *) dns,
1177 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1178 "Failed to parse DNS reply for `%s'\n",
1180 GNUNET_STATISTICS_update (stats,
1181 "# DNS parser errors",
1186 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1187 t = GNUNET_SCHEDULER_add_now (&process_queue,
1190 if (req->issue_num > MAX_RETRIES)
1194 GNUNET_STATISTICS_update (stats,
1195 "# requests given up on",
1200 insert_sorted (req);
1203 /* import new records */
1204 req->issue_num = 0; /* success, reset counter! */
1206 struct ProcessRecordContext prc = {
1215 GNUNET_DNSPARSER_free_packet (p);
1216 /* count records found, determine minimum expiration time */
1217 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1219 struct GNUNET_TIME_Relative dns_latency;
1221 dns_latency = GNUNET_TIME_absolute_get_duration (req->op_start_time);
1222 total_dns_latency = GNUNET_TIME_relative_add (total_dns_latency,
1224 total_dns_latency_cnt++;
1225 if (0 == (total_dns_latency_cnt % 1000))
1227 GNUNET_STATISTICS_set (stats,
1228 "# average DNS lookup latency (μs)",
1229 total_dns_latency.rel_value_us / total_dns_latency_cnt,
1234 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1236 struct GNUNET_TIME_Absolute at;
1238 at.abs_value_us = rec->grd.expiration_time;
1239 req->expires = GNUNET_TIME_absolute_min (req->expires,
1243 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1244 "Obtained %u records for `%s'\n",
1247 /* Instead of going for SOA, simplified for now to look each
1248 day in case we got an empty response */
1252 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1253 GNUNET_STATISTICS_update (stats,
1254 "# empty DNS replies (usually NXDOMAIN)",
1262 /* convert records to namestore import format */
1264 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
1265 unsigned int off = 0;
1267 /* convert linked list into array */
1268 for (rec = req->rec_head; NULL != rec; rec =rec->next)
1269 rd[off++] = rec->grd;
1271 req->op_start_time = GNUNET_TIME_absolute_get ();
1272 req->qe = GNUNET_NAMESTORE_records_store (ns,
1277 &store_completed_cb,
1279 GNUNET_assert (NULL != req->qe);
1281 insert_sorted (req);
1286 * Process as many requests as possible from the queue.
1291 process_queue (void *cls)
1293 struct Request *req;
1294 unsigned int series;
1297 struct GNUNET_TIME_Relative delay;
1300 delay = GNUNET_TIME_absolute_get_duration (sleep_time_reg_proc);
1301 idle_time = GNUNET_TIME_relative_add (idle_time,
1305 while (pending + pending_rs < THRESH)
1307 req = GNUNET_CONTAINER_heap_peek (req_heap);
1310 if (NULL != req->qe)
1311 return; /* namestore op still pending */
1312 if (NULL != req->rs)
1315 return; /* already submitted */
1317 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1319 GNUNET_assert (req ==
1320 GNUNET_CONTAINER_heap_remove_root (req_heap));
1322 GNUNET_CONTAINER_DLL_insert (req_head,
1325 GNUNET_assert (NULL == req->rs);
1326 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1327 "Requesting resolution for `%s'\n",
1329 raw = build_dns_query (req,
1337 req->op_start_time = GNUNET_TIME_absolute_get ();
1338 req->rs = GNUNET_DNSSTUB_resolve (ctx,
1343 GNUNET_assert (NULL != req->rs);
1348 if (series > MAX_SERIES)
1351 if (pending + pending_rs >= THRESH)
1353 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1354 "Stopped processing queue (%u+%u/%u)]\n",
1358 return; /* wait for replies */
1360 req = GNUNET_CONTAINER_heap_peek (req_heap);
1363 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1364 "Stopped processing queue: empty queue\n");
1367 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1369 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1370 "Waiting until %s for next record (`%s') to expire\n",
1371 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1374 GNUNET_SCHEDULER_cancel (t);
1375 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1376 t = GNUNET_SCHEDULER_add_at (req->expires,
1381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1384 GNUNET_SCHEDULER_cancel (t);
1385 sleep_time_reg_proc = GNUNET_TIME_absolute_get ();
1386 t = GNUNET_SCHEDULER_add_delayed (SERIES_DELAY,
1393 * Iterator called during #do_shutdown() to free requests in
1394 * the #ns_pending map.
1398 * @param value the `struct Request` to free
1399 * @return #GNUNET_OK
1402 free_request_it (void *cls,
1403 const struct GNUNET_HashCode *key,
1406 struct Request *req = value;
1416 * Clean up and terminate the process.
1421 do_shutdown (void *cls)
1423 struct Request *req;
1429 GNUNET_IDENTITY_disconnect (id);
1434 GNUNET_SCHEDULER_cancel (t);
1437 while (NULL != (req = req_head))
1439 GNUNET_CONTAINER_DLL_remove (req_head,
1442 if (NULL != req->qe)
1443 GNUNET_NAMESTORE_cancel (req->qe);
1446 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1449 if (NULL != req->qe)
1450 GNUNET_NAMESTORE_cancel (req->qe);
1453 if (NULL != zone_it)
1455 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1460 GNUNET_NAMESTORE_disconnect (ns);
1465 GNUNET_DNSSTUB_stop (ctx);
1468 if (NULL != req_heap)
1470 GNUNET_CONTAINER_heap_destroy (req_heap);
1473 if (NULL != ns_pending)
1475 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1478 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1481 while (NULL != (zone = zone_head))
1483 GNUNET_CONTAINER_DLL_remove (zone_head,
1486 GNUNET_free (zone->domain);
1491 GNUNET_STATISTICS_destroy (stats, GNUNET_NO);
1498 * Iterate over all of the zones we care about and see which records
1499 * we may need to re-fetch when.
1504 iterate_zones (void *cls);
1508 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1509 * Just logs an error.
1511 * @param cls a `struct Zone`
1514 ns_lookup_error_cb (void *cls)
1516 struct Zone *zone = cls;
1518 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1519 "Failed to load data from namestore for zone `%s'\n",
1522 ns_iterator_trigger_next = 0;
1523 iterate_zones (NULL);
1528 * Process a record that was stored in the namestore.
1530 * @param cls a `struct Zone *`
1531 * @param key private key of the zone
1532 * @param label label of the records
1533 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1534 * @param rd array of records with data to store
1537 ns_lookup_result_cb (void *cls,
1538 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1540 unsigned int rd_count,
1541 const struct GNUNET_GNSRECORD_Data *rd)
1543 struct Zone *zone = cls;
1544 struct Request *req;
1545 struct GNUNET_HashCode hc;
1548 ns_iterator_trigger_next--;
1549 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1550 "Obtained NAMESTORE reply, %llu left in round\n",
1551 (unsigned long long) ns_iterator_trigger_next);
1552 if (0 == ns_iterator_trigger_next)
1554 ns_iterator_trigger_next = NS_BATCH_SIZE;
1555 GNUNET_STATISTICS_update (stats,
1556 "# NAMESTORE records requested from cache",
1557 ns_iterator_trigger_next,
1559 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1560 ns_iterator_trigger_next);
1562 GNUNET_asprintf (&fqdn,
1566 GNUNET_CRYPTO_hash (fqdn,
1570 req = GNUNET_CONTAINER_multihashmap_get (ns_pending,
1574 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1575 "Ignoring record `%s' in zone `%s': not on my list!\n",
1580 GNUNET_assert (GNUNET_OK ==
1581 GNUNET_CONTAINER_multihashmap_remove (ns_pending,
1584 GNUNET_break (0 == memcmp (key,
1587 GNUNET_break (0 == strcasecmp (label,
1589 for (unsigned int i=0;i<rd_count;i++)
1591 struct GNUNET_TIME_Absolute at;
1593 if (0 != (rd->flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
1595 struct GNUNET_TIME_Relative rel;
1597 rel.rel_value_us = rd->expiration_time;
1598 at = GNUNET_TIME_relative_to_absolute (rel);
1602 at.abs_value_us = rd->expiration_time;
1612 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1613 "Empty record set in namestore for `%s'\n",
1618 unsigned int pos = 0;
1621 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1622 for (struct Record *rec = req->rec_head;
1626 struct GNUNET_TIME_Absolute at;
1628 at.abs_value_us = rec->grd.expiration_time;
1629 req->expires = GNUNET_TIME_absolute_min (req->expires,
1634 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1635 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1636 "Hot-start with %u existing records for `%s'\n",
1642 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1643 "Adding `%s' to worklist to start at %s\n",
1645 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1646 insert_sorted (req);
1651 * Add @a hostname to the list of requests to be made.
1653 * @param hostname name to resolve
1656 queue (const char *hostname)
1658 struct Request *req;
1662 struct GNUNET_HashCode hc;
1665 GNUNET_DNSPARSER_check_name (hostname))
1667 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1668 "Refusing invalid hostname `%s'\n",
1673 dot = strchr (hostname,
1674 (unsigned char) '.');
1677 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1678 "Refusing invalid hostname `%s' (lacks '.')\n",
1683 for (zone = zone_head; NULL != zone; zone = zone->next)
1684 if (0 == strcmp (zone->domain,
1690 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1691 "Domain name `%s' not in ego list!\n",
1696 hlen = strlen (hostname) + 1;
1697 req = GNUNET_malloc (sizeof (struct Request) + hlen);
1699 req->hostname = (char *) &req[1];
1700 GNUNET_memcpy (req->hostname,
1703 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1705 GNUNET_CRYPTO_hash (req->hostname,
1709 GNUNET_CONTAINER_multihashmap_put (ns_pending,
1712 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1714 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1715 "Duplicate hostname `%s' ignored\n",
1724 * We have completed the initial iteration over the namestore's database.
1725 * This function is called on each of the remaining records in
1726 * #move_to_queue to #queue() them, as we will simply not find existing
1727 * records for them any longer.
1731 * @param value a `struct Request`
1732 * @return #GNUNET_OK (continue to iterate)
1735 move_to_queue (void *cls,
1736 const struct GNUNET_HashCode *key,
1739 struct Request *req = value;
1743 insert_sorted (req);
1749 * Iterate over all of the zones we care about and see which records
1750 * we may need to re-fetch when.
1755 iterate_zones (void *cls)
1757 static struct Zone *last;
1760 if (NULL != zone_it)
1763 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1764 "Finished iteration over zone `%s'!\n",
1766 /* subtract left-overs from previous iteration */
1767 GNUNET_STATISTICS_update (stats,
1768 "# NAMESTORE records requested from cache",
1769 (long long) (- ns_iterator_trigger_next),
1771 ns_iterator_trigger_next = 0;
1773 GNUNET_assert (NULL != zone_tail);
1774 if (zone_tail == last)
1776 /* Done iterating over relevant zones in NAMESTORE, move
1777 rest of hash map to work queue as well. */
1778 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1779 "Finished all NAMESTORE iterations!\n");
1780 GNUNET_STATISTICS_set (stats,
1781 "# Domain names without cached reply",
1782 GNUNET_CONTAINER_multihashmap_size (ns_pending),
1784 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1787 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1789 start_time_reg_proc = GNUNET_TIME_absolute_get ();
1790 total_reg_proc_dns = 0;
1791 total_reg_proc_dns_ns = 0;
1798 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1799 "Starting iteration over zone `%s'!\n",
1801 /* subtract left-overs from previous iteration */
1802 GNUNET_STATISTICS_update (stats,
1803 "# NAMESTORE records requested from cache",
1806 ns_iterator_trigger_next = 1;
1807 GNUNET_STATISTICS_update (stats,
1811 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1813 &ns_lookup_error_cb,
1815 &ns_lookup_result_cb,
1824 * Begin processing hostnames from stdin.
1829 process_stdin (void *cls)
1831 static struct GNUNET_TIME_Absolute last;
1832 static uint64_t idot;
1839 GNUNET_IDENTITY_disconnect (id);
1848 hn[strlen(hn)-1] = '\0'; /* eat newline */
1850 last = GNUNET_TIME_absolute_get ();
1852 if (0 == idot % 100000)
1854 struct GNUNET_TIME_Relative delta;
1856 delta = GNUNET_TIME_absolute_get_duration (last);
1857 last = GNUNET_TIME_absolute_get ();
1859 "Read 100000 domain names in %s\n",
1860 GNUNET_STRINGS_relative_time_to_string (delta,
1862 GNUNET_STATISTICS_set (stats,
1863 "# domain names provided",
1870 "Done reading %llu domain names\n",
1871 (unsigned long long) idot);
1872 GNUNET_STATISTICS_set (stats,
1873 "# domain names provided",
1876 iterate_zones (NULL);
1881 * Method called to inform about the egos of this peer.
1883 * When used with #GNUNET_IDENTITY_connect, this function is
1884 * initially called for all egos and then again whenever a
1885 * ego's name changes or if it is deleted. At the end of
1886 * the initial pass over all egos, the function is once called
1887 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1888 * be invoked in the future or that there was an error.
1890 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
1891 * this function is only called ONCE, and 'NULL' being passed in
1892 * @a ego does indicate an error (i.e. name is taken or no default
1893 * value is known). If @a ego is non-NULL and if '*ctx'
1894 * is set in those callbacks, the value WILL be passed to a subsequent
1895 * call to the identity callback of #GNUNET_IDENTITY_connect (if
1896 * that one was not NULL).
1898 * When an identity is renamed, this function is called with the
1899 * (known) @a ego but the NEW @a name.
1901 * When an identity is deleted, this function is called with the
1902 * (known) ego and "NULL" for the @a name. In this case,
1903 * the @a ego is henceforth invalid (and the @a ctx should also be
1906 * @param cls closure
1907 * @param ego ego handle
1908 * @param ctx context for application to store data for this ego
1909 * (during the lifetime of this process, initially NULL)
1910 * @param name name assigned by the user for this ego,
1911 * NULL if the user just deleted the ego and it
1912 * must thus no longer be used
1915 identity_cb (void *cls,
1916 struct GNUNET_IDENTITY_Ego *ego,
1924 if (NULL != zone_head)
1926 t = GNUNET_SCHEDULER_add_now (&process_stdin,
1931 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1933 GNUNET_SCHEDULER_shutdown ();
1941 zone = GNUNET_new (struct Zone);
1942 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1943 zone->domain = GNUNET_strdup (name);
1944 GNUNET_CONTAINER_DLL_insert (zone_head,
1952 * Process requests from the queue, then if the queue is
1953 * not empty, try again.
1956 * @param args remaining command-line arguments
1957 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1958 * @param cfg configuration
1963 const char *cfgfile,
1964 const struct GNUNET_CONFIGURATION_Handle *cfg)
1969 stats = GNUNET_STATISTICS_create ("zoneimport",
1971 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1972 ns_pending = GNUNET_CONTAINER_multihashmap_create (map_size,
1974 if (NULL == ns_pending)
1977 "Failed to allocate memory for main hash map\n");
1980 ctx = GNUNET_DNSSTUB_start (256);
1984 "Failed to initialize GNUnet DNS STUB\n");
1987 if (NULL == args[0])
1990 "You must provide a list of DNS resolvers on the command line\n");
1993 for (unsigned int i=0;NULL != args[i];i++)
1996 GNUNET_DNSSTUB_add_dns_ip (ctx,
2000 "Failed to use `%s' for DNS resolver\n",
2007 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
2009 ns = GNUNET_NAMESTORE_connect (cfg);
2012 GNUNET_SCHEDULER_shutdown ();
2015 id = GNUNET_IDENTITY_connect (cfg,
2022 * Call with IP address of resolver to query.
2024 * @param argc should be 2
2025 * @param argv[1] should contain IP address
2026 * @return 0 on success
2032 struct GNUNET_GETOPT_CommandLineOption options[] = {
2033 GNUNET_GETOPT_option_uint ('s',
2036 gettext_noop ("size to use for the main hash map"),
2038 GNUNET_GETOPT_option_relative_time ('m',
2039 "minimum-expiration",
2041 gettext_noop ("minimum expiration time we assume for imported records"),
2042 &minimum_expiration_time),
2043 GNUNET_GETOPT_OPTION_END
2047 GNUNET_STRINGS_get_utf8_args (argc, argv,
2050 GNUNET_PROGRAM_run (argc,
2052 "gnunet-zoneimport",
2053 "import DNS zone into namestore",
2057 GNUNET_free ((void*) argv);
2059 "Rejected %u names, had %u cached, did %u lookups, stored %u record sets\n"
2060 "Found %u records, %u lookups failed, %u/%u pending on shutdown\n",
2072 /* end of gnunet-zoneimport.c */