2 This file is part of GNUnet
3 Copyright (C) 2018 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
21 * @file src/namestore/gnunet-zoneimport.c
22 * @brief import a DNS zone for publication in GNS, incremental
23 * @author Christian Grothoff
26 #include <gnunet_util_lib.h>
27 #include <gnunet_dnsstub_lib.h>
28 #include <gnunet_dnsparser_lib.h>
29 #include <gnunet_gnsrecord_lib.h>
30 #include <gnunet_namestore_service.h>
31 #include <gnunet_identity_service.h>
35 * Maximum number of queries pending at the same time.
40 * TIME_THRESH is in usecs. How quickly do we submit fresh queries.
41 * Used as an additional throttle.
43 #define TIME_THRESH 10
46 * How often do we retry a query before giving up for good?
51 * How many requests do we request from NAMESTORE in one batch
52 * during our initial iteration?
54 #define NS_BATCH_SIZE 1024
57 * Some zones may include authoritative records for other
58 * zones, such as foo.com.uk or bar.com.fr. As for GNS
59 * each dot represents a zone cut, we then need to create a
60 * zone on-the-fly to capture those records properly.
76 * Domain of the zone (i.e. "fr" or "com.fr")
81 * Private key of the zone.
83 struct GNUNET_CRYPTO_EcdsaPrivateKey key;
89 * Record for the request to be stored by GNS.
106 struct GNUNET_GNSRECORD_Data grd;
112 * Request we should make. We keep this struct in memory per request,
113 * thus optimizing it is crucial for the overall memory consumption of
119 * Requests are kept in a heap while waiting to be resolved.
121 struct GNUNET_CONTAINER_HeapNode *hn;
124 * Active requests are kept in a DLL.
126 struct Request *next;
129 * Active requests are kept in a DLL.
131 struct Request *prev;
134 * Head of records that should be published in GNS for
137 struct Record *rec_head;
140 * Tail of records that should be published in GNS for
143 struct Record *rec_tail;
146 * Socket used to make the request, NULL if not active.
148 struct GNUNET_DNSSTUB_RequestSocket *rs;
151 * Hostname we are resolving, allocated at the end of
152 * this struct (optimizing memory consumption by reducing
153 * total number of allocations).
158 * Namestore operation pending for this record.
160 struct GNUNET_NAMESTORE_QueueEntry *qe;
163 * Zone responsible for this request.
165 const struct Zone *zone;
168 * At what time does the (earliest) of the returned records
169 * for this name expire? At this point, we need to re-fetch
172 struct GNUNET_TIME_Absolute expires;
175 * How often did we issue this query? (And failed, reset
176 * to zero once we were successful.)
178 unsigned int issue_num;
181 * random 16-bit DNS query identifier.
188 * Handle to the identity service.
190 static struct GNUNET_IDENTITY_Handle *id;
195 static struct GNUNET_NAMESTORE_Handle *ns;
198 * Context for DNS resolution.
200 static struct GNUNET_DNSSTUB_Context *ctx;
203 * The number of queries that are outstanding
205 static unsigned int pending;
208 * Number of lookups we performed overall.
210 static unsigned int lookups;
213 * How many hostnames did we reject (malformed).
215 static unsigned int rejects;
218 * Number of lookups that failed.
220 static unsigned int failures;
223 * Number of records we found.
225 static unsigned int records;
228 * Heap of all requests to perform, sorted by
229 * the time we should next do the request (i.e. by expires).
231 static struct GNUNET_CONTAINER_Heap *req_heap;
234 * Active requests are kept in a DLL.
236 static struct Request *req_head;
239 * Active requests are kept in a DLL.
241 static struct Request *req_tail;
246 static struct GNUNET_SCHEDULER_Task *t;
249 * Hash map of requests for which we may still get a response from
250 * the namestore. Set to NULL once the initial namestore iteration
253 static struct GNUNET_CONTAINER_MultiHashMap *ns_pending;
256 * Current zone iteration handle.
258 static struct GNUNET_NAMESTORE_ZoneIterator *zone_it;
261 * Head of list of zones we are managing.
263 static struct Zone *zone_head;
266 * Tail of list of zones we are managing.
268 static struct Zone *zone_tail;
271 * After how many more results must #ns_lookup_result_cb() ask
272 * the namestore for more?
274 static uint64_t ns_iterator_trigger_next;
277 * Callback for #for_all_records
280 * @param rec a DNS record
283 (*RecordProcessor) (void *cls,
284 const struct GNUNET_DNSPARSER_Record *rec);
288 * Call @a rp for each record in @a p, regardless of
289 * what response section it is in.
291 * @param p packet from DNS
292 * @param rp function to call
293 * @param rp_cls closure for @a rp
296 for_all_records (const struct GNUNET_DNSPARSER_Packet *p,
300 for (unsigned int i=0;i<p->num_answers;i++)
302 struct GNUNET_DNSPARSER_Record *rs = &p->answers[i];
307 for (unsigned int i=0;i<p->num_authority_records;i++)
309 struct GNUNET_DNSPARSER_Record *rs = &p->authority_records[i];
314 for (unsigned int i=0;i<p->num_additional_records;i++)
316 struct GNUNET_DNSPARSER_Record *rs = &p->additional_records[i];
325 * Return just the label of the hostname in @a req.
327 * @param req request to process hostname of
328 * @return statically allocated pointer to the label,
329 * overwritten upon the next request!
332 get_label (struct Request *req)
334 static char label[64];
337 dot = strchr (req->hostname,
338 (unsigned char) '.');
344 if (((size_t) (dot - req->hostname)) >= sizeof (label))
351 dot - req->hostname);
352 label[dot - req->hostname] = '\0';
358 * Build DNS query for @a hostname.
360 * @param hostname host to build query for
361 * @param raw_size[out] number of bytes in the query
362 * @return NULL on error, otherwise pointer to statically (!)
363 * allocated query buffer
366 build_dns_query (struct Request *req,
369 static char raw[512];
371 struct GNUNET_DNSPARSER_Packet p;
372 struct GNUNET_DNSPARSER_Query q;
374 q.name = (char *) req->hostname;
375 q.type = GNUNET_DNSPARSER_TYPE_NS;
376 q.dns_traffic_class = GNUNET_TUN_DNS_CLASS_INTERNET;
385 GNUNET_DNSPARSER_pack (&p,
390 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
391 "Failed to pack query for hostname `%s'\n",
396 if (*raw_size > sizeof (raw))
398 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
399 "Failed to pack query for hostname `%s'\n",
416 * Free records associated with @a req.
418 * @param req request to free records of
421 free_records (struct Request *req)
426 while (NULL != (rec = req->rec_head))
428 GNUNET_CONTAINER_DLL_remove (req->rec_head,
437 * Free @a req and data structures reachable from it.
439 * @param req request to free
442 free_request (struct Request *req)
450 * Process as many requests as possible from the queue.
455 process_queue (void *cls);
459 * Insert @a req into DLL sorted by next fetch time.
461 * @param req request to insert into #req_heap
464 insert_sorted (struct Request *req)
466 req->hn = GNUNET_CONTAINER_heap_insert (req_heap,
468 req->expires.abs_value_us);
469 if (req == GNUNET_CONTAINER_heap_peek (req_heap))
472 GNUNET_SCHEDULER_cancel (t);
473 t = GNUNET_SCHEDULER_add_at (req->expires,
481 * Add record to the GNS record set for @a req.
483 * @param req the request to expand GNS record set for
484 * @param type type to use
485 * @param expiration_time when should @a rec expire
486 * @param data raw data to store
487 * @param data_len number of bytes in @a data
490 add_record (struct Request *req,
492 struct GNUNET_TIME_Absolute expiration_time,
498 rec = GNUNET_malloc (sizeof (struct Record) + data_len);
499 rec->grd.data = &rec[1];
500 rec->grd.expiration_time = expiration_time.abs_value_us;
501 rec->grd.data_size = data_len;
502 rec->grd.record_type = type;
503 rec->grd.flags = GNUNET_GNSRECORD_RF_NONE;
504 GNUNET_memcpy (&rec[1],
507 GNUNET_CONTAINER_DLL_insert (req->rec_head,
514 * Closure for #check_for_glue.
519 * Overall request we are processing.
524 * NS name we are looking for glue for.
529 * Set to #GNUNET_YES if glue was found.
536 * Try to find glue records for a given NS record.
538 * @param cls a `struct GlueClosure *`
539 * @param rec record that may contain glue information
542 check_for_glue (void *cls,
543 const struct GNUNET_DNSPARSER_Record *rec)
545 struct GlueClosure *gc = cls;
549 char ip[INET6_ADDRSTRLEN+1];
550 socklen_t ip_size = (socklen_t) sizeof (ip);
552 if (0 != strcasecmp (rec->name,
555 dst_len = sizeof (dst);
559 case GNUNET_DNSPARSER_TYPE_A:
560 if (sizeof (struct in_addr) != rec->data.raw.data_len)
575 GNUNET_DNSPARSER_builder_add_name (dst,
578 gc->req->hostname)) &&
580 GNUNET_DNSPARSER_builder_add_name (dst,
586 GNUNET_GNSRECORD_TYPE_GNS2DNS,
587 rec->expiration_time,
590 gc->found = GNUNET_YES;
593 case GNUNET_DNSPARSER_TYPE_AAAA:
594 if (sizeof (struct in6_addr) != rec->data.raw.data_len)
609 GNUNET_DNSPARSER_builder_add_name (dst,
612 gc->req->hostname)) &&
614 GNUNET_DNSPARSER_builder_add_name (dst,
620 GNUNET_GNSRECORD_TYPE_GNS2DNS,
621 rec->expiration_time,
624 gc->found = GNUNET_YES;
627 case GNUNET_DNSPARSER_TYPE_CNAME:
629 GNUNET_DNSPARSER_builder_add_name (dst,
632 gc->req->hostname)) &&
634 GNUNET_DNSPARSER_builder_add_name (dst,
637 rec->data.hostname)) )
640 GNUNET_GNSRECORD_TYPE_GNS2DNS,
641 rec->expiration_time,
644 gc->found = GNUNET_YES;
648 /* useless, do nothing */
655 * Closure for #process_record().
657 struct ProcessRecordContext
660 * Answer we got back and are currently parsing, or NULL
663 struct GNUNET_DNSPARSER_Packet *p;
666 * Request we are processing.
673 * We received @a rec for @a req. Remember the answer.
675 * @param cls a `struct ProcessRecordContext`
676 * @param rec response
679 process_record (void *cls,
680 const struct GNUNET_DNSPARSER_Record *rec)
682 struct ProcessRecordContext *prc = cls;
683 struct Request *req = prc->req;
688 dst_len = sizeof (dst);
691 if (0 != strcasecmp (rec->name,
694 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
695 "DNS returned record for `%s' of type %u while resolving `%s'\n",
697 (unsigned int) rec->type,
699 return; /* does not match hostname, might be glue, but
700 not useful for this pass! */
703 GNUNET_TIME_absolute_get_remaining (rec->expiration_time).rel_value_us)
705 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
706 "DNS returned expired record for `%s'\n",
708 return; /* record expired */
710 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
711 "DNS returned record that expires at %s for `%s'\n",
712 GNUNET_STRINGS_absolute_time_to_string (rec->expiration_time),
716 case GNUNET_DNSPARSER_TYPE_NS:
718 struct GlueClosure gc;
722 gc.ns = rec->data.hostname;
723 gc.found = GNUNET_NO;
724 for_all_records (prc->p,
727 if ( (GNUNET_NO == gc.found) &&
729 GNUNET_DNSPARSER_builder_add_name (dst,
734 GNUNET_DNSPARSER_builder_add_name (dst,
737 rec->data.hostname)) )
739 /* FIXME: actually check if this is out-of-bailiwick,
740 and if not request explicit resolution... */
741 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
742 "Converted OOB (`%s') NS record for `%s'\n",
746 GNUNET_GNSRECORD_TYPE_GNS2DNS,
747 rec->expiration_time,
753 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
754 "Converted NS record for `%s' using glue\n",
759 case GNUNET_DNSPARSER_TYPE_CNAME:
761 GNUNET_DNSPARSER_builder_add_name (dst,
766 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
767 "Converting CNAME (`%s') record for `%s'\n",
772 rec->expiration_time,
777 case GNUNET_DNSPARSER_TYPE_DNAME:
778 /* No support for DNAME in GNS yet! FIXME: support later! */
779 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
780 "FIXME: not supported: %s DNAME %s\n",
784 case GNUNET_DNSPARSER_TYPE_MX:
786 GNUNET_DNSPARSER_builder_add_mx (dst,
791 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
792 "Converting MX (`%s') record for `%s'\n",
793 rec->data.mx->mxhost,
797 rec->expiration_time,
802 case GNUNET_DNSPARSER_TYPE_SOA:
804 GNUNET_DNSPARSER_builder_add_soa (dst,
809 /* NOTE: GNS does not really use SOAs */
810 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
811 "Converting SOA record for `%s'\n",
815 rec->expiration_time,
820 case GNUNET_DNSPARSER_TYPE_SRV:
822 GNUNET_DNSPARSER_builder_add_srv (dst,
827 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
828 "Converting SRV record for `%s'\n",
832 rec->expiration_time,
837 case GNUNET_DNSPARSER_TYPE_PTR:
839 GNUNET_DNSPARSER_builder_add_name (dst,
844 /* !?: what does a PTR record do in a regular TLD??? */
845 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
846 "Converting PTR record for `%s' (weird)\n",
850 rec->expiration_time,
855 case GNUNET_DNSPARSER_TYPE_CERT:
857 GNUNET_DNSPARSER_builder_add_cert (dst,
862 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
863 "Converting CERT record for `%s'\n",
867 rec->expiration_time,
872 /* Rest is 'raw' encoded and just needs to be copied IF
873 the hostname matches the requested name; otherwise we
874 simply cannot use it. */
875 case GNUNET_DNSPARSER_TYPE_A:
876 case GNUNET_DNSPARSER_TYPE_AAAA:
877 case GNUNET_DNSPARSER_TYPE_TXT:
879 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
880 "Converting record of type %u for `%s'\n",
881 (unsigned int) rec->type,
885 rec->expiration_time,
887 rec->data.raw.data_len);
894 * Continuation called to notify client about result of the
897 * @param cls closure with our `struct Request`
898 * @param success #GNUNET_SYSERR on failure (including timeout/queue drop/failure to validate)
899 * #GNUNET_NO if content was already there or not found
900 * #GNUNET_YES (or other positive value) on success
901 * @param emsg NULL on success, otherwise an error message
904 store_completed_cb (void *cls,
908 static struct GNUNET_TIME_Absolute last;
909 static unsigned int pdot;
910 struct Request *req = cls;
914 if (GNUNET_SYSERR == success)
916 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
917 "Failed to store zone data for `%s': %s\n",
923 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
924 "Stored records under `%s'\n",
927 last = GNUNET_TIME_absolute_get ();
929 if (0 == pdot % 1000)
931 struct GNUNET_TIME_Relative delta;
933 delta = GNUNET_TIME_absolute_get_duration (last);
934 last = GNUNET_TIME_absolute_get ();
935 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
936 "Processed 1000 records in %s\n",
937 GNUNET_STRINGS_relative_time_to_string (delta,
946 * Function called with the result of a DNS resolution.
948 * @param cls closure with the `struct Request`
949 * @param dns dns response, never NULL
950 * @param dns_len number of bytes in @a dns
953 process_result (void *cls,
954 const struct GNUNET_TUN_DnsHeader *dns,
957 struct Request *req = cls;
959 struct GNUNET_DNSPARSER_Packet *p;
960 unsigned int rd_count;
962 GNUNET_assert (NULL == req->hn);
966 GNUNET_CONTAINER_DLL_remove (req_head,
970 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
971 "Stub gave up on DNS reply for `%s'\n",
973 if (req->issue_num > MAX_RETRIES)
983 if (req->id != dns->id)
985 GNUNET_CONTAINER_DLL_remove (req_head,
988 GNUNET_DNSSTUB_resolve_cancel (req->rs);
990 p = GNUNET_DNSPARSER_parse ((const char *) dns,
994 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
995 "Failed to parse DNS reply for `%s'\n",
997 if (req->issue_num > MAX_RETRIES)
1000 insert_sorted (req);
1004 insert_sorted (req);
1008 /* import new records */
1009 req->issue_num = 0; /* success, reset counter! */
1011 struct ProcessRecordContext prc = {
1020 GNUNET_DNSPARSER_free_packet (p);
1021 /* count records found, determine minimum expiration time */
1022 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1024 for (rec = req->rec_head; NULL != rec; rec = rec->next)
1026 struct GNUNET_TIME_Absolute at;
1028 at.abs_value_us = rec->grd.expiration_time;
1029 req->expires = GNUNET_TIME_absolute_min (req->expires,
1033 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1034 "Obtained %u records for `%s'\n",
1037 /* Instead of going for SOA, simplified for now to look each
1038 day in case we got an empty response */
1041 = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_DAYS);
1042 /* convert records to namestore import format */
1044 struct GNUNET_GNSRECORD_Data rd[GNUNET_NZL(rd_count)];
1045 unsigned int off = 0;
1047 /* convert linked list into array */
1048 for (rec = req->rec_head; NULL != rec; rec =rec->next)
1049 rd[off++] = rec->grd;
1050 req->qe = GNUNET_NAMESTORE_records_store (ns,
1055 &store_completed_cb,
1058 insert_sorted (req);
1063 * Submit a request to DNS unless we need to slow down because
1064 * we are at the rate limit.
1066 * @param req request to submit
1067 * @return #GNUNET_OK if request was submitted
1068 * #GNUNET_NO if request was already submitted
1069 * #GNUNET_SYSERR if we are at the rate limit
1072 submit_req (struct Request *req)
1074 static struct GNUNET_TIME_Absolute last_request;
1075 struct GNUNET_TIME_Absolute now;
1079 if (NULL != req->qe)
1080 return GNUNET_NO; /* namestore op still pending */
1081 if (NULL != req->rs)
1084 return GNUNET_NO; /* already submitted */
1086 now = GNUNET_TIME_absolute_get ();
1087 if ( (now.abs_value_us - last_request.abs_value_us < TIME_THRESH) ||
1088 (pending >= THRESH) )
1089 return GNUNET_SYSERR;
1090 GNUNET_CONTAINER_DLL_insert (req_head,
1093 GNUNET_assert (NULL == req->rs);
1094 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1095 "Requesting resolution for `%s'\n",
1097 raw = build_dns_query (req,
1103 return GNUNET_SYSERR;
1105 req->rs = GNUNET_DNSSTUB_resolve (ctx,
1110 GNUNET_assert (NULL != req->rs);
1120 * Process as many requests as possible from the queue.
1125 process_queue (void *cls)
1127 struct Request *req;
1133 req = GNUNET_CONTAINER_heap_peek (req_heap);
1136 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1138 if (GNUNET_OK != submit_req (req))
1140 GNUNET_assert (req ==
1141 GNUNET_CONTAINER_heap_remove_root (req_heap));
1145 req = GNUNET_CONTAINER_heap_peek (req_heap);
1148 if (GNUNET_TIME_absolute_get_remaining (req->expires).rel_value_us > 0)
1150 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1151 "Waiting until %s for next record (`%s') to expire\n",
1152 GNUNET_STRINGS_absolute_time_to_string (req->expires),
1155 GNUNET_SCHEDULER_cancel (t);
1156 t = GNUNET_SCHEDULER_add_at (req->expires,
1162 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1163 "Throttling for 1ms\n");
1165 GNUNET_SCHEDULER_cancel (t);
1166 t = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MILLISECONDS,
1174 * Iterator called during #do_shutdown() to free requests in
1175 * the #ns_pending map.
1179 * @param value the `struct Request` to free
1180 * @return #GNUNET_OK
1183 free_request_it (void *cls,
1184 const struct GNUNET_HashCode *key,
1187 struct Request *req = value;
1197 * Clean up and terminate the process.
1202 do_shutdown (void *cls)
1204 struct Request *req;
1210 GNUNET_IDENTITY_disconnect (id);
1215 GNUNET_SCHEDULER_cancel (t);
1218 while (NULL != (req = req_head))
1220 GNUNET_CONTAINER_DLL_remove (req_head,
1223 if (NULL != req->qe)
1224 GNUNET_NAMESTORE_cancel (req->qe);
1227 while (NULL != (req = GNUNET_CONTAINER_heap_remove_root (req_heap)))
1230 if (NULL != req->qe)
1231 GNUNET_NAMESTORE_cancel (req->qe);
1234 if (NULL != zone_it)
1236 GNUNET_NAMESTORE_zone_iteration_stop (zone_it);
1241 GNUNET_NAMESTORE_disconnect (ns);
1246 GNUNET_DNSSTUB_stop (ctx);
1249 if (NULL != req_heap)
1251 GNUNET_CONTAINER_heap_destroy (req_heap);
1254 if (NULL != ns_pending)
1256 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1259 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1262 while (NULL != (zone = zone_head))
1264 GNUNET_CONTAINER_DLL_remove (zone_head,
1267 GNUNET_free (zone->domain);
1274 * Function called if #GNUNET_NAMESTORE_records_lookup() failed.
1275 * Just logs an error.
1277 * @param cls a `struct Zone`
1280 ns_lookup_error_cb (void *cls)
1282 struct Zone *zone = cls;
1284 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1285 "Failed to load data from namestore for zone `%s'\n",
1291 * Process a record that was stored in the namestore.
1293 * @param cls a `struct Zone *`
1294 * @param key private key of the zone
1295 * @param label label of the records
1296 * @param rd_count number of entries in @a rd array, 0 if label was deleted
1297 * @param rd array of records with data to store
1300 ns_lookup_result_cb (void *cls,
1301 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
1303 unsigned int rd_count,
1304 const struct GNUNET_GNSRECORD_Data *rd)
1306 struct Zone *zone = cls;
1307 struct Request *req;
1308 struct GNUNET_HashCode hc;
1311 ns_iterator_trigger_next--;
1312 if (0 == ns_iterator_trigger_next)
1314 ns_iterator_trigger_next = NS_BATCH_SIZE;
1315 GNUNET_NAMESTORE_zone_iterator_next (zone_it,
1316 ns_iterator_trigger_next);
1318 GNUNET_asprintf (&fqdn,
1322 GNUNET_CRYPTO_hash (fqdn,
1326 req = GNUNET_CONTAINER_multihashmap_get (ns_pending,
1330 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1331 "Ignoring record `%s' in zone `%s': not on my list!\n",
1336 GNUNET_assert (GNUNET_OK ==
1337 GNUNET_CONTAINER_multihashmap_remove (ns_pending,
1340 GNUNET_break (0 == memcmp (key,
1343 GNUNET_break (0 == strcasecmp (label,
1345 for (unsigned int i=0;i<rd_count;i++)
1347 struct GNUNET_TIME_Absolute at;
1349 at.abs_value_us = rd->expiration_time;
1358 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1359 "Empty record set in namestore for `%s'\n",
1364 unsigned int pos = 0;
1366 req->expires = GNUNET_TIME_UNIT_FOREVER_ABS;
1367 for (struct Record *rec = req->rec_head;
1371 struct GNUNET_TIME_Absolute at;
1373 at.abs_value_us = rec->grd.expiration_time;
1374 req->expires = GNUNET_TIME_absolute_min (req->expires,
1379 req->expires = GNUNET_TIME_UNIT_ZERO_ABS;
1380 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1381 "Hot-start with %u existing records for `%s'\n",
1387 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1388 "Adding `%s' to worklist to start at %s\n",
1390 GNUNET_STRINGS_absolute_time_to_string (req->expires));
1391 insert_sorted (req);
1396 * Add @a hostname to the list of requests to be made.
1398 * @param hostname name to resolve
1401 queue (const char *hostname)
1403 struct Request *req;
1407 struct GNUNET_HashCode hc;
1410 GNUNET_DNSPARSER_check_name (hostname))
1412 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1413 "Refusing invalid hostname `%s'\n",
1418 dot = strchr (hostname,
1419 (unsigned char) '.');
1422 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1423 "Refusing invalid hostname `%s' (lacks '.')\n",
1428 for (zone = zone_head; NULL != zone; zone = zone->next)
1429 if (0 == strcmp (zone->domain,
1435 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1436 "Domain name `%s' not in ego list!\n",
1442 hlen = strlen (hostname) + 1;
1443 req = GNUNET_malloc (sizeof (struct Request) + hlen);
1445 req->hostname = (char *) &req[1];
1446 memcpy (req->hostname,
1449 req->id = (uint16_t) GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_NONCE,
1451 GNUNET_CRYPTO_hash (req->hostname,
1455 GNUNET_CONTAINER_multihashmap_put (ns_pending,
1458 GNUNET_CONTAINER_MULTIHASHMAPOPTION_UNIQUE_ONLY))
1460 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1461 "Duplicate hostname `%s' ignored\n",
1470 * We have completed the initial iteration over the namestore's database.
1471 * This function is called on each of the remaining records in
1472 * #move_to_queue to #queue() them, as we will simply not find existing
1473 * records for them any longer.
1477 * @param value a `struct Request`
1478 * @return #GNUNET_OK (continue to iterate)
1481 move_to_queue (void *cls,
1482 const struct GNUNET_HashCode *key,
1485 struct Request *req = value;
1489 insert_sorted (req);
1495 * Iterate over all of the zones we care about and see which records
1496 * we may need to re-fetch when.
1501 iterate_zones (void *cls)
1503 static struct Zone *last;
1507 GNUNET_assert (NULL != zone_tail);
1508 if (zone_tail == last)
1510 GNUNET_assert (NULL == t);
1511 /* Done iterating over relevant zones in NAMESTORE, move
1512 rest of hash map to work queue as well. */
1513 GNUNET_CONTAINER_multihashmap_iterate (ns_pending,
1516 GNUNET_CONTAINER_multihashmap_destroy (ns_pending);
1518 GNUNET_assert (NULL == t);
1519 t = GNUNET_SCHEDULER_add_now (&process_queue,
1527 ns_iterator_trigger_next = 1;
1528 zone_it = GNUNET_NAMESTORE_zone_iteration_start (ns,
1530 &ns_lookup_error_cb,
1532 &ns_lookup_result_cb,
1541 * Begin processing hostnames from stdin.
1546 process_stdin (void *cls)
1548 static struct GNUNET_TIME_Absolute last;
1549 static unsigned int pdot;
1556 GNUNET_IDENTITY_disconnect (id);
1565 hn[strlen(hn)-1] = '\0'; /* eat newline */
1567 last = GNUNET_TIME_absolute_get ();
1569 if (0 == pdot % 1000)
1571 struct GNUNET_TIME_Relative delta;
1573 delta = GNUNET_TIME_absolute_get_duration (last);
1574 last = GNUNET_TIME_absolute_get ();
1575 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1576 "Imported 1000 records in %s\n",
1577 GNUNET_STRINGS_relative_time_to_string (delta,
1582 fprintf (stderr, "\n");
1583 iterate_zones (NULL);
1588 * Method called to inform about the egos of this peer.
1590 * When used with #GNUNET_IDENTITY_connect, this function is
1591 * initially called for all egos and then again whenever a
1592 * ego's name changes or if it is deleted. At the end of
1593 * the initial pass over all egos, the function is once called
1594 * with 'NULL' for @a ego. That does NOT mean that the callback won't
1595 * be invoked in the future or that there was an error.
1597 * When used with #GNUNET_IDENTITY_create or #GNUNET_IDENTITY_get,
1598 * this function is only called ONCE, and 'NULL' being passed in
1599 * @a ego does indicate an error (i.e. name is taken or no default
1600 * value is known). If @a ego is non-NULL and if '*ctx'
1601 * is set in those callbacks, the value WILL be passed to a subsequent
1602 * call to the identity callback of #GNUNET_IDENTITY_connect (if
1603 * that one was not NULL).
1605 * When an identity is renamed, this function is called with the
1606 * (known) @a ego but the NEW @a name.
1608 * When an identity is deleted, this function is called with the
1609 * (known) ego and "NULL" for the @a name. In this case,
1610 * the @a ego is henceforth invalid (and the @a ctx should also be
1613 * @param cls closure
1614 * @param ego ego handle
1615 * @param ctx context for application to store data for this ego
1616 * (during the lifetime of this process, initially NULL)
1617 * @param name name assigned by the user for this ego,
1618 * NULL if the user just deleted the ego and it
1619 * must thus no longer be used
1622 identity_cb (void *cls,
1623 struct GNUNET_IDENTITY_Ego *ego,
1631 if (NULL != zone_head)
1633 t = GNUNET_SCHEDULER_add_now (&process_stdin,
1638 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1640 GNUNET_SCHEDULER_shutdown ();
1648 zone = GNUNET_new (struct Zone);
1649 zone->key = *GNUNET_IDENTITY_ego_get_private_key (ego);
1650 zone->domain = GNUNET_strdup (name);
1651 GNUNET_CONTAINER_DLL_insert (zone_head,
1659 * Process requests from the queue, then if the queue is
1660 * not empty, try again.
1663 * @param args remaining command-line arguments
1664 * @param cfgfile name of the configuration file used (for saving, can be NULL!)
1665 * @param cfg configuration
1670 const char *cfgfile,
1671 const struct GNUNET_CONFIGURATION_Handle *cfg)
1676 req_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
1677 ns_pending = GNUNET_CONTAINER_multihashmap_create (1024,
1679 ctx = GNUNET_DNSSTUB_start (256);
1683 "Failed to initialize GNUnet DNS STUB\n");
1686 if (NULL == args[0])
1689 "You must provide a list of DNS resolvers on the command line\n");
1692 for (unsigned int i=0;NULL != args[i];i++)
1695 GNUNET_DNSSTUB_add_dns_ip (ctx,
1699 "Failed to use `%s' for DNS resolver\n",
1706 GNUNET_SCHEDULER_add_shutdown (&do_shutdown,
1708 ns = GNUNET_NAMESTORE_connect (cfg);
1711 GNUNET_SCHEDULER_shutdown ();
1714 id = GNUNET_IDENTITY_connect (cfg,
1721 * Call with IP address of resolver to query.
1723 * @param argc should be 2
1724 * @param argv[1] should contain IP address
1725 * @return 0 on success
1731 struct GNUNET_GETOPT_CommandLineOption options[] = {
1732 GNUNET_GETOPT_OPTION_END
1736 GNUNET_STRINGS_get_utf8_args (argc, argv,
1739 GNUNET_PROGRAM_run (argc,
1741 "gnunet-zoneimport",
1742 "import DNS zone into namestore",
1746 GNUNET_free ((void*) argv);
1748 "Rejected %u names, did %u lookups, found %u records, %u lookups failed, %u pending on shutdown\n",
1757 /* end of gnunet-zoneimport.c */