merge
[oweals/gnunet.git] / src / zonemaster / gnunet-service-zonemaster.c
1 /*
2      This file is part of GNUnet.
3      Copyright (C) 2012, 2013, 2014, 2017 GNUnet e.V.
4
5      GNUnet is free software; you can redistribute it and/or modify
6      it under the terms of the GNU General Public License as published
7      by the Free Software Foundation; either version 3, or (at your
8      option) any later version.
9
10      GNUnet is distributed in the hope that it will be useful, but
11      WITHOUT ANY WARRANTY; without even the implied warranty of
12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
13      General Public License for more details.
14
15      You should have received a copy of the GNU General Public License
16      along with GNUnet; see the file COPYING.  If not, write to the
17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18      Boston, MA 02110-1301, USA.
19 */
20
21 /**
22  * @file zonemaster/gnunet-service-zonemaster.c
23  * @brief publish records from namestore to GNUnet name system
24  * @author Christian Grothoff
25  */
26 #include "platform.h"
27 #include "gnunet_util_lib.h"
28 #include "gnunet_dnsparser_lib.h"
29 #include "gnunet_dht_service.h"
30 #include "gnunet_namestore_service.h"
31 #include "gnunet_statistics_service.h"
32 #include "gnunet_namestore_plugin.h"
33 #include "gnunet_signatures.h"
34
35
36 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
37
38
39 /**
40  * How often should we (re)publish each record before
41  * it expires?
42  */
43 #define PUBLISH_OPS_PER_EXPIRATION 4
44
45 /**
46  * How often do we measure the delta between desired zone
47  * iteration speed and actual speed, and tell statistics
48  * service about it?
49  */
50 #define DELTA_INTERVAL 100
51
52 /**
53  * How many records do we fetch in one shot from the namestore?
54  */
55 #define NS_BLOCK_SIZE 1000
56
57 /**
58  * How many pending DHT operations do we allow at most?
59  */
60 #define DHT_QUEUE_LIMIT 2000
61
62 /**
63  * The initial interval in milliseconds btween puts in
64  * a zone iteration
65  */
66 #define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
67
68 /**
69  * The upper bound for the zone iteration interval
70  * (per record).
71  */
72 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
73
74 /**
75  * The default put interval for the zone iteration. In case
76  * no option is found
77  */
78 #define DEFAULT_ZONE_PUBLISH_TIME_WINDOW GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
79
80 /**
81  * The factor the current zone iteration interval is divided by for each
82  * additional new record
83  */
84 #define LATE_ITERATION_SPEEDUP_FACTOR 2
85
86 /**
87  * How long until a DHT PUT attempt should time out?
88  */
89 #define DHT_OPERATION_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
90
91 /**
92  * What replication level do we use for DHT PUT operations?
93  */
94 #define DHT_GNS_REPLICATION_LEVEL 5
95
96
97 /**
98  * Handle for DHT PUT activity triggered from the namestore monitor.
99  */
100 struct DhtPutActivity
101 {
102   /**
103    * Kept in a DLL.
104    */
105   struct DhtPutActivity *next;
106
107   /**
108    * Kept in a DLL.
109    */
110   struct DhtPutActivity *prev;
111
112   /**
113    * Handle for the DHT PUT operation.
114    */
115   struct GNUNET_DHT_PutHandle *ph;
116
117   /**
118    * When was this PUT initiated?
119    */
120   struct GNUNET_TIME_Absolute start_date;
121 };
122
123
124 /**
125  * Handle to the statistics service
126  */
127 static struct GNUNET_STATISTICS_Handle *statistics;
128
129 /**
130  * Our handle to the DHT
131  */
132 static struct GNUNET_DHT_Handle *dht_handle;
133
134 /**
135  * Active DHT put operation (or NULL)
136  */
137 static struct GNUNET_DHT_PutHandle *active_put;
138
139 /**
140  * Our handle to the namestore service
141  */
142 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
143
144 /**
145  * Handle to iterate over our authoritative zone in namestore
146  */
147 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
148
149 /**
150  * Handle to monitor namestore changes to instant propagation.
151  */
152 static struct GNUNET_NAMESTORE_ZoneMonitor *zmon;
153
154 /**
155  * Head of monitor activities; kept in a DLL.
156  */
157 static struct DhtPutActivity *ma_head;
158
159 /**
160  * Tail of monitor activities; kept in a DLL.
161  */
162 static struct DhtPutActivity *ma_tail;
163
164 /**
165  * Head of iteration put activities; kept in a DLL.
166  */
167 static struct DhtPutActivity *it_head;
168
169 /**
170  * Tail of iteration put activities; kept in a DLL.
171  */
172 static struct DhtPutActivity *it_tail;
173
174 /**
175  * Number of entries in the DHT queue #it_head.
176  */
177 static unsigned int dht_queue_length;
178
179 /**
180  * Number of entries in the DHT queue #ma_head.
181  */
182 static unsigned int ma_queue_length;
183
184 /**
185  * Useful for zone update for DHT put
186  */
187 static unsigned long long num_public_records;
188
189 /**
190  * Last seen record count
191  */
192 static unsigned long long last_num_public_records;
193
194 /**
195  * Number of successful put operations performed in the current
196  * measurement cycle (as measured in #check_zone_namestore_next()).
197  */
198 static unsigned long long put_cnt;
199
200 /**
201  * What is the frequency at which we currently would like
202  * to perform DHT puts (per record)?  Calculated in
203  * update_velocity() from the #zone_publish_time_window()
204  * and the total number of record sets we have (so far)
205  * observed in the zone.
206  */
207 static struct GNUNET_TIME_Relative next_put_interval;
208
209 /**
210  * Minimum relative expiration time of records seem during the current
211  * zone iteration.
212  */
213 static struct GNUNET_TIME_Relative min_relative_record_time;
214
215 /**
216  * Minimum relative expiration time of records seem during the last
217  * zone iteration.
218  */
219 static struct GNUNET_TIME_Relative last_min_relative_record_time;
220
221 /**
222  * Default time window for zone iteration
223  */
224 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
225
226 /**
227  * Time window for zone iteration, adjusted based on relative record
228  * expiration times in our zone.
229  */
230 static struct GNUNET_TIME_Relative zone_publish_time_window;
231
232 /**
233  * When did we last start measuring the #DELTA_INTERVAL successful
234  * DHT puts? Used for velocity calculations.
235  */
236 static struct GNUNET_TIME_Absolute last_put_100;
237
238 /**
239  * By how much should we try to increase our per-record iteration speed
240  * (over the desired speed calculated directly from the #put_interval)?
241  * Basically this value corresponds to the per-record CPU time overhead
242  * we have.
243  */
244 static struct GNUNET_TIME_Relative sub_delta;
245
246 /**
247  * zone publish task
248  */
249 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
250
251 /**
252  * How many more values are left for the current query before we need
253  * to explicitly ask the namestore for more?
254  */
255 static unsigned int ns_iteration_left;
256
257 /**
258  * #GNUNET_YES if zone has never been published before
259  */
260 static int first_zone_iteration;
261
262 /**
263  * Optimize block insertion by caching map of private keys to
264  * public keys in memory?
265  */
266 static int cache_keys;
267
268
269 /**
270  * Task run during shutdown.
271  *
272  * @param cls unused
273  * @param tc unused
274  */
275 static void
276 shutdown_task (void *cls)
277 {
278   struct DhtPutActivity *ma;
279
280   (void) cls;
281   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
282               "Shutting down!\n");
283   while (NULL != (ma = ma_head))
284   {
285     GNUNET_DHT_put_cancel (ma->ph);
286     GNUNET_CONTAINER_DLL_remove (ma_head,
287                                  ma_tail,
288                                  ma);
289     GNUNET_free (ma);
290   }
291   while (NULL != (ma = it_head))
292   {
293     GNUNET_DHT_put_cancel (ma->ph);
294     GNUNET_CONTAINER_DLL_remove (it_head,
295                                  it_tail,
296                                  ma);
297     dht_queue_length--;
298     GNUNET_free (ma);
299   }
300   if (NULL != statistics)
301   {
302     GNUNET_STATISTICS_destroy (statistics,
303                                GNUNET_NO);
304     statistics = NULL;
305   }
306   if (NULL != zone_publish_task)
307   {
308     GNUNET_SCHEDULER_cancel (zone_publish_task);
309     zone_publish_task = NULL;
310   }
311   if (NULL != namestore_iter)
312   {
313     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
314     namestore_iter = NULL;
315   }
316   if (NULL != zmon)
317   {
318     GNUNET_NAMESTORE_zone_monitor_stop (zmon);
319     zmon = NULL;
320   }
321   if (NULL != namestore_handle)
322   {
323     GNUNET_NAMESTORE_disconnect (namestore_handle);
324     namestore_handle = NULL;
325   }
326   if (NULL != active_put)
327   {
328     GNUNET_DHT_put_cancel (active_put);
329     active_put = NULL;
330   }
331   if (NULL != dht_handle)
332   {
333     GNUNET_DHT_disconnect (dht_handle);
334     dht_handle = NULL;
335   }
336 }
337
338
339 /**
340  * Method called periodically that triggers iteration over authoritative records
341  *
342  * @param cls NULL
343  */
344 static void
345 publish_zone_namestore_next (void *cls)
346 {
347   (void) cls;
348   zone_publish_task = NULL;
349   GNUNET_assert (NULL != namestore_iter);
350   GNUNET_assert (0 == ns_iteration_left);
351   ns_iteration_left = NS_BLOCK_SIZE;
352   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
353                                        NS_BLOCK_SIZE);
354 }
355
356
357 /**
358  * Periodically iterate over our zone and store everything in dht
359  *
360  * @param cls NULL
361  */
362 static void
363 publish_zone_dht_start (void *cls);
364
365
366 /**
367  * Continuation called from DHT once the PUT operation triggered
368  * by a monitor is done.
369  *
370  * @param cls a `struct DhtPutActivity`
371  */
372 static void
373 dht_put_monitor_continuation (void *cls)
374 {
375   struct DhtPutActivity *ma = cls;
376
377   ma_queue_length--;
378   GNUNET_CONTAINER_DLL_remove (ma_head,
379                                ma_tail,
380                                ma);
381   GNUNET_free (ma);
382 }
383
384
385 /**
386  * Calculate #next_put_interval.
387  */
388 static void
389 calculate_put_interval ()
390 {
391   if (0 == num_public_records)
392   {
393     /**
394      * If no records are known (startup) or none present
395      * we can safely set the interval to the value for a single
396      * record
397      */
398     next_put_interval = zone_publish_time_window;
399     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
400                 "No records in namestore database.\n");
401   }
402   else
403   {
404     last_min_relative_record_time
405       = GNUNET_TIME_relative_min (last_min_relative_record_time,
406                                   min_relative_record_time);
407     zone_publish_time_window
408       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
409                                                                PUBLISH_OPS_PER_EXPIRATION),
410                                   zone_publish_time_window_default);
411     next_put_interval
412       = GNUNET_TIME_relative_divide (zone_publish_time_window,
413                                      last_num_public_records);
414   }
415   next_put_interval
416     = GNUNET_TIME_relative_min (next_put_interval,
417                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
418   GNUNET_STATISTICS_set (statistics,
419                          "Minimum relative record expiration (in ms)",
420                          last_min_relative_record_time.rel_value_us / 1000LL,
421                          GNUNET_NO); 
422   GNUNET_STATISTICS_set (statistics,
423                          "Zone publication time window (in ms)",
424                          zone_publish_time_window.rel_value_us / 1000LL,
425                          GNUNET_NO);
426   GNUNET_STATISTICS_set (statistics,
427                          "Target zone iteration velocity (μs)",
428                          next_put_interval.rel_value_us,
429                          GNUNET_NO);
430 }
431
432
433 /**
434  * Re-calculate our velocity and the desired velocity.
435  * We have succeeded in making #DELTA_INTERVAL puts, so
436  * now calculate the new desired delay between puts.
437  *
438  * @param cnt how many records were processed since the last call?
439  */
440 static void
441 update_velocity (unsigned int cnt)
442 {
443   struct GNUNET_TIME_Relative delta;
444   unsigned long long pct = 0;
445
446   if (0 == cnt)
447     return;
448   /* How fast were we really? */
449   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
450   delta.rel_value_us /= cnt;
451   last_put_100 = GNUNET_TIME_absolute_get ();
452
453   /* calculate expected frequency */
454   if ( (num_public_records > last_num_public_records) &&
455        (GNUNET_NO == first_zone_iteration) )
456   {
457     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
458                 "Last record count was lower than current record count.  Reducing interval.\n");
459     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
460     calculate_put_interval ();
461   }
462   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
463               "Desired global zone iteration interval is %s/record!\n",
464               GNUNET_STRINGS_relative_time_to_string (next_put_interval,
465                                                       GNUNET_YES));
466
467   /* Tell statistics actual vs. desired speed */
468   GNUNET_STATISTICS_set (statistics,
469                          "Current zone iteration velocity (μs/record)",
470                          delta.rel_value_us,
471                          GNUNET_NO);
472   /* update "sub_delta" based on difference, taking
473      previous sub_delta into account! */
474   if (next_put_interval.rel_value_us > delta.rel_value_us)
475   {
476     /* We were too fast, reduce sub_delta! */
477     struct GNUNET_TIME_Relative corr;
478
479     corr = GNUNET_TIME_relative_subtract (next_put_interval,
480                                           delta);
481     if (sub_delta.rel_value_us > delta.rel_value_us)
482     {
483       /* Reduce sub_delta by corr */
484       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
485                                                  corr);
486     }
487     else
488     {
489       /* We're doing fine with waiting the full time, this
490          should theoretically only happen if we run at
491          infinite speed. */
492       sub_delta = GNUNET_TIME_UNIT_ZERO;
493     }
494   }
495   else if (next_put_interval.rel_value_us < delta.rel_value_us)
496   {
497     /* We were too slow, increase sub_delta! */
498     struct GNUNET_TIME_Relative corr;
499
500     corr = GNUNET_TIME_relative_subtract (delta,
501                                           next_put_interval);
502     sub_delta = GNUNET_TIME_relative_add (sub_delta,
503                                           corr);
504     if (sub_delta.rel_value_us > next_put_interval.rel_value_us)
505     {
506       /* CPU overload detected, we cannot go at desired speed,
507          as this would mean using a negative delay. */
508       /* compute how much faster we would want to be for
509          the desired velocity */
510       if (0 == next_put_interval.rel_value_us)
511         pct = UINT64_MAX; /* desired speed is infinity ... */
512       else
513         pct = (sub_delta.rel_value_us -
514                next_put_interval.rel_value_us) * 100LLU
515           / next_put_interval.rel_value_us;
516       sub_delta = next_put_interval;
517     }
518   }
519   GNUNET_STATISTICS_set (statistics,
520                          "# size of the DHT queue (it)",
521                          dht_queue_length,
522                          GNUNET_NO);
523   GNUNET_STATISTICS_set (statistics,
524                          "# size of the DHT queue (mon)",
525                          ma_queue_length,
526                          GNUNET_NO);
527   GNUNET_STATISTICS_set (statistics,
528                          "% speed increase needed for target velocity",
529                          pct,
530                          GNUNET_NO);
531   GNUNET_STATISTICS_set (statistics,
532                          "# records processed in current iteration",
533                          num_public_records,
534                          GNUNET_NO);
535 }
536
537
538 /**
539  * Check if the current zone iteration needs to be continued
540  * by calling #publish_zone_namestore_next(), and if so with what delay.
541  */
542 static void
543 check_zone_namestore_next ()
544 {
545   struct GNUNET_TIME_Relative delay;
546
547   if (0 != ns_iteration_left)
548     return; /* current NAMESTORE iteration not yet done */
549   update_velocity (put_cnt);
550   put_cnt = 0;
551   delay = GNUNET_TIME_relative_subtract (next_put_interval,
552                                          sub_delta);
553   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
554      per-record delay calculated so far with the #NS_BLOCK_SIZE */
555   GNUNET_STATISTICS_set (statistics,
556                          "Current artificial NAMESTORE delay (μs/record)",
557                          delay.rel_value_us,
558                          GNUNET_NO);
559   delay = GNUNET_TIME_relative_multiply (delay,
560                                          NS_BLOCK_SIZE);
561   GNUNET_assert (NULL == zone_publish_task);
562   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
563                                                     &publish_zone_namestore_next,
564                                                     NULL);
565 }
566
567
568 /**
569  * Continuation called from DHT once the PUT operation is done.
570  *
571  * @param cls a `struct DhtPutActivity`
572  */
573 static void
574 dht_put_continuation (void *cls)
575 {
576   struct DhtPutActivity *ma = cls;
577
578   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
579               "PUT complete\n");
580   dht_queue_length--;
581   GNUNET_CONTAINER_DLL_remove (it_head,
582                                it_tail,
583                                ma);
584   GNUNET_free (ma);
585 }
586
587
588 /**
589  * Convert namestore records from the internal format to that
590  * suitable for publication (removes private records, converts
591  * to absolute expiration time).
592  *
593  * @param rd input records
594  * @param rd_count size of the @a rd and @a rd_public arrays
595  * @param rd_public where to write the converted records
596  * @return number of records written to @a rd_public
597  */
598 static unsigned int
599 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
600                             unsigned int rd_count,
601                             struct GNUNET_GNSRECORD_Data *rd_public)
602 {
603   struct GNUNET_TIME_Absolute now;
604   unsigned int rd_public_count;
605
606   rd_public_count = 0;
607   now = GNUNET_TIME_absolute_get ();
608   for (unsigned int i=0;i<rd_count;i++)
609     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
610     {
611       rd_public[rd_public_count] = rd[i];
612       if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
613       {
614         /* GNUNET_GNSRECORD_block_create will convert to absolute time;
615            we just need to adjust our iteration frequency */
616         min_relative_record_time.rel_value_us =
617           GNUNET_MIN (rd_public[rd_public_count].expiration_time,
618                       min_relative_record_time.rel_value_us);
619       }
620       else if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
621       {
622         /* record already expired, skip it */
623         continue;
624       }
625       rd_public_count++;
626     }
627   return rd_public_count;
628 }
629
630
631 /**
632  * Store GNS records in the DHT.
633  *
634  * @param key key of the zone
635  * @param label label to store under
636  * @param rd_public public record data
637  * @param rd_public_count number of records in @a rd_public
638  * @param cont function to call with PUT result
639  * @param cont_cls closure for @a cont
640  * @return DHT PUT handle, NULL on error
641  */
642 static struct GNUNET_DHT_PutHandle *
643 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
644                  const char *label,
645                  const struct GNUNET_GNSRECORD_Data *rd_public,
646                  unsigned int rd_public_count,
647                  GNUNET_SCHEDULER_TaskCallback cont,
648                  void *cont_cls)
649 {
650   struct GNUNET_GNSRECORD_Block *block;
651   struct GNUNET_HashCode query;
652   struct GNUNET_TIME_Absolute expire;
653   size_t block_size;
654   struct GNUNET_DHT_PutHandle *ret;
655
656   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
657                                                         rd_public);
658   if (cache_keys)
659     block = GNUNET_GNSRECORD_block_create2 (key,
660                                             expire,
661                                             label,
662                                             rd_public,
663                                             rd_public_count);
664   else
665     block = GNUNET_GNSRECORD_block_create (key,
666                                            expire,
667                                            label,
668                                            rd_public,
669                                            rd_public_count);
670   if (NULL == block)
671   {
672     GNUNET_break (0);
673     return NULL; /* whoops */
674   }
675   block_size = ntohl (block->purpose.size)
676     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
677     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
678   GNUNET_GNSRECORD_query_from_private_key (key,
679                                            label,
680                                            &query);
681   GNUNET_STATISTICS_update (statistics,
682                             "DHT put operations initiated",
683                             1,
684                             GNUNET_NO);
685   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
686               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
687               rd_public_count,
688               label,
689               GNUNET_STRINGS_absolute_time_to_string (expire),
690               GNUNET_h2s (&query));
691   num_public_records++;
692   ret = GNUNET_DHT_put (dht_handle,
693                         &query,
694                         DHT_GNS_REPLICATION_LEVEL,
695                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
696                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
697                         block_size,
698                         block,
699                         expire,
700                         cont,
701                         cont_cls);
702   GNUNET_free (block);
703   return ret;
704 }
705
706
707 /**
708  * We encountered an error in our zone iteration.
709  *
710  * @param cls NULL
711  */
712 static void
713 zone_iteration_error (void *cls)
714 {
715   (void) cls;
716   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
717               "Got disconnected from namestore database, retrying.\n");
718   namestore_iter = NULL;
719   /* We end up here on error/disconnect/shutdown, so potentially
720      while a zone publish task or a DHT put is still running; hence
721      we need to cancel those. */
722   if (NULL != zone_publish_task)
723   {
724     GNUNET_SCHEDULER_cancel (zone_publish_task);
725     zone_publish_task = NULL;
726   }
727   if (NULL != active_put)
728   {
729     GNUNET_DHT_put_cancel (active_put);
730     active_put = NULL;
731   }
732   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
733                                                 NULL);
734 }
735
736
737 /**
738  * Zone iteration is completed.
739  *
740  * @param cls NULL
741  */
742 static void
743 zone_iteration_finished (void *cls)
744 {
745   (void) cls;
746   /* we're done with one iteration, calculate when to do the next one */
747   namestore_iter = NULL;
748   last_num_public_records = num_public_records;
749   first_zone_iteration = GNUNET_NO;
750   last_min_relative_record_time = min_relative_record_time;
751   calculate_put_interval ();
752   /* reset for next iteration */
753   min_relative_record_time
754     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
755                                      PUBLISH_OPS_PER_EXPIRATION);
756   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
757               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
758               GNUNET_STRINGS_relative_time_to_string (next_put_interval,
759                                                       GNUNET_YES));
760   GNUNET_STATISTICS_set (statistics,
761                          "Current zone iteration interval (in ms)",
762                          next_put_interval.rel_value_us / 1000LL,
763                          GNUNET_NO);
764   GNUNET_STATISTICS_set (statistics,
765                          "Number of public records in DHT",
766                          last_num_public_records,
767                          GNUNET_NO);
768   GNUNET_assert (NULL == zone_publish_task);
769   if (0 == last_num_public_records)
770     zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
771                                                       &publish_zone_dht_start,
772                                                       NULL);
773   else
774     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
775                                                   NULL);
776 }
777
778
779 /**
780  * Function used to put all records successively into the DHT.
781  *
782  * @param cls the closure (NULL)
783  * @param key the private key of the authority (ours)
784  * @param label the name of the records, NULL once the iteration is done
785  * @param rd_count the number of records in @a rd
786  * @param rd the record data
787  */
788 static void
789 put_gns_record (void *cls,
790                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
791                 const char *label,
792                 unsigned int rd_count,
793                 const struct GNUNET_GNSRECORD_Data *rd)
794 {
795   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
796   unsigned int rd_public_count;
797   struct DhtPutActivity *ma;
798
799   (void) cls;
800   ns_iteration_left--;
801   rd_public_count = convert_records_for_export (rd,
802                                                 rd_count,
803                                                 rd_public);
804   if (0 == rd_public_count)
805   {
806     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
807                 "Record set empty, moving to next record set\n");
808     check_zone_namestore_next ();
809     return;
810   }
811   /* We got a set of records to publish */
812   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
813               "Starting DHT PUT\n");
814   ma = GNUNET_new (struct DhtPutActivity);
815   ma->start_date = GNUNET_TIME_absolute_get ();
816   ma->ph = perform_dht_put (key,
817                             label,
818                             rd_public,
819                             rd_public_count,
820                             &dht_put_continuation,
821                             ma);
822   put_cnt++;
823   if (0 == put_cnt % DELTA_INTERVAL)
824     update_velocity (DELTA_INTERVAL);
825   check_zone_namestore_next ();
826   if (NULL == ma->ph)
827   {
828     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
829                 "Could not perform DHT PUT, is the DHT running?\n");
830     GNUNET_free (ma);
831     return;
832   }
833   dht_queue_length++;
834   GNUNET_CONTAINER_DLL_insert_tail (it_head,
835                                     it_tail,
836                                     ma);
837   if (dht_queue_length > DHT_QUEUE_LIMIT)
838   {
839     ma = it_head;
840     GNUNET_CONTAINER_DLL_remove (it_head,
841                                  it_tail,
842                                  ma);
843     GNUNET_DHT_put_cancel (ma->ph);
844     dht_queue_length--;
845     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
846                 "DHT PUT unconfirmed after %s, aborting PUT\n",
847                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
848                                                         GNUNET_YES));
849     GNUNET_free (ma);
850   }
851 }
852
853
854 /**
855  * Periodically iterate over all zones and store everything in DHT
856  *
857  * @param cls NULL
858  */
859 static void
860 publish_zone_dht_start (void *cls)
861 {
862   (void) cls;
863   zone_publish_task = NULL;
864   GNUNET_STATISTICS_update (statistics,
865                             "Full zone iterations launched",
866                             1,
867                             GNUNET_NO);
868   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
869               "Starting DHT zone update!\n");
870   /* start counting again */
871   num_public_records = 0;
872   GNUNET_assert (NULL == namestore_iter);
873   ns_iteration_left = 1;
874   namestore_iter
875     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
876                                              NULL, /* All zones */
877                                              &zone_iteration_error,
878                                              NULL,
879                                              &put_gns_record,
880                                              NULL,
881                                              &zone_iteration_finished,
882                                              NULL);
883   GNUNET_assert (NULL != namestore_iter);
884 }
885
886
887 /**
888  * Process a record that was stored in the namestore
889  * (invoked by the monitor).
890  *
891  * @param cls closure, NULL
892  * @param zone private key of the zone; NULL on disconnect
893  * @param label label of the records; NULL on disconnect
894  * @param rd_count number of entries in @a rd array, 0 if label was deleted
895  * @param rd array of records with data to store
896  */
897 static void
898 handle_monitor_event (void *cls,
899                       const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
900                       const char *label,
901                       unsigned int rd_count,
902                       const struct GNUNET_GNSRECORD_Data *rd)
903 {
904   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
905   unsigned int rd_public_count;
906   struct DhtPutActivity *ma;
907
908   (void) cls;
909   GNUNET_STATISTICS_update (statistics,
910                             "Namestore monitor events received",
911                             1,
912                             GNUNET_NO);
913   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
914               "Received %u records for label `%s' via namestore monitor\n",
915               rd_count,
916               label);
917   /* filter out records that are not public, and convert to
918      absolute expiration time. */
919   rd_public_count = convert_records_for_export (rd,
920                                                 rd_count,
921                                                 rd_public);
922   if (0 == rd_public_count)
923     return; /* nothing to do */
924   num_public_records++;
925   ma = GNUNET_new (struct DhtPutActivity);
926   ma->start_date = GNUNET_TIME_absolute_get ();
927   ma->ph = perform_dht_put (zone,
928                             label,
929                             rd,
930                             rd_count,
931                             &dht_put_monitor_continuation,
932                             ma);
933   if (NULL == ma->ph)
934   {
935     /* PUT failed, do not remember operation */
936     GNUNET_free (ma);
937     return;
938   }
939   GNUNET_CONTAINER_DLL_insert_tail (ma_head,
940                                     ma_tail,
941                                     ma);
942   ma_queue_length++;
943   if (ma_queue_length > DHT_QUEUE_LIMIT)
944   {
945     ma = it_head;
946     GNUNET_CONTAINER_DLL_remove (ma_head,
947                                  ma_tail,
948                                  ma);
949     GNUNET_DHT_put_cancel (ma->ph);
950     ma_queue_length--;
951     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
952                 "DHT PUT unconfirmed after %s, aborting PUT\n",
953                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
954                                                         GNUNET_YES));
955     GNUNET_free (ma);
956   }
957 }
958
959
960 /**
961  * The zone monitor is now in SYNC with the current state of the
962  * name store.  Start to perform periodic iterations.
963  *
964  * @param cls NULL
965  */
966 static void
967 monitor_sync_event (void *cls)
968 {
969   (void) cls;
970   if ( (NULL == zone_publish_task) &&
971        (NULL == namestore_iter) )
972     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
973                                                   NULL);
974 }
975
976
977 /**
978  * The zone monitor encountered an IPC error trying to to get in
979  * sync. Restart from the beginning.
980  *
981  * @param cls NULL
982  */
983 static void
984 handle_monitor_error (void *cls)
985 {
986   (void) cls;
987   GNUNET_STATISTICS_update (statistics,
988                             "Namestore monitor errors encountered",
989                             1,
990                             GNUNET_NO);
991   if (NULL != zone_publish_task)
992   {
993     GNUNET_SCHEDULER_cancel (zone_publish_task);
994     zone_publish_task = NULL;
995   }
996   if (NULL != namestore_iter)
997   {
998     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
999     namestore_iter = NULL;
1000   }
1001   if (NULL != active_put)
1002   {
1003     GNUNET_DHT_put_cancel (active_put);
1004     active_put = NULL;
1005   }
1006   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
1007                                                 NULL);
1008 }
1009
1010
1011 /**
1012  * Performe zonemaster duties: watch namestore, publish records.
1013  *
1014  * @param cls closure
1015  * @param server the initialized server
1016  * @param c configuration to use
1017  */
1018 static void
1019 run (void *cls,
1020      const struct GNUNET_CONFIGURATION_Handle *c,
1021      struct GNUNET_SERVICE_Handle *service)
1022 {
1023   unsigned long long max_parallel_bg_queries = 128;
1024
1025   (void) cls;
1026   (void) service;
1027   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
1028   min_relative_record_time
1029     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
1030                                      PUBLISH_OPS_PER_EXPIRATION);
1031   next_put_interval = INITIAL_PUT_INTERVAL;
1032   namestore_handle = GNUNET_NAMESTORE_connect (c);
1033   if (NULL == namestore_handle)
1034   {
1035     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1036                 _("Failed to connect to the namestore!\n"));
1037     GNUNET_SCHEDULER_shutdown ();
1038     return;
1039   }
1040   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
1041                                                      "namestore",
1042                                                      "CACHE_KEYS");
1043   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
1044   if (GNUNET_OK ==
1045       GNUNET_CONFIGURATION_get_value_time (c,
1046                                            "zonemaster",
1047                                            "ZONE_PUBLISH_TIME_WINDOW",
1048                                            &zone_publish_time_window_default))
1049   {
1050     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1051                 "Time window for zone iteration: %s\n",
1052                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
1053                                                         GNUNET_YES));
1054   }
1055   zone_publish_time_window = zone_publish_time_window_default;
1056   if (GNUNET_OK ==
1057       GNUNET_CONFIGURATION_get_value_number (c,
1058                                              "zonemaster",
1059                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
1060                                              &max_parallel_bg_queries))
1061   {
1062     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1063                 "Number of allowed parallel background queries: %llu\n",
1064                 max_parallel_bg_queries);
1065   }
1066   if (0 == max_parallel_bg_queries)
1067     max_parallel_bg_queries = 1;
1068   dht_handle = GNUNET_DHT_connect (c,
1069                                    (unsigned int) max_parallel_bg_queries);
1070   if (NULL == dht_handle)
1071   {
1072     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1073                 _("Could not connect to DHT!\n"));
1074     GNUNET_SCHEDULER_add_now (&shutdown_task,
1075                               NULL);
1076     return;
1077   }
1078
1079   /* Schedule periodic put for our records. */
1080   first_zone_iteration = GNUNET_YES;\
1081   statistics = GNUNET_STATISTICS_create ("zonemaster",
1082                                          c);
1083   GNUNET_STATISTICS_set (statistics,
1084                          "Target zone iteration velocity (μs)",
1085                          next_put_interval.rel_value_us,
1086                          GNUNET_NO);
1087   zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
1088                                               NULL,
1089                                               GNUNET_NO,
1090                                               &handle_monitor_error,
1091                                               NULL,
1092                                               &handle_monitor_event,
1093                                               NULL,
1094                                               &monitor_sync_event,
1095                                               NULL);
1096   GNUNET_break (NULL != zmon);
1097   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
1098                                  NULL);
1099 }
1100
1101
1102 /**
1103  * Define "main" method using service macro.
1104  */
1105 GNUNET_SERVICE_MAIN
1106 ("zonemaster",
1107  GNUNET_SERVICE_OPTION_NONE,
1108  &run,
1109  NULL,
1110  NULL,
1111  NULL,
1112  GNUNET_MQ_handler_end());
1113
1114
1115 /* end of gnunet-service-zonemaster.c */