src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software: you can redistribute it and/or modify it
   6      under the terms of the GNU Affero General Public License as published
   7      by the Free Software Foundation, either version 3 of the License,
   8      or (at your option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      Affero General Public License for more details.
  14
  15      You should have received a copy of the GNU Affero General Public License
  16      along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17
  18      SPDX-License-Identifier: AGPL3.0-or-later
  19  */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32
  33
  34 #define LOG_STRERROR_FILE(kind, syscall, \
  35                           filename) GNUNET_log_from_strerror_file (kind, "util", \
  36                                                                    syscall, \
  37                                                                    filename)
  38
  39
  40 /**
  41  * How often should we (re)publish each record before
  42  * it expires?
  43  */
  44 #define PUBLISH_OPS_PER_EXPIRATION 4
  45
  46 /**
  47  * How often do we measure the delta between desired zone
  48  * iteration speed and actual speed, and tell statistics
  49  * service about it?
  50  */
  51 #define DELTA_INTERVAL 100
  52
  53 /**
  54  * How many records do we fetch in one shot from the namestore?
  55  */
  56 #define NS_BLOCK_SIZE 1000
  57
  58 /**
  59  * How many pending DHT operations do we allow at most?
  60  */
  61 #define DHT_QUEUE_LIMIT 2000
  62
  63 /**
  64  * How many events may the namestore give us before it has to wait
  65  * for us to keep up?
  66  */
  67 #define NAMESTORE_QUEUE_LIMIT 50
  68
  69 /**
  70  * The initial interval in milliseconds btween puts in
  71  * a zone iteration
  72  */
  73 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  74
  75 /**
  76  * The upper bound for the zone iteration interval
  77  * (per record).
  78  */
  79 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply ( \
  80     GNUNET_TIME_UNIT_MINUTES, 15)
  81
  82 /**
  83  * The factor the current zone iteration interval is divided by for each
  84  * additional new record
  85  */
  86 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  87
  88 /**
  89  * What replication level do we use for DHT PUT operations?
  90  */
  91 #define DHT_GNS_REPLICATION_LEVEL 5
  92
  93
  94 /**
  95  * Handle for DHT PUT activity triggered from the namestore monitor.
  96  */
  97 struct DhtPutActivity
  98 {
  99   /**
 100    * Kept in a DLL.
 101    */
 102   struct DhtPutActivity *next;
 103
 104   /**
 105    * Kept in a DLL.
 106    */
 107   struct DhtPutActivity *prev;
 108
 109   /**
 110    * Handle for the DHT PUT operation.
 111    */
 112   struct GNUNET_DHT_PutHandle *ph;
 113
 114   /**
 115    * When was this PUT initiated?
 116    */
 117   struct GNUNET_TIME_Absolute start_date;
 118 };
 119
 120
 121 /**
 122  * Handle to the statistics service
 123  */
 124 static struct GNUNET_STATISTICS_Handle *statistics;
 125
 126 /**
 127  * Our handle to the DHT
 128  */
 129 static struct GNUNET_DHT_Handle *dht_handle;
 130
 131 /**
 132  * Our handle to the namestore service
 133  */
 134 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 135
 136 /**
 137  * Handle to iterate over our authoritative zone in namestore
 138  */
 139 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 140
 141 /**
 142  * Head of iteration put activities; kept in a DLL.
 143  */
 144 static struct DhtPutActivity *it_head;
 145
 146 /**
 147  * Tail of iteration put activities; kept in a DLL.
 148  */
 149 static struct DhtPutActivity *it_tail;
 150
 151 /**
 152  * Number of entries in the DHT queue #it_head.
 153  */
 154 static unsigned int dht_queue_length;
 155
 156 /**
 157  * Useful for zone update for DHT put
 158  */
 159 static unsigned long long num_public_records;
 160
 161 /**
 162  * Last seen record count
 163  */
 164 static unsigned long long last_num_public_records;
 165
 166 /**
 167  * Number of successful put operations performed in the current
 168  * measurement cycle (as measured in #check_zone_namestore_next()).
 169  */
 170 static unsigned long long put_cnt;
 171
 172 /**
 173  * What is the frequency at which we currently would like
 174  * to perform DHT puts (per record)?  Calculated in
 175  * update_velocity() from the #zone_publish_time_window()
 176  * and the total number of record sets we have (so far)
 177  * observed in the zone.
 178  */
 179 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 180
 181 /**
 182  * Minimum relative expiration time of records seem during the current
 183  * zone iteration.
 184  */
 185 static struct GNUNET_TIME_Relative min_relative_record_time;
 186
 187 /**
 188  * Minimum relative expiration time of records seem during the last
 189  * zone iteration.
 190  */
 191 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 192
 193 /**
 194  * Default time window for zone iteration
 195  */
 196 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 197
 198 /**
 199  * Time window for zone iteration, adjusted based on relative record
 200  * expiration times in our zone.
 201  */
 202 static struct GNUNET_TIME_Relative zone_publish_time_window;
 203
 204 /**
 205  * When did we last start measuring the #DELTA_INTERVAL successful
 206  * DHT puts? Used for velocity calculations.
 207  */
 208 static struct GNUNET_TIME_Absolute last_put_100;
 209
 210 /**
 211  * By how much should we try to increase our per-record iteration speed
 212  * (over the desired speed calculated directly from the #put_interval)?
 213  * Basically this value corresponds to the per-record CPU time overhead
 214  * we have.
 215  */
 216 static struct GNUNET_TIME_Relative sub_delta;
 217
 218 /**
 219  * zone publish task
 220  */
 221 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 222
 223 /**
 224  * How many more values are left for the current query before we need
 225  * to explicitly ask the namestore for more?
 226  */
 227 static unsigned int ns_iteration_left;
 228
 229 /**
 230  * #GNUNET_YES if zone has never been published before
 231  */
 232 static int first_zone_iteration;
 233
 234 /**
 235  * Optimize block insertion by caching map of private keys to
 236  * public keys in memory?
 237  */
 238 static int cache_keys;
 239
 240
 241 /**
 242  * Task run during shutdown.
 243  *
 244  * @param cls unused
 245  * @param tc unused
 246  */
 247 static void
 248 shutdown_task (void *cls)
 249 {
 250   struct DhtPutActivity *ma;
 251
 252   (void) cls;
 253   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 254               "Shutting down!\n");
 255   while (NULL != (ma = it_head))
 256   {
 257     GNUNET_DHT_put_cancel (ma->ph);
 258     dht_queue_length--;
 259     GNUNET_CONTAINER_DLL_remove (it_head,
 260                                  it_tail,
 261                                  ma);
 262     dht_queue_length--;
 263     GNUNET_free (ma);
 264   }
 265   if (NULL != statistics)
 266   {
 267     GNUNET_STATISTICS_destroy (statistics,
 268                                GNUNET_NO);
 269     statistics = NULL;
 270   }
 271   if (NULL != zone_publish_task)
 272   {
 273     GNUNET_SCHEDULER_cancel (zone_publish_task);
 274     zone_publish_task = NULL;
 275   }
 276   if (NULL != namestore_iter)
 277   {
 278     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 279     namestore_iter = NULL;
 280   }
 281   if (NULL != namestore_handle)
 282   {
 283     GNUNET_NAMESTORE_disconnect (namestore_handle);
 284     namestore_handle = NULL;
 285   }
 286   if (NULL != dht_handle)
 287   {
 288     GNUNET_DHT_disconnect (dht_handle);
 289     dht_handle = NULL;
 290   }
 291 }
 292
 293
 294 /**
 295  * Method called periodically that triggers iteration over authoritative records
 296  *
 297  * @param cls NULL
 298  */
 299 static void
 300 publish_zone_namestore_next (void *cls)
 301 {
 302   (void) cls;
 303   zone_publish_task = NULL;
 304   GNUNET_assert (NULL != namestore_iter);
 305   GNUNET_assert (0 == ns_iteration_left);
 306   ns_iteration_left = NS_BLOCK_SIZE;
 307   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 308                                        NS_BLOCK_SIZE);
 309 }
 310
 311
 312 /**
 313  * Periodically iterate over our zone and store everything in dht
 314  *
 315  * @param cls NULL
 316  */
 317 static void
 318 publish_zone_dht_start (void *cls);
 319
 320
 321 /**
 322  * Calculate #target_iteration_velocity_per_record.
 323  */
 324 static void
 325 calculate_put_interval ()
 326 {
 327   if (0 == num_public_records)
 328   {
 329     /**
 330      * If no records are known (startup) or none present
 331      * we can safely set the interval to the value for a single
 332      * record
 333      */target_iteration_velocity_per_record = zone_publish_time_window;
 334     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 335                 "No records in namestore database.\n");
 336   }
 337   else
 338   {
 339     last_min_relative_record_time
 340       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 341                                   min_relative_record_time);
 342     zone_publish_time_window
 343       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (
 344                                     last_min_relative_record_time,
 345                                     PUBLISH_OPS_PER_EXPIRATION),
 346                                   zone_publish_time_window_default);
 347     target_iteration_velocity_per_record
 348       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 349                                      last_num_public_records);
 350   }
 351   target_iteration_velocity_per_record
 352     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 353                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 354   GNUNET_STATISTICS_set (statistics,
 355                          "Minimum relative record expiration (in μs)",
 356                          last_min_relative_record_time.rel_value_us,
 357                          GNUNET_NO);
 358   GNUNET_STATISTICS_set (statistics,
 359                          "Zone publication time window (in μs)",
 360                          zone_publish_time_window.rel_value_us,
 361                          GNUNET_NO);
 362   GNUNET_STATISTICS_set (statistics,
 363                          "Target zone iteration velocity (μs)",
 364                          target_iteration_velocity_per_record.rel_value_us,
 365                          GNUNET_NO);
 366 }
 367
 368
 369 /**
 370  * Re-calculate our velocity and the desired velocity.
 371  * We have succeeded in making #DELTA_INTERVAL puts, so
 372  * now calculate the new desired delay between puts.
 373  *
 374  * @param cnt how many records were processed since the last call?
 375  */
 376 static void
 377 update_velocity (unsigned int cnt)
 378 {
 379   struct GNUNET_TIME_Relative delta;
 380   unsigned long long pct = 0;
 381
 382   if (0 == cnt)
 383     return;
 384   /* How fast were we really? */
 385   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 386   delta.rel_value_us /= cnt;
 387   last_put_100 = GNUNET_TIME_absolute_get ();
 388
 389   /* calculate expected frequency */
 390   if ((num_public_records > last_num_public_records) &&
 391       (GNUNET_NO == first_zone_iteration))
 392   {
 393     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 394                 "Last record count was lower than current record count.  Reducing interval.\n");
 395     last_num_public_records = num_public_records
 396                               * LATE_ITERATION_SPEEDUP_FACTOR;
 397     calculate_put_interval ();
 398   }
 399   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 400               "Desired global zone iteration interval is %s/record!\n",
 401               GNUNET_STRINGS_relative_time_to_string (
 402                 target_iteration_velocity_per_record,
 403                 GNUNET_YES));
 404
 405   /* Tell statistics actual vs. desired speed */
 406   GNUNET_STATISTICS_set (statistics,
 407                          "Current zone iteration velocity (μs/record)",
 408                          delta.rel_value_us,
 409                          GNUNET_NO);
 410   /* update "sub_delta" based on difference, taking
 411      previous sub_delta into account! */
 412   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 413   {
 414     /* We were too fast, reduce sub_delta! */
 415     struct GNUNET_TIME_Relative corr;
 416
 417     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 418                                           delta);
 419     if (sub_delta.rel_value_us > delta.rel_value_us)
 420     {
 421       /* Reduce sub_delta by corr */
 422       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 423                                                  corr);
 424     }
 425     else
 426     {
 427       /* We're doing fine with waiting the full time, this
 428          should theoretically only happen if we run at
 429          infinite speed. */
 430       sub_delta = GNUNET_TIME_UNIT_ZERO;
 431     }
 432   }
 433   else if (target_iteration_velocity_per_record.rel_value_us <
 434            delta.rel_value_us)
 435   {
 436     /* We were too slow, increase sub_delta! */
 437     struct GNUNET_TIME_Relative corr;
 438
 439     corr = GNUNET_TIME_relative_subtract (delta,
 440                                           target_iteration_velocity_per_record);
 441     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 442                                           corr);
 443     if (sub_delta.rel_value_us >
 444         target_iteration_velocity_per_record.rel_value_us)
 445     {
 446       /* CPU overload detected, we cannot go at desired speed,
 447          as this would mean using a negative delay. */
 448       /* compute how much faster we would want to be for
 449          the desired velocity */
 450       if (0 == target_iteration_velocity_per_record.rel_value_us)
 451         pct = UINT64_MAX;     /* desired speed is infinity ... */
 452       else
 453         pct = (sub_delta.rel_value_us
 454                - target_iteration_velocity_per_record.rel_value_us) * 100LLU
 455               / target_iteration_velocity_per_record.rel_value_us;
 456       sub_delta = target_iteration_velocity_per_record;
 457     }
 458   }
 459   GNUNET_STATISTICS_set (statistics,
 460                          "# size of the DHT queue (it)",
 461                          dht_queue_length,
 462                          GNUNET_NO);
 463   GNUNET_STATISTICS_set (statistics,
 464                          "% speed increase needed for target velocity",
 465                          pct,
 466                          GNUNET_NO);
 467   GNUNET_STATISTICS_set (statistics,
 468                          "# records processed in current iteration",
 469                          num_public_records,
 470                          GNUNET_NO);
 471 }
 472
 473
 474 /**
 475  * Check if the current zone iteration needs to be continued
 476  * by calling #publish_zone_namestore_next(), and if so with what delay.
 477  */
 478 static void
 479 check_zone_namestore_next ()
 480 {
 481   struct GNUNET_TIME_Relative delay;
 482
 483   if (0 != ns_iteration_left)
 484     return; /* current NAMESTORE iteration not yet done */
 485   update_velocity (put_cnt);
 486   put_cnt = 0;
 487   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 488                                          sub_delta);
 489   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 490      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 491   GNUNET_STATISTICS_set (statistics,
 492                          "Current artificial NAMESTORE delay (μs/record)",
 493                          delay.rel_value_us,
 494                          GNUNET_NO);
 495   delay = GNUNET_TIME_relative_multiply (delay,
 496                                          NS_BLOCK_SIZE);
 497   /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
 498   delay = GNUNET_TIME_relative_min (MAXIMUM_ZONE_ITERATION_INTERVAL,
 499                                     delay);
 500   /* no delays on first iteration */
 501   if (GNUNET_YES == first_zone_iteration)
 502     delay = GNUNET_TIME_UNIT_ZERO;
 503   GNUNET_assert (NULL == zone_publish_task);
 504   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 505                                                     &publish_zone_namestore_next,
 506                                                     NULL);
 507 }
 508
 509
 510 /**
 511  * Continuation called from DHT once the PUT operation is done.
 512  *
 513  * @param cls a `struct DhtPutActivity`
 514  */
 515 static void
 516 dht_put_continuation (void *cls)
 517 {
 518   struct DhtPutActivity *ma = cls;
 519
 520   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 521               "PUT complete\n");
 522   dht_queue_length--;
 523   GNUNET_CONTAINER_DLL_remove (it_head,
 524                                it_tail,
 525                                ma);
 526   GNUNET_free (ma);
 527 }
 528
 529
 530 /**
 531  * Convert namestore records from the internal format to that
 532  * suitable for publication (removes private records, converts
 533  * to absolute expiration time).
 534  *
 535  * @param rd input records
 536  * @param rd_count size of the @a rd and @a rd_public arrays
 537  * @param rd_public where to write the converted records
 538  * @return number of records written to @a rd_public
 539  */
 540 static unsigned int
 541 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 542                             unsigned int rd_count,
 543                             struct GNUNET_GNSRECORD_Data *rd_public)
 544 {
 545   struct GNUNET_TIME_Absolute now;
 546   unsigned int rd_public_count;
 547
 548   rd_public_count = 0;
 549   now = GNUNET_TIME_absolute_get ();
 550   for (unsigned int i = 0; i < rd_count; i++)
 551   {
 552     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 553       continue;
 554     if ((0 == (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) &&
 555         (rd[i].expiration_time < now.abs_value_us))
 556       continue;   /* record already expired, skip it */
 557     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 558     {
 559       /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 560          we just need to adjust our iteration frequency */
 561       min_relative_record_time.rel_value_us =
 562         GNUNET_MIN (rd[i].expiration_time,
 563                     min_relative_record_time.rel_value_us);
 564     }
 565     rd_public[rd_public_count++] = rd[i];
 566   }
 567   return rd_public_count;
 568 }
 569
 570
 571 /**
 572  * Store GNS records in the DHT.
 573  *
 574  * @param key key of the zone
 575  * @param label label to store under
 576  * @param rd_public public record data
 577  * @param rd_public_count number of records in @a rd_public
 578  * @param ma handle for the put operation
 579  * @return DHT PUT handle, NULL on error
 580  */
 581 static struct GNUNET_DHT_PutHandle *
 582 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 583                  const char *label,
 584                  const struct GNUNET_GNSRECORD_Data *rd_public,
 585                  unsigned int rd_public_count,
 586                  struct DhtPutActivity *ma)
 587 {
 588   struct GNUNET_GNSRECORD_Block *block;
 589   struct GNUNET_HashCode query;
 590   struct GNUNET_TIME_Absolute expire;
 591   size_t block_size;
 592   struct GNUNET_DHT_PutHandle *ret;
 593
 594   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 595                                                         rd_public);
 596   if (cache_keys)
 597     block = GNUNET_GNSRECORD_block_create2 (key,
 598                                             expire,
 599                                             label,
 600                                             rd_public,
 601                                             rd_public_count);
 602   else
 603     block = GNUNET_GNSRECORD_block_create (key,
 604                                            expire,
 605                                            label,
 606                                            rd_public,
 607                                            rd_public_count);
 608   if (NULL == block)
 609   {
 610     GNUNET_break (0);
 611     return NULL;   /* whoops */
 612   }
 613   block_size = ntohl (block->purpose.size)
 614                + sizeof(struct GNUNET_CRYPTO_EcdsaSignature)
 615                + sizeof(struct GNUNET_CRYPTO_EcdsaPublicKey);
 616   GNUNET_GNSRECORD_query_from_private_key (key,
 617                                            label,
 618                                            &query);
 619   GNUNET_STATISTICS_update (statistics,
 620                             "DHT put operations initiated",
 621                             1,
 622                             GNUNET_NO);
 623   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 624               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 625               rd_public_count,
 626               label,
 627               GNUNET_STRINGS_absolute_time_to_string (expire),
 628               GNUNET_h2s (&query));
 629   num_public_records++;
 630   ret = GNUNET_DHT_put (dht_handle,
 631                         &query,
 632                         DHT_GNS_REPLICATION_LEVEL,
 633                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 634                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 635                         block_size,
 636                         block,
 637                         expire,
 638                         &dht_put_continuation,
 639                         ma);
 640   GNUNET_free (block);
 641   return ret;
 642 }
 643
 644
 645 /**
 646  * We encountered an error in our zone iteration.
 647  *
 648  * @param cls NULL
 649  */
 650 static void
 651 zone_iteration_error (void *cls)
 652 {
 653   (void) cls;
 654   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 655               "Got disconnected from namestore database, retrying.\n");
 656   namestore_iter = NULL;
 657   /* We end up here on error/disconnect/shutdown, so potentially
 658      while a zone publish task or a DHT put is still running; hence
 659      we need to cancel those. */
 660   if (NULL != zone_publish_task)
 661   {
 662     GNUNET_SCHEDULER_cancel (zone_publish_task);
 663     zone_publish_task = NULL;
 664   }
 665   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 666                                                 NULL);
 667 }
 668
 669
 670 /**
 671  * Zone iteration is completed.
 672  *
 673  * @param cls NULL
 674  */
 675 static void
 676 zone_iteration_finished (void *cls)
 677 {
 678   (void) cls;
 679   /* we're done with one iteration, calculate when to do the next one */
 680   namestore_iter = NULL;
 681   last_num_public_records = num_public_records;
 682   first_zone_iteration = GNUNET_NO;
 683   last_min_relative_record_time = min_relative_record_time;
 684   calculate_put_interval ();
 685   /* reset for next iteration */
 686   min_relative_record_time
 687     = GNUNET_TIME_UNIT_FOREVER_REL;
 688   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 689               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 690               GNUNET_STRINGS_relative_time_to_string (
 691                 target_iteration_velocity_per_record,
 692                 GNUNET_YES));
 693   GNUNET_STATISTICS_set (statistics,
 694                          "Target zone iteration velocity (μs)",
 695                          target_iteration_velocity_per_record.rel_value_us,
 696                          GNUNET_NO);
 697   GNUNET_STATISTICS_set (statistics,
 698                          "Number of public records in DHT",
 699                          last_num_public_records,
 700                          GNUNET_NO);
 701   GNUNET_assert (NULL == zone_publish_task);
 702   if (0 == last_num_public_records)
 703   {
 704     zone_publish_task = GNUNET_SCHEDULER_add_delayed (
 705       target_iteration_velocity_per_record,
 706       &publish_zone_dht_start,
 707       NULL);
 708   }
 709   else
 710   {
 711     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 712                                                   NULL);
 713   }
 714 }
 715
 716
 717 /**
 718  * Function used to put all records successively into the DHT.
 719  *
 720  * @param cls the closure (NULL)
 721  * @param key the private key of the authority (ours)
 722  * @param label the name of the records, NULL once the iteration is done
 723  * @param rd_count the number of records in @a rd
 724  * @param rd the record data
 725  */
 726 static void
 727 put_gns_record (void *cls,
 728                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 729                 const char *label,
 730                 unsigned int rd_count,
 731                 const struct GNUNET_GNSRECORD_Data *rd)
 732 {
 733   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 734   unsigned int rd_public_count;
 735   struct DhtPutActivity *ma;
 736
 737   (void) cls;
 738   ns_iteration_left--;
 739   rd_public_count = convert_records_for_export (rd,
 740                                                 rd_count,
 741                                                 rd_public);
 742   if (0 == rd_public_count)
 743   {
 744     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 745                 "Record set empty, moving to next record set\n");
 746     check_zone_namestore_next ();
 747     return;
 748   }
 749   /* We got a set of records to publish */
 750   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 751               "Starting DHT PUT\n");
 752   ma = GNUNET_new (struct DhtPutActivity);
 753   ma->start_date = GNUNET_TIME_absolute_get ();
 754   ma->ph = perform_dht_put (key,
 755                             label,
 756                             rd_public,
 757                             rd_public_count,
 758                             ma);
 759   put_cnt++;
 760   if (0 == put_cnt % DELTA_INTERVAL)
 761     update_velocity (DELTA_INTERVAL);
 762   check_zone_namestore_next ();
 763   if (NULL == ma->ph)
 764   {
 765     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 766                 "Could not perform DHT PUT, is the DHT running?\n");
 767     GNUNET_free (ma);
 768     return;
 769   }
 770   dht_queue_length++;
 771   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 772                                     it_tail,
 773                                     ma);
 774   if (dht_queue_length > DHT_QUEUE_LIMIT)
 775   {
 776     ma = it_head;
 777     GNUNET_CONTAINER_DLL_remove (it_head,
 778                                  it_tail,
 779                                  ma);
 780     GNUNET_DHT_put_cancel (ma->ph);
 781     dht_queue_length--;
 782     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 783                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 784                 GNUNET_STRINGS_relative_time_to_string (
 785                   GNUNET_TIME_absolute_get_duration (ma->start_date),
 786                   GNUNET_YES));
 787     GNUNET_free (ma);
 788   }
 789 }
 790
 791
 792 /**
 793  * Periodically iterate over all zones and store everything in DHT
 794  *
 795  * @param cls NULL
 796  */
 797 static void
 798 publish_zone_dht_start (void *cls)
 799 {
 800   (void) cls;
 801   zone_publish_task = NULL;
 802   GNUNET_STATISTICS_update (statistics,
 803                             "Full zone iterations launched",
 804                             1,
 805                             GNUNET_NO);
 806   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 807               "Starting DHT zone update!\n");
 808   /* start counting again */
 809   num_public_records = 0;
 810   GNUNET_assert (NULL == namestore_iter);
 811   ns_iteration_left = 1;
 812   namestore_iter
 813     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 814                                              NULL, /* All zones */
 815                                              &zone_iteration_error,
 816                                              NULL,
 817                                              &put_gns_record,
 818                                              NULL,
 819                                              &zone_iteration_finished,
 820                                              NULL);
 821   GNUNET_assert (NULL != namestore_iter);
 822 }
 823
 824
 825 /**
 826  * Performe zonemaster duties: watch namestore, publish records.
 827  *
 828  * @param cls closure
 829  * @param server the initialized server
 830  * @param c configuration to use
 831  */
 832 static void
 833 run (void *cls,
 834      const struct GNUNET_CONFIGURATION_Handle *c,
 835      struct GNUNET_SERVICE_Handle *service)
 836 {
 837   unsigned long long max_parallel_bg_queries = 128;
 838
 839   (void) cls;
 840   (void) service;
 841   last_put_100 = GNUNET_TIME_absolute_get ();  /* first time! */
 842   min_relative_record_time
 843     = GNUNET_TIME_UNIT_FOREVER_REL;
 844   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 845   namestore_handle = GNUNET_NAMESTORE_connect (c);
 846   if (NULL == namestore_handle)
 847   {
 848     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 849                 _ ("Failed to connect to the namestore!\n"));
 850     GNUNET_SCHEDULER_shutdown ();
 851     return;
 852   }
 853   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 854                                                      "namestore",
 855                                                      "CACHE_KEYS");
 856   zone_publish_time_window_default = GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY;
 857   if (GNUNET_OK ==
 858       GNUNET_CONFIGURATION_get_value_time (c,
 859                                            "zonemaster",
 860                                            "ZONE_PUBLISH_TIME_WINDOW",
 861                                            &zone_publish_time_window_default))
 862   {
 863     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 864                 "Time window for zone iteration: %s\n",
 865                 GNUNET_STRINGS_relative_time_to_string (
 866                   zone_publish_time_window,
 867                   GNUNET_YES));
 868   }
 869   zone_publish_time_window = zone_publish_time_window_default;
 870   if (GNUNET_OK ==
 871       GNUNET_CONFIGURATION_get_value_number (c,
 872                                              "zonemaster",
 873                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 874                                              &max_parallel_bg_queries))
 875   {
 876     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 877                 "Number of allowed parallel background queries: %llu\n",
 878                 max_parallel_bg_queries);
 879   }
 880   if (0 == max_parallel_bg_queries)
 881     max_parallel_bg_queries = 1;
 882   dht_handle = GNUNET_DHT_connect (c,
 883                                    (unsigned int) max_parallel_bg_queries);
 884   if (NULL == dht_handle)
 885   {
 886     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 887                 _ ("Could not connect to DHT!\n"));
 888     GNUNET_SCHEDULER_add_now (&shutdown_task,
 889                               NULL);
 890     return;
 891   }
 892
 893   /* Schedule periodic put for our records. */
 894   first_zone_iteration = GNUNET_YES;
 895   statistics = GNUNET_STATISTICS_create ("zonemaster",
 896                                          c);
 897   GNUNET_STATISTICS_set (statistics,
 898                          "Target zone iteration velocity (μs)",
 899                          target_iteration_velocity_per_record.rel_value_us,
 900                          GNUNET_NO);
 901   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 902                                                 NULL);
 903   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 904                                  NULL);
 905 }
 906
 907
 908 /**
 909  * Define "main" method using service macro.
 910  */
 911 GNUNET_SERVICE_MAIN
 912   ("zonemaster",
 913   GNUNET_SERVICE_OPTION_NONE,
 914   &run,
 915   NULL,
 916   NULL,
 917   NULL,
 918   GNUNET_MQ_handler_end ());
 919
 920
 921 /* end of gnunet-service-zonemaster.c */