src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32
  33
  34 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  35
  36
  37 /**
  38  * How often should we (re)publish each record before
  39  * it expires?
  40  */
  41 #define PUBLISH_OPS_PER_EXPIRATION 4
  42
  43 /**
  44  * How often do we measure the delta between desired zone
  45  * iteration speed and actual speed, and tell statistics
  46  * service about it?
  47  */
  48 #define DELTA_INTERVAL 100
  49
  50 /**
  51  * How many records do we fetch in one shot from the namestore?
  52  */
  53 #define NS_BLOCK_SIZE 1000
  54
  55 /**
  56  * How many pending DHT operations do we allow at most?
  57  */
  58 #define DHT_QUEUE_LIMIT 2000
  59
  60 /**
  61  * How many events may the namestore give us before it has to wait
  62  * for us to keep up?
  63  */
  64 #define NAMESTORE_QUEUE_LIMIT 50
  65
  66 /**
  67  * The initial interval in milliseconds btween puts in
  68  * a zone iteration
  69  */
  70 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  71
  72 /**
  73  * The upper bound for the zone iteration interval
  74  * (per record).
  75  */
  76 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  77
  78 /**
  79  * The default put interval for the zone iteration. In case
  80  * no option is found
  81  */
  82 #define DEFAULT_ZONE_PUBLISH_TIME_WINDOW GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
  83
  84 /**
  85  * The factor the current zone iteration interval is divided by for each
  86  * additional new record
  87  */
  88 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  89
  90 /**
  91  * What replication level do we use for DHT PUT operations?
  92  */
  93 #define DHT_GNS_REPLICATION_LEVEL 5
  94
  95
  96 /**
  97  * Handle for DHT PUT activity triggered from the namestore monitor.
  98  */
  99 struct DhtPutActivity
 100 {
 101   /**
 102    * Kept in a DLL.
 103    */
 104   struct DhtPutActivity *next;
 105
 106   /**
 107    * Kept in a DLL.
 108    */
 109   struct DhtPutActivity *prev;
 110
 111   /**
 112    * Handle for the DHT PUT operation.
 113    */
 114   struct GNUNET_DHT_PutHandle *ph;
 115
 116   /**
 117    * When was this PUT initiated?
 118    */
 119   struct GNUNET_TIME_Absolute start_date;
 120 };
 121
 122
 123 /**
 124  * Handle to the statistics service
 125  */
 126 static struct GNUNET_STATISTICS_Handle *statistics;
 127
 128 /**
 129  * Our handle to the DHT
 130  */
 131 static struct GNUNET_DHT_Handle *dht_handle;
 132
 133 /**
 134  * Our handle to the namestore service
 135  */
 136 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 137
 138 /**
 139  * Handle to iterate over our authoritative zone in namestore
 140  */
 141 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 142
 143 /**
 144  * Head of iteration put activities; kept in a DLL.
 145  */
 146 static struct DhtPutActivity *it_head;
 147
 148 /**
 149  * Tail of iteration put activities; kept in a DLL.
 150  */
 151 static struct DhtPutActivity *it_tail;
 152
 153 /**
 154  * Number of entries in the DHT queue #it_head.
 155  */
 156 static unsigned int dht_queue_length;
 157
 158 /**
 159  * Number of entries in the DHT queue #ma_head.
 160  */
 161 static unsigned int ma_queue_length;
 162
 163 /**
 164  * Useful for zone update for DHT put
 165  */
 166 static unsigned long long num_public_records;
 167
 168 /**
 169  * Last seen record count
 170  */
 171 static unsigned long long last_num_public_records;
 172
 173 /**
 174  * Number of successful put operations performed in the current
 175  * measurement cycle (as measured in #check_zone_namestore_next()).
 176  */
 177 static unsigned long long put_cnt;
 178
 179 /**
 180  * What is the frequency at which we currently would like
 181  * to perform DHT puts (per record)?  Calculated in
 182  * update_velocity() from the #zone_publish_time_window()
 183  * and the total number of record sets we have (so far)
 184  * observed in the zone.
 185  */
 186 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 187
 188 /**
 189  * Minimum relative expiration time of records seem during the current
 190  * zone iteration.
 191  */
 192 static struct GNUNET_TIME_Relative min_relative_record_time;
 193
 194 /**
 195  * Minimum relative expiration time of records seem during the last
 196  * zone iteration.
 197  */
 198 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 199
 200 /**
 201  * Default time window for zone iteration
 202  */
 203 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 204
 205 /**
 206  * Time window for zone iteration, adjusted based on relative record
 207  * expiration times in our zone.
 208  */
 209 static struct GNUNET_TIME_Relative zone_publish_time_window;
 210
 211 /**
 212  * When did we last start measuring the #DELTA_INTERVAL successful
 213  * DHT puts? Used for velocity calculations.
 214  */
 215 static struct GNUNET_TIME_Absolute last_put_100;
 216
 217 /**
 218  * By how much should we try to increase our per-record iteration speed
 219  * (over the desired speed calculated directly from the #put_interval)?
 220  * Basically this value corresponds to the per-record CPU time overhead
 221  * we have.
 222  */
 223 static struct GNUNET_TIME_Relative sub_delta;
 224
 225 /**
 226  * zone publish task
 227  */
 228 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 229
 230 /**
 231  * How many more values are left for the current query before we need
 232  * to explicitly ask the namestore for more?
 233  */
 234 static unsigned int ns_iteration_left;
 235
 236 /**
 237  * #GNUNET_YES if zone has never been published before
 238  */
 239 static int first_zone_iteration;
 240
 241 /**
 242  * Optimize block insertion by caching map of private keys to
 243  * public keys in memory?
 244  */
 245 static int cache_keys;
 246
 247
 248 /**
 249  * Task run during shutdown.
 250  *
 251  * @param cls unused
 252  * @param tc unused
 253  */
 254 static void
 255 shutdown_task (void *cls)
 256 {
 257   struct DhtPutActivity *ma;
 258
 259   (void) cls;
 260   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 261               "Shutting down!\n");
 262   while (NULL != (ma = it_head))
 263   {
 264     GNUNET_DHT_put_cancel (ma->ph);
 265     dht_queue_length--;
 266     GNUNET_CONTAINER_DLL_remove (it_head,
 267                                  it_tail,
 268                                  ma);
 269     dht_queue_length--;
 270     GNUNET_free (ma);
 271   }
 272   if (NULL != statistics)
 273   {
 274     GNUNET_STATISTICS_destroy (statistics,
 275                                GNUNET_NO);
 276     statistics = NULL;
 277   }
 278   if (NULL != zone_publish_task)
 279   {
 280     GNUNET_SCHEDULER_cancel (zone_publish_task);
 281     zone_publish_task = NULL;
 282   }
 283   if (NULL != namestore_iter)
 284   {
 285     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 286     namestore_iter = NULL;
 287   }
 288   if (NULL != namestore_handle)
 289   {
 290     GNUNET_NAMESTORE_disconnect (namestore_handle);
 291     namestore_handle = NULL;
 292   }
 293   if (NULL != dht_handle)
 294   {
 295     GNUNET_DHT_disconnect (dht_handle);
 296     dht_handle = NULL;
 297   }
 298 }
 299
 300
 301 /**
 302  * Method called periodically that triggers iteration over authoritative records
 303  *
 304  * @param cls NULL
 305  */
 306 static void
 307 publish_zone_namestore_next (void *cls)
 308 {
 309   (void) cls;
 310   zone_publish_task = NULL;
 311   GNUNET_assert (NULL != namestore_iter);
 312   GNUNET_assert (0 == ns_iteration_left);
 313   ns_iteration_left = NS_BLOCK_SIZE;
 314   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 315                                        NS_BLOCK_SIZE);
 316 }
 317
 318
 319 /**
 320  * Periodically iterate over our zone and store everything in dht
 321  *
 322  * @param cls NULL
 323  */
 324 static void
 325 publish_zone_dht_start (void *cls);
 326
 327
 328 /**
 329  * Calculate #target_iteration_velocity_per_record.
 330  */
 331 static void
 332 calculate_put_interval ()
 333 {
 334   if (0 == num_public_records)
 335   {
 336     /**
 337      * If no records are known (startup) or none present
 338      * we can safely set the interval to the value for a single
 339      * record
 340      */
 341     target_iteration_velocity_per_record = zone_publish_time_window;
 342     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 343                 "No records in namestore database.\n");
 344   }
 345   else
 346   {
 347     last_min_relative_record_time
 348       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 349                                   min_relative_record_time);
 350     zone_publish_time_window
 351       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 352                                                                PUBLISH_OPS_PER_EXPIRATION),
 353                                   zone_publish_time_window_default);
 354     target_iteration_velocity_per_record
 355       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 356                                      last_num_public_records);
 357   }
 358   target_iteration_velocity_per_record
 359     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 360                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 361   GNUNET_STATISTICS_set (statistics,
 362                          "Minimum relative record expiration (in μs)",
 363                          last_min_relative_record_time.rel_value_us,
 364                          GNUNET_NO);
 365   GNUNET_STATISTICS_set (statistics,
 366                          "Zone publication time window (in μs)",
 367                          zone_publish_time_window.rel_value_us,
 368                          GNUNET_NO);
 369   GNUNET_STATISTICS_set (statistics,
 370                          "Target zone iteration velocity (μs)",
 371                          target_iteration_velocity_per_record.rel_value_us,
 372                          GNUNET_NO);
 373 }
 374
 375
 376 /**
 377  * Re-calculate our velocity and the desired velocity.
 378  * We have succeeded in making #DELTA_INTERVAL puts, so
 379  * now calculate the new desired delay between puts.
 380  *
 381  * @param cnt how many records were processed since the last call?
 382  */
 383 static void
 384 update_velocity (unsigned int cnt)
 385 {
 386   struct GNUNET_TIME_Relative delta;
 387   unsigned long long pct = 0;
 388
 389   if (0 == cnt)
 390     return;
 391   /* How fast were we really? */
 392   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 393   delta.rel_value_us /= cnt;
 394   last_put_100 = GNUNET_TIME_absolute_get ();
 395
 396   /* calculate expected frequency */
 397   if ( (num_public_records > last_num_public_records) &&
 398        (GNUNET_NO == first_zone_iteration) )
 399   {
 400     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 401                 "Last record count was lower than current record count.  Reducing interval.\n");
 402     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 403     calculate_put_interval ();
 404   }
 405   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 406               "Desired global zone iteration interval is %s/record!\n",
 407               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 408                                                       GNUNET_YES));
 409
 410   /* Tell statistics actual vs. desired speed */
 411   GNUNET_STATISTICS_set (statistics,
 412                          "Current zone iteration velocity (μs/record)",
 413                          delta.rel_value_us,
 414                          GNUNET_NO);
 415   /* update "sub_delta" based on difference, taking
 416      previous sub_delta into account! */
 417   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 418   {
 419     /* We were too fast, reduce sub_delta! */
 420     struct GNUNET_TIME_Relative corr;
 421
 422     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 423                                           delta);
 424     if (sub_delta.rel_value_us > delta.rel_value_us)
 425     {
 426       /* Reduce sub_delta by corr */
 427       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 428                                                  corr);
 429     }
 430     else
 431     {
 432       /* We're doing fine with waiting the full time, this
 433          should theoretically only happen if we run at
 434          infinite speed. */
 435       sub_delta = GNUNET_TIME_UNIT_ZERO;
 436     }
 437   }
 438   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 439   {
 440     /* We were too slow, increase sub_delta! */
 441     struct GNUNET_TIME_Relative corr;
 442
 443     corr = GNUNET_TIME_relative_subtract (delta,
 444                                           target_iteration_velocity_per_record);
 445     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 446                                           corr);
 447     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 448     {
 449       /* CPU overload detected, we cannot go at desired speed,
 450          as this would mean using a negative delay. */
 451       /* compute how much faster we would want to be for
 452          the desired velocity */
 453       if (0 == target_iteration_velocity_per_record.rel_value_us)
 454         pct = UINT64_MAX; /* desired speed is infinity ... */
 455       else
 456         pct = (sub_delta.rel_value_us -
 457                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 458           / target_iteration_velocity_per_record.rel_value_us;
 459       sub_delta = target_iteration_velocity_per_record;
 460     }
 461   }
 462   GNUNET_STATISTICS_set (statistics,
 463                          "# size of the DHT queue (it)",
 464                          dht_queue_length,
 465                          GNUNET_NO);
 466   GNUNET_STATISTICS_set (statistics,
 467                          "# size of the DHT queue (mon)",
 468                          ma_queue_length,
 469                          GNUNET_NO);
 470   GNUNET_STATISTICS_set (statistics,
 471                          "% speed increase needed for target velocity",
 472                          pct,
 473                          GNUNET_NO);
 474   GNUNET_STATISTICS_set (statistics,
 475                          "# records processed in current iteration",
 476                          num_public_records,
 477                          GNUNET_NO);
 478 }
 479
 480
 481 /**
 482  * Check if the current zone iteration needs to be continued
 483  * by calling #publish_zone_namestore_next(), and if so with what delay.
 484  */
 485 static void
 486 check_zone_namestore_next ()
 487 {
 488   struct GNUNET_TIME_Relative delay;
 489
 490   if (0 != ns_iteration_left)
 491     return; /* current NAMESTORE iteration not yet done */
 492   update_velocity (put_cnt);
 493   put_cnt = 0;
 494   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 495                                          sub_delta);
 496   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 497      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 498   GNUNET_STATISTICS_set (statistics,
 499                          "Current artificial NAMESTORE delay (μs/record)",
 500                          delay.rel_value_us,
 501                          GNUNET_NO);
 502   delay = GNUNET_TIME_relative_multiply (delay,
 503                                          NS_BLOCK_SIZE);
 504   GNUNET_assert (NULL == zone_publish_task);
 505   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 506                                                     &publish_zone_namestore_next,
 507                                                     NULL);
 508 }
 509
 510
 511 /**
 512  * Continuation called from DHT once the PUT operation is done.
 513  *
 514  * @param cls a `struct DhtPutActivity`
 515  */
 516 static void
 517 dht_put_continuation (void *cls)
 518 {
 519   struct DhtPutActivity *ma = cls;
 520
 521   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 522               "PUT complete\n");
 523   dht_queue_length--;
 524   GNUNET_CONTAINER_DLL_remove (it_head,
 525                                it_tail,
 526                                ma);
 527   GNUNET_free (ma);
 528 }
 529
 530
 531 /**
 532  * Convert namestore records from the internal format to that
 533  * suitable for publication (removes private records, converts
 534  * to absolute expiration time).
 535  *
 536  * @param rd input records
 537  * @param rd_count size of the @a rd and @a rd_public arrays
 538  * @param rd_public where to write the converted records
 539  * @return number of records written to @a rd_public
 540  */
 541 static unsigned int
 542 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 543                             unsigned int rd_count,
 544                             struct GNUNET_GNSRECORD_Data *rd_public)
 545 {
 546   struct GNUNET_TIME_Absolute now;
 547   unsigned int rd_public_count;
 548
 549   rd_public_count = 0;
 550   now = GNUNET_TIME_absolute_get ();
 551   for (unsigned int i=0;i<rd_count;i++)
 552     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 553     {
 554       rd_public[rd_public_count] = rd[i];
 555       if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 556       {
 557         /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 558            we just need to adjust our iteration frequency */
 559         min_relative_record_time.rel_value_us =
 560           GNUNET_MIN (rd_public[rd_public_count].expiration_time,
 561                       min_relative_record_time.rel_value_us);
 562       }
 563       else if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
 564       {
 565         /* record already expired, skip it */
 566         continue;
 567       }
 568       rd_public_count++;
 569     }
 570   return rd_public_count;
 571 }
 572
 573
 574 /**
 575  * Store GNS records in the DHT.
 576  *
 577  * @param key key of the zone
 578  * @param label label to store under
 579  * @param rd_public public record data
 580  * @param rd_public_count number of records in @a rd_public
 581  * @param ma handle for the put operation
 582  * @return DHT PUT handle, NULL on error
 583  */
 584 static struct GNUNET_DHT_PutHandle *
 585 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 586                  const char *label,
 587                  const struct GNUNET_GNSRECORD_Data *rd_public,
 588                  unsigned int rd_public_count,
 589                  struct DhtPutActivity *ma)
 590 {
 591   struct GNUNET_GNSRECORD_Block *block;
 592   struct GNUNET_HashCode query;
 593   struct GNUNET_TIME_Absolute expire;
 594   size_t block_size;
 595   struct GNUNET_DHT_PutHandle *ret;
 596
 597   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 598                                                         rd_public);
 599   if (cache_keys)
 600     block = GNUNET_GNSRECORD_block_create2 (key,
 601                                             expire,
 602                                             label,
 603                                             rd_public,
 604                                             rd_public_count);
 605   else
 606     block = GNUNET_GNSRECORD_block_create (key,
 607                                            expire,
 608                                            label,
 609                                            rd_public,
 610                                            rd_public_count);
 611   if (NULL == block)
 612   {
 613     GNUNET_break (0);
 614     return NULL; /* whoops */
 615   }
 616   block_size = ntohl (block->purpose.size)
 617     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 618     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 619   GNUNET_GNSRECORD_query_from_private_key (key,
 620                                            label,
 621                                            &query);
 622   GNUNET_STATISTICS_update (statistics,
 623                             "DHT put operations initiated",
 624                             1,
 625                             GNUNET_NO);
 626   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 627               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 628               rd_public_count,
 629               label,
 630               GNUNET_STRINGS_absolute_time_to_string (expire),
 631               GNUNET_h2s (&query));
 632   num_public_records++;
 633   ret = GNUNET_DHT_put (dht_handle,
 634                         &query,
 635                         DHT_GNS_REPLICATION_LEVEL,
 636                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 637                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 638                         block_size,
 639                         block,
 640                         expire,
 641                         &dht_put_continuation,
 642                         ma);
 643   GNUNET_free (block);
 644   return ret;
 645 }
 646
 647
 648 /**
 649  * We encountered an error in our zone iteration.
 650  *
 651  * @param cls NULL
 652  */
 653 static void
 654 zone_iteration_error (void *cls)
 655 {
 656   (void) cls;
 657   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 658               "Got disconnected from namestore database, retrying.\n");
 659   namestore_iter = NULL;
 660   /* We end up here on error/disconnect/shutdown, so potentially
 661      while a zone publish task or a DHT put is still running; hence
 662      we need to cancel those. */
 663   if (NULL != zone_publish_task)
 664   {
 665     GNUNET_SCHEDULER_cancel (zone_publish_task);
 666     zone_publish_task = NULL;
 667   }
 668   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 669                                                 NULL);
 670 }
 671
 672
 673 /**
 674  * Zone iteration is completed.
 675  *
 676  * @param cls NULL
 677  */
 678 static void
 679 zone_iteration_finished (void *cls)
 680 {
 681   (void) cls;
 682   /* we're done with one iteration, calculate when to do the next one */
 683   namestore_iter = NULL;
 684   last_num_public_records = num_public_records;
 685   first_zone_iteration = GNUNET_NO;
 686   last_min_relative_record_time = min_relative_record_time;
 687   calculate_put_interval ();
 688   /* reset for next iteration */
 689   min_relative_record_time
 690     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 691                                      PUBLISH_OPS_PER_EXPIRATION);
 692   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 693               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 694               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 695                                                       GNUNET_YES));
 696   GNUNET_STATISTICS_set (statistics,
 697                          "Target zone iteration velocity (μs)",
 698                          target_iteration_velocity_per_record.rel_value_us,
 699                          GNUNET_NO);
 700   GNUNET_STATISTICS_set (statistics,
 701                          "Number of public records in DHT",
 702                          last_num_public_records,
 703                          GNUNET_NO);
 704   GNUNET_assert (NULL == zone_publish_task);
 705   if (0 == last_num_public_records)
 706   {
 707     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 708                                                       &publish_zone_dht_start,
 709                                                       NULL);
 710   }
 711   else
 712   {
 713     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 714                                                   NULL);
 715   }
 716 }
 717
 718
 719 /**
 720  * Function used to put all records successively into the DHT.
 721  *
 722  * @param cls the closure (NULL)
 723  * @param key the private key of the authority (ours)
 724  * @param label the name of the records, NULL once the iteration is done
 725  * @param rd_count the number of records in @a rd
 726  * @param rd the record data
 727  */
 728 static void
 729 put_gns_record (void *cls,
 730                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 731                 const char *label,
 732                 unsigned int rd_count,
 733                 const struct GNUNET_GNSRECORD_Data *rd)
 734 {
 735   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 736   unsigned int rd_public_count;
 737   struct DhtPutActivity *ma;
 738
 739   (void) cls;
 740   ns_iteration_left--;
 741   rd_public_count = convert_records_for_export (rd,
 742                                                 rd_count,
 743                                                 rd_public);
 744   if (0 == rd_public_count)
 745   {
 746     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 747                 "Record set empty, moving to next record set\n");
 748     check_zone_namestore_next ();
 749     return;
 750   }
 751   /* We got a set of records to publish */
 752   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 753               "Starting DHT PUT\n");
 754   ma = GNUNET_new (struct DhtPutActivity);
 755   ma->start_date = GNUNET_TIME_absolute_get ();
 756   ma->ph = perform_dht_put (key,
 757                             label,
 758                             rd_public,
 759                             rd_public_count,
 760                             ma);
 761   put_cnt++;
 762   if (0 == put_cnt % DELTA_INTERVAL)
 763     update_velocity (DELTA_INTERVAL);
 764   check_zone_namestore_next ();
 765   if (NULL == ma->ph)
 766   {
 767     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 768                 "Could not perform DHT PUT, is the DHT running?\n");
 769     GNUNET_free (ma);
 770     return;
 771   }
 772   dht_queue_length++;
 773   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 774                                     it_tail,
 775                                     ma);
 776   if (dht_queue_length > DHT_QUEUE_LIMIT)
 777   {
 778     ma = it_head;
 779     GNUNET_CONTAINER_DLL_remove (it_head,
 780                                  it_tail,
 781                                  ma);
 782     GNUNET_DHT_put_cancel (ma->ph);
 783     dht_queue_length--;
 784     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 785                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 786                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 787                                                         GNUNET_YES));
 788     GNUNET_free (ma);
 789   }
 790 }
 791
 792
 793 /**
 794  * Periodically iterate over all zones and store everything in DHT
 795  *
 796  * @param cls NULL
 797  */
 798 static void
 799 publish_zone_dht_start (void *cls)
 800 {
 801   (void) cls;
 802   zone_publish_task = NULL;
 803   GNUNET_STATISTICS_update (statistics,
 804                             "Full zone iterations launched",
 805                             1,
 806                             GNUNET_NO);
 807   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 808               "Starting DHT zone update!\n");
 809   /* start counting again */
 810   num_public_records = 0;
 811   GNUNET_assert (NULL == namestore_iter);
 812   ns_iteration_left = 1;
 813   namestore_iter
 814     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 815                                              NULL, /* All zones */
 816                                              &zone_iteration_error,
 817                                              NULL,
 818                                              &put_gns_record,
 819                                              NULL,
 820                                              &zone_iteration_finished,
 821                                              NULL);
 822   GNUNET_assert (NULL != namestore_iter);
 823 }
 824
 825
 826 /**
 827  * Performe zonemaster duties: watch namestore, publish records.
 828  *
 829  * @param cls closure
 830  * @param server the initialized server
 831  * @param c configuration to use
 832  */
 833 static void
 834 run (void *cls,
 835      const struct GNUNET_CONFIGURATION_Handle *c,
 836      struct GNUNET_SERVICE_Handle *service)
 837 {
 838   unsigned long long max_parallel_bg_queries = 128;
 839
 840   (void) cls;
 841   (void) service;
 842   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 843   min_relative_record_time
 844     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 845                                      PUBLISH_OPS_PER_EXPIRATION);
 846   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 847   namestore_handle = GNUNET_NAMESTORE_connect (c);
 848   if (NULL == namestore_handle)
 849   {
 850     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 851                 _("Failed to connect to the namestore!\n"));
 852     GNUNET_SCHEDULER_shutdown ();
 853     return;
 854   }
 855   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 856                                                      "namestore",
 857                                                      "CACHE_KEYS");
 858   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
 859   if (GNUNET_OK ==
 860       GNUNET_CONFIGURATION_get_value_time (c,
 861                                            "zonemaster",
 862                                            "ZONE_PUBLISH_TIME_WINDOW",
 863                                            &zone_publish_time_window_default))
 864   {
 865     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 866                 "Time window for zone iteration: %s\n",
 867                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
 868                                                         GNUNET_YES));
 869   }
 870   zone_publish_time_window = zone_publish_time_window_default;
 871   if (GNUNET_OK ==
 872       GNUNET_CONFIGURATION_get_value_number (c,
 873                                              "zonemaster",
 874                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 875                                              &max_parallel_bg_queries))
 876   {
 877     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 878                 "Number of allowed parallel background queries: %llu\n",
 879                 max_parallel_bg_queries);
 880   }
 881   if (0 == max_parallel_bg_queries)
 882     max_parallel_bg_queries = 1;
 883   dht_handle = GNUNET_DHT_connect (c,
 884                                    (unsigned int) max_parallel_bg_queries);
 885   if (NULL == dht_handle)
 886   {
 887     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 888                 _("Could not connect to DHT!\n"));
 889     GNUNET_SCHEDULER_add_now (&shutdown_task,
 890                               NULL);
 891     return;
 892   }
 893
 894   /* Schedule periodic put for our records. */
 895   first_zone_iteration = GNUNET_YES;
 896   statistics = GNUNET_STATISTICS_create ("zonemaster",
 897                                          c);
 898   GNUNET_STATISTICS_set (statistics,
 899                          "Target zone iteration velocity (μs)",
 900                          target_iteration_velocity_per_record.rel_value_us,
 901                          GNUNET_NO);
 902   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 903                                                 NULL);
 904   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 905                                  NULL);
 906 }
 907
 908
 909 /**
 910  * Define "main" method using service macro.
 911  */
 912 GNUNET_SERVICE_MAIN
 913 ("zonemaster",
 914  GNUNET_SERVICE_OPTION_NONE,
 915  &run,
 916  NULL,
 917  NULL,
 918  NULL,
 919  GNUNET_MQ_handler_end());
 920
 921
 922 /* end of gnunet-service-zonemaster.c */