src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32
  33
  34 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  35
  36
  37 /**
  38  * How often should we (re)publish each record before
  39  * it expires?
  40  */
  41 #define PUBLISH_OPS_PER_EXPIRATION 4
  42
  43 /**
  44  * How often do we measure the delta between desired zone
  45  * iteration speed and actual speed, and tell statistics
  46  * service about it?
  47  */
  48 #define DELTA_INTERVAL 100
  49
  50 /**
  51  * How many records do we fetch in one shot from the namestore?
  52  */
  53 #define NS_BLOCK_SIZE 1000
  54
  55 /**
  56  * How many pending DHT operations do we allow at most?
  57  */
  58 #define DHT_QUEUE_LIMIT 2000
  59
  60 /**
  61  * How many events may the namestore give us before it has to wait
  62  * for us to keep up?
  63  */
  64 #define NAMESTORE_QUEUE_LIMIT 50
  65
  66 /**
  67  * The initial interval in milliseconds btween puts in
  68  * a zone iteration
  69  */
  70 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  71
  72 /**
  73  * The upper bound for the zone iteration interval
  74  * (per record).
  75  */
  76 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  77
  78 /**
  79  * The default put interval for the zone iteration. In case
  80  * no option is found
  81  */
  82 #define DEFAULT_ZONE_PUBLISH_TIME_WINDOW GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
  83
  84 /**
  85  * The factor the current zone iteration interval is divided by for each
  86  * additional new record
  87  */
  88 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  89
  90 /**
  91  * What replication level do we use for DHT PUT operations?
  92  */
  93 #define DHT_GNS_REPLICATION_LEVEL 5
  94
  95
  96 /**
  97  * Handle for DHT PUT activity triggered from the namestore monitor.
  98  */
  99 struct DhtPutActivity
 100 {
 101   /**
 102    * Kept in a DLL.
 103    */
 104   struct DhtPutActivity *next;
 105
 106   /**
 107    * Kept in a DLL.
 108    */
 109   struct DhtPutActivity *prev;
 110
 111   /**
 112    * Handle for the DHT PUT operation.
 113    */
 114   struct GNUNET_DHT_PutHandle *ph;
 115
 116   /**
 117    * When was this PUT initiated?
 118    */
 119   struct GNUNET_TIME_Absolute start_date;
 120 };
 121
 122
 123 /**
 124  * Handle to the statistics service
 125  */
 126 static struct GNUNET_STATISTICS_Handle *statistics;
 127
 128 /**
 129  * Our handle to the DHT
 130  */
 131 static struct GNUNET_DHT_Handle *dht_handle;
 132
 133 /**
 134  * Our handle to the namestore service
 135  */
 136 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 137
 138 /**
 139  * Handle to iterate over our authoritative zone in namestore
 140  */
 141 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 142
 143 /**
 144  * Head of iteration put activities; kept in a DLL.
 145  */
 146 static struct DhtPutActivity *it_head;
 147
 148 /**
 149  * Tail of iteration put activities; kept in a DLL.
 150  */
 151 static struct DhtPutActivity *it_tail;
 152
 153 /**
 154  * Number of entries in the DHT queue #it_head.
 155  */
 156 static unsigned int dht_queue_length;
 157
 158 /**
 159  * Useful for zone update for DHT put
 160  */
 161 static unsigned long long num_public_records;
 162
 163 /**
 164  * Last seen record count
 165  */
 166 static unsigned long long last_num_public_records;
 167
 168 /**
 169  * Number of successful put operations performed in the current
 170  * measurement cycle (as measured in #check_zone_namestore_next()).
 171  */
 172 static unsigned long long put_cnt;
 173
 174 /**
 175  * What is the frequency at which we currently would like
 176  * to perform DHT puts (per record)?  Calculated in
 177  * update_velocity() from the #zone_publish_time_window()
 178  * and the total number of record sets we have (so far)
 179  * observed in the zone.
 180  */
 181 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 182
 183 /**
 184  * Minimum relative expiration time of records seem during the current
 185  * zone iteration.
 186  */
 187 static struct GNUNET_TIME_Relative min_relative_record_time;
 188
 189 /**
 190  * Minimum relative expiration time of records seem during the last
 191  * zone iteration.
 192  */
 193 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 194
 195 /**
 196  * Default time window for zone iteration
 197  */
 198 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 199
 200 /**
 201  * Time window for zone iteration, adjusted based on relative record
 202  * expiration times in our zone.
 203  */
 204 static struct GNUNET_TIME_Relative zone_publish_time_window;
 205
 206 /**
 207  * When did we last start measuring the #DELTA_INTERVAL successful
 208  * DHT puts? Used for velocity calculations.
 209  */
 210 static struct GNUNET_TIME_Absolute last_put_100;
 211
 212 /**
 213  * By how much should we try to increase our per-record iteration speed
 214  * (over the desired speed calculated directly from the #put_interval)?
 215  * Basically this value corresponds to the per-record CPU time overhead
 216  * we have.
 217  */
 218 static struct GNUNET_TIME_Relative sub_delta;
 219
 220 /**
 221  * zone publish task
 222  */
 223 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 224
 225 /**
 226  * How many more values are left for the current query before we need
 227  * to explicitly ask the namestore for more?
 228  */
 229 static unsigned int ns_iteration_left;
 230
 231 /**
 232  * #GNUNET_YES if zone has never been published before
 233  */
 234 static int first_zone_iteration;
 235
 236 /**
 237  * Optimize block insertion by caching map of private keys to
 238  * public keys in memory?
 239  */
 240 static int cache_keys;
 241
 242
 243 /**
 244  * Task run during shutdown.
 245  *
 246  * @param cls unused
 247  * @param tc unused
 248  */
 249 static void
 250 shutdown_task (void *cls)
 251 {
 252   struct DhtPutActivity *ma;
 253
 254   (void) cls;
 255   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 256               "Shutting down!\n");
 257   while (NULL != (ma = it_head))
 258   {
 259     GNUNET_DHT_put_cancel (ma->ph);
 260     dht_queue_length--;
 261     GNUNET_CONTAINER_DLL_remove (it_head,
 262                                  it_tail,
 263                                  ma);
 264     dht_queue_length--;
 265     GNUNET_free (ma);
 266   }
 267   if (NULL != statistics)
 268   {
 269     GNUNET_STATISTICS_destroy (statistics,
 270                                GNUNET_NO);
 271     statistics = NULL;
 272   }
 273   if (NULL != zone_publish_task)
 274   {
 275     GNUNET_SCHEDULER_cancel (zone_publish_task);
 276     zone_publish_task = NULL;
 277   }
 278   if (NULL != namestore_iter)
 279   {
 280     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 281     namestore_iter = NULL;
 282   }
 283   if (NULL != namestore_handle)
 284   {
 285     GNUNET_NAMESTORE_disconnect (namestore_handle);
 286     namestore_handle = NULL;
 287   }
 288   if (NULL != dht_handle)
 289   {
 290     GNUNET_DHT_disconnect (dht_handle);
 291     dht_handle = NULL;
 292   }
 293 }
 294
 295
 296 /**
 297  * Method called periodically that triggers iteration over authoritative records
 298  *
 299  * @param cls NULL
 300  */
 301 static void
 302 publish_zone_namestore_next (void *cls)
 303 {
 304   (void) cls;
 305   zone_publish_task = NULL;
 306   GNUNET_assert (NULL != namestore_iter);
 307   GNUNET_assert (0 == ns_iteration_left);
 308   ns_iteration_left = NS_BLOCK_SIZE;
 309   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 310                                        NS_BLOCK_SIZE);
 311 }
 312
 313
 314 /**
 315  * Periodically iterate over our zone and store everything in dht
 316  *
 317  * @param cls NULL
 318  */
 319 static void
 320 publish_zone_dht_start (void *cls);
 321
 322
 323 /**
 324  * Calculate #target_iteration_velocity_per_record.
 325  */
 326 static void
 327 calculate_put_interval ()
 328 {
 329   if (0 == num_public_records)
 330   {
 331     /**
 332      * If no records are known (startup) or none present
 333      * we can safely set the interval to the value for a single
 334      * record
 335      */
 336     target_iteration_velocity_per_record = zone_publish_time_window;
 337     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 338                 "No records in namestore database.\n");
 339   }
 340   else
 341   {
 342     last_min_relative_record_time
 343       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 344                                   min_relative_record_time);
 345     zone_publish_time_window
 346       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 347                                                                PUBLISH_OPS_PER_EXPIRATION),
 348                                   zone_publish_time_window_default);
 349     target_iteration_velocity_per_record
 350       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 351                                      last_num_public_records);
 352   }
 353   target_iteration_velocity_per_record
 354     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 355                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 356   GNUNET_STATISTICS_set (statistics,
 357                          "Minimum relative record expiration (in μs)",
 358                          last_min_relative_record_time.rel_value_us,
 359                          GNUNET_NO);
 360   GNUNET_STATISTICS_set (statistics,
 361                          "Zone publication time window (in μs)",
 362                          zone_publish_time_window.rel_value_us,
 363                          GNUNET_NO);
 364   GNUNET_STATISTICS_set (statistics,
 365                          "Target zone iteration velocity (μs)",
 366                          target_iteration_velocity_per_record.rel_value_us,
 367                          GNUNET_NO);
 368 }
 369
 370
 371 /**
 372  * Re-calculate our velocity and the desired velocity.
 373  * We have succeeded in making #DELTA_INTERVAL puts, so
 374  * now calculate the new desired delay between puts.
 375  *
 376  * @param cnt how many records were processed since the last call?
 377  */
 378 static void
 379 update_velocity (unsigned int cnt)
 380 {
 381   struct GNUNET_TIME_Relative delta;
 382   unsigned long long pct = 0;
 383
 384   if (0 == cnt)
 385     return;
 386   /* How fast were we really? */
 387   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 388   delta.rel_value_us /= cnt;
 389   last_put_100 = GNUNET_TIME_absolute_get ();
 390
 391   /* calculate expected frequency */
 392   if ( (num_public_records > last_num_public_records) &&
 393        (GNUNET_NO == first_zone_iteration) )
 394   {
 395     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 396                 "Last record count was lower than current record count.  Reducing interval.\n");
 397     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 398     calculate_put_interval ();
 399   }
 400   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 401               "Desired global zone iteration interval is %s/record!\n",
 402               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 403                                                       GNUNET_YES));
 404
 405   /* Tell statistics actual vs. desired speed */
 406   GNUNET_STATISTICS_set (statistics,
 407                          "Current zone iteration velocity (μs/record)",
 408                          delta.rel_value_us,
 409                          GNUNET_NO);
 410   /* update "sub_delta" based on difference, taking
 411      previous sub_delta into account! */
 412   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 413   {
 414     /* We were too fast, reduce sub_delta! */
 415     struct GNUNET_TIME_Relative corr;
 416
 417     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 418                                           delta);
 419     if (sub_delta.rel_value_us > delta.rel_value_us)
 420     {
 421       /* Reduce sub_delta by corr */
 422       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 423                                                  corr);
 424     }
 425     else
 426     {
 427       /* We're doing fine with waiting the full time, this
 428          should theoretically only happen if we run at
 429          infinite speed. */
 430       sub_delta = GNUNET_TIME_UNIT_ZERO;
 431     }
 432   }
 433   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 434   {
 435     /* We were too slow, increase sub_delta! */
 436     struct GNUNET_TIME_Relative corr;
 437
 438     corr = GNUNET_TIME_relative_subtract (delta,
 439                                           target_iteration_velocity_per_record);
 440     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 441                                           corr);
 442     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 443     {
 444       /* CPU overload detected, we cannot go at desired speed,
 445          as this would mean using a negative delay. */
 446       /* compute how much faster we would want to be for
 447          the desired velocity */
 448       if (0 == target_iteration_velocity_per_record.rel_value_us)
 449         pct = UINT64_MAX; /* desired speed is infinity ... */
 450       else
 451         pct = (sub_delta.rel_value_us -
 452                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 453           / target_iteration_velocity_per_record.rel_value_us;
 454       sub_delta = target_iteration_velocity_per_record;
 455     }
 456   }
 457   GNUNET_STATISTICS_set (statistics,
 458                          "# size of the DHT queue (it)",
 459                          dht_queue_length,
 460                          GNUNET_NO);
 461   GNUNET_STATISTICS_set (statistics,
 462                          "% speed increase needed for target velocity",
 463                          pct,
 464                          GNUNET_NO);
 465   GNUNET_STATISTICS_set (statistics,
 466                          "# records processed in current iteration",
 467                          num_public_records,
 468                          GNUNET_NO);
 469 }
 470
 471
 472 /**
 473  * Check if the current zone iteration needs to be continued
 474  * by calling #publish_zone_namestore_next(), and if so with what delay.
 475  */
 476 static void
 477 check_zone_namestore_next ()
 478 {
 479   struct GNUNET_TIME_Relative delay;
 480
 481   if (0 != ns_iteration_left)
 482     return; /* current NAMESTORE iteration not yet done */
 483   update_velocity (put_cnt);
 484   put_cnt = 0;
 485   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 486                                          sub_delta);
 487   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 488      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 489   GNUNET_STATISTICS_set (statistics,
 490                          "Current artificial NAMESTORE delay (μs/record)",
 491                          delay.rel_value_us,
 492                          GNUNET_NO);
 493   delay = GNUNET_TIME_relative_multiply (delay,
 494                                          NS_BLOCK_SIZE);
 495   /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
 496   delay = GNUNET_TIME_relative_min (MAXIMUM_ZONE_ITERATION_INTERVAL,
 497                                     delay);
 498   /* no delays on first iteration */
 499   if (GNUNET_YES == first_zone_iteration)
 500     delay = GNUNET_TIME_UNIT_ZERO;
 501   GNUNET_assert (NULL == zone_publish_task);
 502   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 503                                                     &publish_zone_namestore_next,
 504                                                     NULL);
 505 }
 506
 507
 508 /**
 509  * Continuation called from DHT once the PUT operation is done.
 510  *
 511  * @param cls a `struct DhtPutActivity`
 512  */
 513 static void
 514 dht_put_continuation (void *cls)
 515 {
 516   struct DhtPutActivity *ma = cls;
 517
 518   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 519               "PUT complete\n");
 520   dht_queue_length--;
 521   GNUNET_CONTAINER_DLL_remove (it_head,
 522                                it_tail,
 523                                ma);
 524   GNUNET_free (ma);
 525 }
 526
 527
 528 /**
 529  * Convert namestore records from the internal format to that
 530  * suitable for publication (removes private records, converts
 531  * to absolute expiration time).
 532  *
 533  * @param rd input records
 534  * @param rd_count size of the @a rd and @a rd_public arrays
 535  * @param rd_public where to write the converted records
 536  * @return number of records written to @a rd_public
 537  */
 538 static unsigned int
 539 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 540                             unsigned int rd_count,
 541                             struct GNUNET_GNSRECORD_Data *rd_public)
 542 {
 543   struct GNUNET_TIME_Absolute now;
 544   unsigned int rd_public_count;
 545
 546   rd_public_count = 0;
 547   now = GNUNET_TIME_absolute_get ();
 548   for (unsigned int i=0;i<rd_count;i++)
 549     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 550     {
 551       rd_public[rd_public_count] = rd[i];
 552       if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 553       {
 554         /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 555            we just need to adjust our iteration frequency */
 556         min_relative_record_time.rel_value_us =
 557           GNUNET_MIN (rd_public[rd_public_count].expiration_time,
 558                       min_relative_record_time.rel_value_us);
 559       }
 560       else if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
 561       {
 562         /* record already expired, skip it */
 563         continue;
 564       }
 565       rd_public_count++;
 566     }
 567   return rd_public_count;
 568 }
 569
 570
 571 /**
 572  * Store GNS records in the DHT.
 573  *
 574  * @param key key of the zone
 575  * @param label label to store under
 576  * @param rd_public public record data
 577  * @param rd_public_count number of records in @a rd_public
 578  * @param ma handle for the put operation
 579  * @return DHT PUT handle, NULL on error
 580  */
 581 static struct GNUNET_DHT_PutHandle *
 582 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 583                  const char *label,
 584                  const struct GNUNET_GNSRECORD_Data *rd_public,
 585                  unsigned int rd_public_count,
 586                  struct DhtPutActivity *ma)
 587 {
 588   struct GNUNET_GNSRECORD_Block *block;
 589   struct GNUNET_HashCode query;
 590   struct GNUNET_TIME_Absolute expire;
 591   size_t block_size;
 592   struct GNUNET_DHT_PutHandle *ret;
 593
 594   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 595                                                         rd_public);
 596   if (cache_keys)
 597     block = GNUNET_GNSRECORD_block_create2 (key,
 598                                             expire,
 599                                             label,
 600                                             rd_public,
 601                                             rd_public_count);
 602   else
 603     block = GNUNET_GNSRECORD_block_create (key,
 604                                            expire,
 605                                            label,
 606                                            rd_public,
 607                                            rd_public_count);
 608   if (NULL == block)
 609   {
 610     GNUNET_break (0);
 611     return NULL; /* whoops */
 612   }
 613   block_size = ntohl (block->purpose.size)
 614     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 615     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 616   GNUNET_GNSRECORD_query_from_private_key (key,
 617                                            label,
 618                                            &query);
 619   GNUNET_STATISTICS_update (statistics,
 620                             "DHT put operations initiated",
 621                             1,
 622                             GNUNET_NO);
 623   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 624               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 625               rd_public_count,
 626               label,
 627               GNUNET_STRINGS_absolute_time_to_string (expire),
 628               GNUNET_h2s (&query));
 629   num_public_records++;
 630   ret = GNUNET_DHT_put (dht_handle,
 631                         &query,
 632                         DHT_GNS_REPLICATION_LEVEL,
 633                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 634                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 635                         block_size,
 636                         block,
 637                         expire,
 638                         &dht_put_continuation,
 639                         ma);
 640   GNUNET_free (block);
 641   return ret;
 642 }
 643
 644
 645 /**
 646  * We encountered an error in our zone iteration.
 647  *
 648  * @param cls NULL
 649  */
 650 static void
 651 zone_iteration_error (void *cls)
 652 {
 653   (void) cls;
 654   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 655               "Got disconnected from namestore database, retrying.\n");
 656   namestore_iter = NULL;
 657   /* We end up here on error/disconnect/shutdown, so potentially
 658      while a zone publish task or a DHT put is still running; hence
 659      we need to cancel those. */
 660   if (NULL != zone_publish_task)
 661   {
 662     GNUNET_SCHEDULER_cancel (zone_publish_task);
 663     zone_publish_task = NULL;
 664   }
 665   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 666                                                 NULL);
 667 }
 668
 669
 670 /**
 671  * Zone iteration is completed.
 672  *
 673  * @param cls NULL
 674  */
 675 static void
 676 zone_iteration_finished (void *cls)
 677 {
 678   (void) cls;
 679   /* we're done with one iteration, calculate when to do the next one */
 680   namestore_iter = NULL;
 681   last_num_public_records = num_public_records;
 682   first_zone_iteration = GNUNET_NO;
 683   last_min_relative_record_time = min_relative_record_time;
 684   calculate_put_interval ();
 685   /* reset for next iteration */
 686   min_relative_record_time
 687     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 688                                      PUBLISH_OPS_PER_EXPIRATION);
 689   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 690               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 691               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 692                                                       GNUNET_YES));
 693   GNUNET_STATISTICS_set (statistics,
 694                          "Target zone iteration velocity (μs)",
 695                          target_iteration_velocity_per_record.rel_value_us,
 696                          GNUNET_NO);
 697   GNUNET_STATISTICS_set (statistics,
 698                          "Number of public records in DHT",
 699                          last_num_public_records,
 700                          GNUNET_NO);
 701   GNUNET_assert (NULL == zone_publish_task);
 702   if (0 == last_num_public_records)
 703   {
 704     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 705                                                       &publish_zone_dht_start,
 706                                                       NULL);
 707   }
 708   else
 709   {
 710     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 711                                                   NULL);
 712   }
 713 }
 714
 715
 716 /**
 717  * Function used to put all records successively into the DHT.
 718  *
 719  * @param cls the closure (NULL)
 720  * @param key the private key of the authority (ours)
 721  * @param label the name of the records, NULL once the iteration is done
 722  * @param rd_count the number of records in @a rd
 723  * @param rd the record data
 724  */
 725 static void
 726 put_gns_record (void *cls,
 727                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 728                 const char *label,
 729                 unsigned int rd_count,
 730                 const struct GNUNET_GNSRECORD_Data *rd)
 731 {
 732   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 733   unsigned int rd_public_count;
 734   struct DhtPutActivity *ma;
 735
 736   (void) cls;
 737   ns_iteration_left--;
 738   rd_public_count = convert_records_for_export (rd,
 739                                                 rd_count,
 740                                                 rd_public);
 741   if (0 == rd_public_count)
 742   {
 743     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 744                 "Record set empty, moving to next record set\n");
 745     check_zone_namestore_next ();
 746     return;
 747   }
 748   /* We got a set of records to publish */
 749   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 750               "Starting DHT PUT\n");
 751   ma = GNUNET_new (struct DhtPutActivity);
 752   ma->start_date = GNUNET_TIME_absolute_get ();
 753   ma->ph = perform_dht_put (key,
 754                             label,
 755                             rd_public,
 756                             rd_public_count,
 757                             ma);
 758   put_cnt++;
 759   if (0 == put_cnt % DELTA_INTERVAL)
 760     update_velocity (DELTA_INTERVAL);
 761   check_zone_namestore_next ();
 762   if (NULL == ma->ph)
 763   {
 764     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 765                 "Could not perform DHT PUT, is the DHT running?\n");
 766     GNUNET_free (ma);
 767     return;
 768   }
 769   dht_queue_length++;
 770   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 771                                     it_tail,
 772                                     ma);
 773   if (dht_queue_length > DHT_QUEUE_LIMIT)
 774   {
 775     ma = it_head;
 776     GNUNET_CONTAINER_DLL_remove (it_head,
 777                                  it_tail,
 778                                  ma);
 779     GNUNET_DHT_put_cancel (ma->ph);
 780     dht_queue_length--;
 781     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 782                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 783                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 784                                                         GNUNET_YES));
 785     GNUNET_free (ma);
 786   }
 787 }
 788
 789
 790 /**
 791  * Periodically iterate over all zones and store everything in DHT
 792  *
 793  * @param cls NULL
 794  */
 795 static void
 796 publish_zone_dht_start (void *cls)
 797 {
 798   (void) cls;
 799   zone_publish_task = NULL;
 800   GNUNET_STATISTICS_update (statistics,
 801                             "Full zone iterations launched",
 802                             1,
 803                             GNUNET_NO);
 804   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 805               "Starting DHT zone update!\n");
 806   /* start counting again */
 807   num_public_records = 0;
 808   GNUNET_assert (NULL == namestore_iter);
 809   ns_iteration_left = 1;
 810   namestore_iter
 811     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 812                                              NULL, /* All zones */
 813                                              &zone_iteration_error,
 814                                              NULL,
 815                                              &put_gns_record,
 816                                              NULL,
 817                                              &zone_iteration_finished,
 818                                              NULL);
 819   GNUNET_assert (NULL != namestore_iter);
 820 }
 821
 822
 823 /**
 824  * Performe zonemaster duties: watch namestore, publish records.
 825  *
 826  * @param cls closure
 827  * @param server the initialized server
 828  * @param c configuration to use
 829  */
 830 static void
 831 run (void *cls,
 832      const struct GNUNET_CONFIGURATION_Handle *c,
 833      struct GNUNET_SERVICE_Handle *service)
 834 {
 835   unsigned long long max_parallel_bg_queries = 128;
 836
 837   (void) cls;
 838   (void) service;
 839   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 840   min_relative_record_time
 841     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 842                                      PUBLISH_OPS_PER_EXPIRATION);
 843   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 844   namestore_handle = GNUNET_NAMESTORE_connect (c);
 845   if (NULL == namestore_handle)
 846   {
 847     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 848                 _("Failed to connect to the namestore!\n"));
 849     GNUNET_SCHEDULER_shutdown ();
 850     return;
 851   }
 852   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 853                                                      "namestore",
 854                                                      "CACHE_KEYS");
 855   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
 856   if (GNUNET_OK ==
 857       GNUNET_CONFIGURATION_get_value_time (c,
 858                                            "zonemaster",
 859                                            "ZONE_PUBLISH_TIME_WINDOW",
 860                                            &zone_publish_time_window_default))
 861   {
 862     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 863                 "Time window for zone iteration: %s\n",
 864                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
 865                                                         GNUNET_YES));
 866   }
 867   zone_publish_time_window = zone_publish_time_window_default;
 868   if (GNUNET_OK ==
 869       GNUNET_CONFIGURATION_get_value_number (c,
 870                                              "zonemaster",
 871                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 872                                              &max_parallel_bg_queries))
 873   {
 874     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 875                 "Number of allowed parallel background queries: %llu\n",
 876                 max_parallel_bg_queries);
 877   }
 878   if (0 == max_parallel_bg_queries)
 879     max_parallel_bg_queries = 1;
 880   dht_handle = GNUNET_DHT_connect (c,
 881                                    (unsigned int) max_parallel_bg_queries);
 882   if (NULL == dht_handle)
 883   {
 884     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 885                 _("Could not connect to DHT!\n"));
 886     GNUNET_SCHEDULER_add_now (&shutdown_task,
 887                               NULL);
 888     return;
 889   }
 890
 891   /* Schedule periodic put for our records. */
 892   first_zone_iteration = GNUNET_YES;
 893   statistics = GNUNET_STATISTICS_create ("zonemaster",
 894                                          c);
 895   GNUNET_STATISTICS_set (statistics,
 896                          "Target zone iteration velocity (μs)",
 897                          target_iteration_velocity_per_record.rel_value_us,
 898                          GNUNET_NO);
 899   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 900                                                 NULL);
 901   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 902                                  NULL);
 903 }
 904
 905
 906 /**
 907  * Define "main" method using service macro.
 908  */
 909 GNUNET_SERVICE_MAIN
 910 ("zonemaster",
 911  GNUNET_SERVICE_OPTION_NONE,
 912  &run,
 913  NULL,
 914  NULL,
 915  NULL,
 916  GNUNET_MQ_handler_end());
 917
 918
 919 /* end of gnunet-service-zonemaster.c */