src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software: you can redistribute it and/or modify it
   6      under the terms of the GNU Affero General Public License as published
   7      by the Free Software Foundation, either version 3 of the License,
   8      or (at your option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      Affero General Public License for more details.
  14
  15      You should have received a copy of the GNU Affero General Public License
  16      along with this program.  If not, see <http://www.gnu.org/licenses/>.
  17 */
  18
  19 /**
  20  * @file zonemaster/gnunet-service-zonemaster.c
  21  * @brief publish records from namestore to GNUnet name system
  22  * @author Christian Grothoff
  23  */
  24 #include "platform.h"
  25 #include "gnunet_util_lib.h"
  26 #include "gnunet_dnsparser_lib.h"
  27 #include "gnunet_dht_service.h"
  28 #include "gnunet_namestore_service.h"
  29 #include "gnunet_statistics_service.h"
  30
  31
  32 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  33
  34
  35 /**
  36  * How often should we (re)publish each record before
  37  * it expires?
  38  */
  39 #define PUBLISH_OPS_PER_EXPIRATION 4
  40
  41 /**
  42  * How often do we measure the delta between desired zone
  43  * iteration speed and actual speed, and tell statistics
  44  * service about it?
  45  */
  46 #define DELTA_INTERVAL 100
  47
  48 /**
  49  * How many records do we fetch in one shot from the namestore?
  50  */
  51 #define NS_BLOCK_SIZE 1000
  52
  53 /**
  54  * How many pending DHT operations do we allow at most?
  55  */
  56 #define DHT_QUEUE_LIMIT 2000
  57
  58 /**
  59  * How many events may the namestore give us before it has to wait
  60  * for us to keep up?
  61  */
  62 #define NAMESTORE_QUEUE_LIMIT 50
  63
  64 /**
  65  * The initial interval in milliseconds btween puts in
  66  * a zone iteration
  67  */
  68 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  69
  70 /**
  71  * The upper bound for the zone iteration interval
  72  * (per record).
  73  */
  74 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  75
  76 /**
  77  * The factor the current zone iteration interval is divided by for each
  78  * additional new record
  79  */
  80 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  81
  82 /**
  83  * What replication level do we use for DHT PUT operations?
  84  */
  85 #define DHT_GNS_REPLICATION_LEVEL 5
  86
  87
  88 /**
  89  * Handle for DHT PUT activity triggered from the namestore monitor.
  90  */
  91 struct DhtPutActivity
  92 {
  93   /**
  94    * Kept in a DLL.
  95    */
  96   struct DhtPutActivity *next;
  97
  98   /**
  99    * Kept in a DLL.
 100    */
 101   struct DhtPutActivity *prev;
 102
 103   /**
 104    * Handle for the DHT PUT operation.
 105    */
 106   struct GNUNET_DHT_PutHandle *ph;
 107
 108   /**
 109    * When was this PUT initiated?
 110    */
 111   struct GNUNET_TIME_Absolute start_date;
 112 };
 113
 114
 115 /**
 116  * Handle to the statistics service
 117  */
 118 static struct GNUNET_STATISTICS_Handle *statistics;
 119
 120 /**
 121  * Our handle to the DHT
 122  */
 123 static struct GNUNET_DHT_Handle *dht_handle;
 124
 125 /**
 126  * Our handle to the namestore service
 127  */
 128 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 129
 130 /**
 131  * Handle to iterate over our authoritative zone in namestore
 132  */
 133 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 134
 135 /**
 136  * Head of iteration put activities; kept in a DLL.
 137  */
 138 static struct DhtPutActivity *it_head;
 139
 140 /**
 141  * Tail of iteration put activities; kept in a DLL.
 142  */
 143 static struct DhtPutActivity *it_tail;
 144
 145 /**
 146  * Number of entries in the DHT queue #it_head.
 147  */
 148 static unsigned int dht_queue_length;
 149
 150 /**
 151  * Useful for zone update for DHT put
 152  */
 153 static unsigned long long num_public_records;
 154
 155 /**
 156  * Last seen record count
 157  */
 158 static unsigned long long last_num_public_records;
 159
 160 /**
 161  * Number of successful put operations performed in the current
 162  * measurement cycle (as measured in #check_zone_namestore_next()).
 163  */
 164 static unsigned long long put_cnt;
 165
 166 /**
 167  * What is the frequency at which we currently would like
 168  * to perform DHT puts (per record)?  Calculated in
 169  * update_velocity() from the #zone_publish_time_window()
 170  * and the total number of record sets we have (so far)
 171  * observed in the zone.
 172  */
 173 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 174
 175 /**
 176  * Minimum relative expiration time of records seem during the current
 177  * zone iteration.
 178  */
 179 static struct GNUNET_TIME_Relative min_relative_record_time;
 180
 181 /**
 182  * Minimum relative expiration time of records seem during the last
 183  * zone iteration.
 184  */
 185 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 186
 187 /**
 188  * Default time window for zone iteration
 189  */
 190 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 191
 192 /**
 193  * Time window for zone iteration, adjusted based on relative record
 194  * expiration times in our zone.
 195  */
 196 static struct GNUNET_TIME_Relative zone_publish_time_window;
 197
 198 /**
 199  * When did we last start measuring the #DELTA_INTERVAL successful
 200  * DHT puts? Used for velocity calculations.
 201  */
 202 static struct GNUNET_TIME_Absolute last_put_100;
 203
 204 /**
 205  * By how much should we try to increase our per-record iteration speed
 206  * (over the desired speed calculated directly from the #put_interval)?
 207  * Basically this value corresponds to the per-record CPU time overhead
 208  * we have.
 209  */
 210 static struct GNUNET_TIME_Relative sub_delta;
 211
 212 /**
 213  * zone publish task
 214  */
 215 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 216
 217 /**
 218  * How many more values are left for the current query before we need
 219  * to explicitly ask the namestore for more?
 220  */
 221 static unsigned int ns_iteration_left;
 222
 223 /**
 224  * #GNUNET_YES if zone has never been published before
 225  */
 226 static int first_zone_iteration;
 227
 228 /**
 229  * Optimize block insertion by caching map of private keys to
 230  * public keys in memory?
 231  */
 232 static int cache_keys;
 233
 234
 235 /**
 236  * Task run during shutdown.
 237  *
 238  * @param cls unused
 239  * @param tc unused
 240  */
 241 static void
 242 shutdown_task (void *cls)
 243 {
 244   struct DhtPutActivity *ma;
 245
 246   (void) cls;
 247   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 248               "Shutting down!\n");
 249   while (NULL != (ma = it_head))
 250   {
 251     GNUNET_DHT_put_cancel (ma->ph);
 252     dht_queue_length--;
 253     GNUNET_CONTAINER_DLL_remove (it_head,
 254                                  it_tail,
 255                                  ma);
 256     dht_queue_length--;
 257     GNUNET_free (ma);
 258   }
 259   if (NULL != statistics)
 260   {
 261     GNUNET_STATISTICS_destroy (statistics,
 262                                GNUNET_NO);
 263     statistics = NULL;
 264   }
 265   if (NULL != zone_publish_task)
 266   {
 267     GNUNET_SCHEDULER_cancel (zone_publish_task);
 268     zone_publish_task = NULL;
 269   }
 270   if (NULL != namestore_iter)
 271   {
 272     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 273     namestore_iter = NULL;
 274   }
 275   if (NULL != namestore_handle)
 276   {
 277     GNUNET_NAMESTORE_disconnect (namestore_handle);
 278     namestore_handle = NULL;
 279   }
 280   if (NULL != dht_handle)
 281   {
 282     GNUNET_DHT_disconnect (dht_handle);
 283     dht_handle = NULL;
 284   }
 285 }
 286
 287
 288 /**
 289  * Method called periodically that triggers iteration over authoritative records
 290  *
 291  * @param cls NULL
 292  */
 293 static void
 294 publish_zone_namestore_next (void *cls)
 295 {
 296   (void) cls;
 297   zone_publish_task = NULL;
 298   GNUNET_assert (NULL != namestore_iter);
 299   GNUNET_assert (0 == ns_iteration_left);
 300   ns_iteration_left = NS_BLOCK_SIZE;
 301   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 302                                        NS_BLOCK_SIZE);
 303 }
 304
 305
 306 /**
 307  * Periodically iterate over our zone and store everything in dht
 308  *
 309  * @param cls NULL
 310  */
 311 static void
 312 publish_zone_dht_start (void *cls);
 313
 314
 315 /**
 316  * Calculate #target_iteration_velocity_per_record.
 317  */
 318 static void
 319 calculate_put_interval ()
 320 {
 321   if (0 == num_public_records)
 322   {
 323     /**
 324      * If no records are known (startup) or none present
 325      * we can safely set the interval to the value for a single
 326      * record
 327      */
 328     target_iteration_velocity_per_record = zone_publish_time_window;
 329     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 330                 "No records in namestore database.\n");
 331   }
 332   else
 333   {
 334     last_min_relative_record_time
 335       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 336                                   min_relative_record_time);
 337     zone_publish_time_window
 338       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 339                                                                PUBLISH_OPS_PER_EXPIRATION),
 340                                   zone_publish_time_window_default);
 341     target_iteration_velocity_per_record
 342       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 343                                      last_num_public_records);
 344   }
 345   target_iteration_velocity_per_record
 346     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 347                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 348   GNUNET_STATISTICS_set (statistics,
 349                          "Minimum relative record expiration (in μs)",
 350                          last_min_relative_record_time.rel_value_us,
 351                          GNUNET_NO);
 352   GNUNET_STATISTICS_set (statistics,
 353                          "Zone publication time window (in μs)",
 354                          zone_publish_time_window.rel_value_us,
 355                          GNUNET_NO);
 356   GNUNET_STATISTICS_set (statistics,
 357                          "Target zone iteration velocity (μs)",
 358                          target_iteration_velocity_per_record.rel_value_us,
 359                          GNUNET_NO);
 360 }
 361
 362
 363 /**
 364  * Re-calculate our velocity and the desired velocity.
 365  * We have succeeded in making #DELTA_INTERVAL puts, so
 366  * now calculate the new desired delay between puts.
 367  *
 368  * @param cnt how many records were processed since the last call?
 369  */
 370 static void
 371 update_velocity (unsigned int cnt)
 372 {
 373   struct GNUNET_TIME_Relative delta;
 374   unsigned long long pct = 0;
 375
 376   if (0 == cnt)
 377     return;
 378   /* How fast were we really? */
 379   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 380   delta.rel_value_us /= cnt;
 381   last_put_100 = GNUNET_TIME_absolute_get ();
 382
 383   /* calculate expected frequency */
 384   if ( (num_public_records > last_num_public_records) &&
 385        (GNUNET_NO == first_zone_iteration) )
 386   {
 387     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 388                 "Last record count was lower than current record count.  Reducing interval.\n");
 389     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 390     calculate_put_interval ();
 391   }
 392   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 393               "Desired global zone iteration interval is %s/record!\n",
 394               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 395                                                       GNUNET_YES));
 396
 397   /* Tell statistics actual vs. desired speed */
 398   GNUNET_STATISTICS_set (statistics,
 399                          "Current zone iteration velocity (μs/record)",
 400                          delta.rel_value_us,
 401                          GNUNET_NO);
 402   /* update "sub_delta" based on difference, taking
 403      previous sub_delta into account! */
 404   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 405   {
 406     /* We were too fast, reduce sub_delta! */
 407     struct GNUNET_TIME_Relative corr;
 408
 409     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 410                                           delta);
 411     if (sub_delta.rel_value_us > delta.rel_value_us)
 412     {
 413       /* Reduce sub_delta by corr */
 414       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 415                                                  corr);
 416     }
 417     else
 418     {
 419       /* We're doing fine with waiting the full time, this
 420          should theoretically only happen if we run at
 421          infinite speed. */
 422       sub_delta = GNUNET_TIME_UNIT_ZERO;
 423     }
 424   }
 425   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 426   {
 427     /* We were too slow, increase sub_delta! */
 428     struct GNUNET_TIME_Relative corr;
 429
 430     corr = GNUNET_TIME_relative_subtract (delta,
 431                                           target_iteration_velocity_per_record);
 432     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 433                                           corr);
 434     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 435     {
 436       /* CPU overload detected, we cannot go at desired speed,
 437          as this would mean using a negative delay. */
 438       /* compute how much faster we would want to be for
 439          the desired velocity */
 440       if (0 == target_iteration_velocity_per_record.rel_value_us)
 441         pct = UINT64_MAX; /* desired speed is infinity ... */
 442       else
 443         pct = (sub_delta.rel_value_us -
 444                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 445           / target_iteration_velocity_per_record.rel_value_us;
 446       sub_delta = target_iteration_velocity_per_record;
 447     }
 448   }
 449   GNUNET_STATISTICS_set (statistics,
 450                          "# size of the DHT queue (it)",
 451                          dht_queue_length,
 452                          GNUNET_NO);
 453   GNUNET_STATISTICS_set (statistics,
 454                          "% speed increase needed for target velocity",
 455                          pct,
 456                          GNUNET_NO);
 457   GNUNET_STATISTICS_set (statistics,
 458                          "# records processed in current iteration",
 459                          num_public_records,
 460                          GNUNET_NO);
 461 }
 462
 463
 464 /**
 465  * Check if the current zone iteration needs to be continued
 466  * by calling #publish_zone_namestore_next(), and if so with what delay.
 467  */
 468 static void
 469 check_zone_namestore_next ()
 470 {
 471   struct GNUNET_TIME_Relative delay;
 472
 473   if (0 != ns_iteration_left)
 474     return; /* current NAMESTORE iteration not yet done */
 475   update_velocity (put_cnt);
 476   put_cnt = 0;
 477   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 478                                          sub_delta);
 479   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 480      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 481   GNUNET_STATISTICS_set (statistics,
 482                          "Current artificial NAMESTORE delay (μs/record)",
 483                          delay.rel_value_us,
 484                          GNUNET_NO);
 485   delay = GNUNET_TIME_relative_multiply (delay,
 486                                          NS_BLOCK_SIZE);
 487   /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
 488   delay = GNUNET_TIME_relative_min (MAXIMUM_ZONE_ITERATION_INTERVAL,
 489                                     delay);
 490   /* no delays on first iteration */
 491   if (GNUNET_YES == first_zone_iteration)
 492     delay = GNUNET_TIME_UNIT_ZERO;
 493   GNUNET_assert (NULL == zone_publish_task);
 494   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 495                                                     &publish_zone_namestore_next,
 496                                                     NULL);
 497 }
 498
 499
 500 /**
 501  * Continuation called from DHT once the PUT operation is done.
 502  *
 503  * @param cls a `struct DhtPutActivity`
 504  */
 505 static void
 506 dht_put_continuation (void *cls)
 507 {
 508   struct DhtPutActivity *ma = cls;
 509
 510   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 511               "PUT complete\n");
 512   dht_queue_length--;
 513   GNUNET_CONTAINER_DLL_remove (it_head,
 514                                it_tail,
 515                                ma);
 516   GNUNET_free (ma);
 517 }
 518
 519
 520 /**
 521  * Convert namestore records from the internal format to that
 522  * suitable for publication (removes private records, converts
 523  * to absolute expiration time).
 524  *
 525  * @param rd input records
 526  * @param rd_count size of the @a rd and @a rd_public arrays
 527  * @param rd_public where to write the converted records
 528  * @return number of records written to @a rd_public
 529  */
 530 static unsigned int
 531 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 532                             unsigned int rd_count,
 533                             struct GNUNET_GNSRECORD_Data *rd_public)
 534 {
 535   struct GNUNET_TIME_Absolute now;
 536   unsigned int rd_public_count;
 537
 538   rd_public_count = 0;
 539   now = GNUNET_TIME_absolute_get ();
 540   for (unsigned int i=0;i<rd_count;i++)
 541   {
 542     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 543       continue;
 544     if ( (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) &&
 545          (rd[i].expiration_time < now.abs_value_us) )
 546       continue;  /* record already expired, skip it */
 547     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 548     {
 549       /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 550          we just need to adjust our iteration frequency */
 551       min_relative_record_time.rel_value_us =
 552         GNUNET_MIN (rd[i].expiration_time,
 553                     min_relative_record_time.rel_value_us);
 554     }
 555     rd_public[rd_public_count++] = rd[i];
 556   }
 557   return rd_public_count;
 558 }
 559
 560
 561 /**
 562  * Store GNS records in the DHT.
 563  *
 564  * @param key key of the zone
 565  * @param label label to store under
 566  * @param rd_public public record data
 567  * @param rd_public_count number of records in @a rd_public
 568  * @param ma handle for the put operation
 569  * @return DHT PUT handle, NULL on error
 570  */
 571 static struct GNUNET_DHT_PutHandle *
 572 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 573                  const char *label,
 574                  const struct GNUNET_GNSRECORD_Data *rd_public,
 575                  unsigned int rd_public_count,
 576                  struct DhtPutActivity *ma)
 577 {
 578   struct GNUNET_GNSRECORD_Block *block;
 579   struct GNUNET_HashCode query;
 580   struct GNUNET_TIME_Absolute expire;
 581   size_t block_size;
 582   struct GNUNET_DHT_PutHandle *ret;
 583
 584   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 585                                                         rd_public);
 586   if (cache_keys)
 587     block = GNUNET_GNSRECORD_block_create2 (key,
 588                                             expire,
 589                                             label,
 590                                             rd_public,
 591                                             rd_public_count);
 592   else
 593     block = GNUNET_GNSRECORD_block_create (key,
 594                                            expire,
 595                                            label,
 596                                            rd_public,
 597                                            rd_public_count);
 598   if (NULL == block)
 599   {
 600     GNUNET_break (0);
 601     return NULL; /* whoops */
 602   }
 603   block_size = ntohl (block->purpose.size)
 604     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 605     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 606   GNUNET_GNSRECORD_query_from_private_key (key,
 607                                            label,
 608                                            &query);
 609   GNUNET_STATISTICS_update (statistics,
 610                             "DHT put operations initiated",
 611                             1,
 612                             GNUNET_NO);
 613   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 614               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 615               rd_public_count,
 616               label,
 617               GNUNET_STRINGS_absolute_time_to_string (expire),
 618               GNUNET_h2s (&query));
 619   num_public_records++;
 620   ret = GNUNET_DHT_put (dht_handle,
 621                         &query,
 622                         DHT_GNS_REPLICATION_LEVEL,
 623                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 624                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 625                         block_size,
 626                         block,
 627                         expire,
 628                         &dht_put_continuation,
 629                         ma);
 630   GNUNET_free (block);
 631   return ret;
 632 }
 633
 634
 635 /**
 636  * We encountered an error in our zone iteration.
 637  *
 638  * @param cls NULL
 639  */
 640 static void
 641 zone_iteration_error (void *cls)
 642 {
 643   (void) cls;
 644   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 645               "Got disconnected from namestore database, retrying.\n");
 646   namestore_iter = NULL;
 647   /* We end up here on error/disconnect/shutdown, so potentially
 648      while a zone publish task or a DHT put is still running; hence
 649      we need to cancel those. */
 650   if (NULL != zone_publish_task)
 651   {
 652     GNUNET_SCHEDULER_cancel (zone_publish_task);
 653     zone_publish_task = NULL;
 654   }
 655   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 656                                                 NULL);
 657 }
 658
 659
 660 /**
 661  * Zone iteration is completed.
 662  *
 663  * @param cls NULL
 664  */
 665 static void
 666 zone_iteration_finished (void *cls)
 667 {
 668   (void) cls;
 669   /* we're done with one iteration, calculate when to do the next one */
 670   namestore_iter = NULL;
 671   last_num_public_records = num_public_records;
 672   first_zone_iteration = GNUNET_NO;
 673   last_min_relative_record_time = min_relative_record_time;
 674   calculate_put_interval ();
 675   /* reset for next iteration */
 676   min_relative_record_time
 677     = GNUNET_TIME_UNIT_FOREVER_REL;
 678   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 679               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 680               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 681                                                       GNUNET_YES));
 682   GNUNET_STATISTICS_set (statistics,
 683                          "Target zone iteration velocity (μs)",
 684                          target_iteration_velocity_per_record.rel_value_us,
 685                          GNUNET_NO);
 686   GNUNET_STATISTICS_set (statistics,
 687                          "Number of public records in DHT",
 688                          last_num_public_records,
 689                          GNUNET_NO);
 690   GNUNET_assert (NULL == zone_publish_task);
 691   if (0 == last_num_public_records)
 692   {
 693     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 694                                                       &publish_zone_dht_start,
 695                                                       NULL);
 696   }
 697   else
 698   {
 699     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 700                                                   NULL);
 701   }
 702 }
 703
 704
 705 /**
 706  * Function used to put all records successively into the DHT.
 707  *
 708  * @param cls the closure (NULL)
 709  * @param key the private key of the authority (ours)
 710  * @param label the name of the records, NULL once the iteration is done
 711  * @param rd_count the number of records in @a rd
 712  * @param rd the record data
 713  */
 714 static void
 715 put_gns_record (void *cls,
 716                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 717                 const char *label,
 718                 unsigned int rd_count,
 719                 const struct GNUNET_GNSRECORD_Data *rd)
 720 {
 721   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 722   unsigned int rd_public_count;
 723   struct DhtPutActivity *ma;
 724
 725   (void) cls;
 726   ns_iteration_left--;
 727   rd_public_count = convert_records_for_export (rd,
 728                                                 rd_count,
 729                                                 rd_public);
 730   if (0 == rd_public_count)
 731   {
 732     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 733                 "Record set empty, moving to next record set\n");
 734     check_zone_namestore_next ();
 735     return;
 736   }
 737   /* We got a set of records to publish */
 738   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 739               "Starting DHT PUT\n");
 740   ma = GNUNET_new (struct DhtPutActivity);
 741   ma->start_date = GNUNET_TIME_absolute_get ();
 742   ma->ph = perform_dht_put (key,
 743                             label,
 744                             rd_public,
 745                             rd_public_count,
 746                             ma);
 747   put_cnt++;
 748   if (0 == put_cnt % DELTA_INTERVAL)
 749     update_velocity (DELTA_INTERVAL);
 750   check_zone_namestore_next ();
 751   if (NULL == ma->ph)
 752   {
 753     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 754                 "Could not perform DHT PUT, is the DHT running?\n");
 755     GNUNET_free (ma);
 756     return;
 757   }
 758   dht_queue_length++;
 759   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 760                                     it_tail,
 761                                     ma);
 762   if (dht_queue_length > DHT_QUEUE_LIMIT)
 763   {
 764     ma = it_head;
 765     GNUNET_CONTAINER_DLL_remove (it_head,
 766                                  it_tail,
 767                                  ma);
 768     GNUNET_DHT_put_cancel (ma->ph);
 769     dht_queue_length--;
 770     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 771                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 772                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 773                                                         GNUNET_YES));
 774     GNUNET_free (ma);
 775   }
 776 }
 777
 778
 779 /**
 780  * Periodically iterate over all zones and store everything in DHT
 781  *
 782  * @param cls NULL
 783  */
 784 static void
 785 publish_zone_dht_start (void *cls)
 786 {
 787   (void) cls;
 788   zone_publish_task = NULL;
 789   GNUNET_STATISTICS_update (statistics,
 790                             "Full zone iterations launched",
 791                             1,
 792                             GNUNET_NO);
 793   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 794               "Starting DHT zone update!\n");
 795   /* start counting again */
 796   num_public_records = 0;
 797   GNUNET_assert (NULL == namestore_iter);
 798   ns_iteration_left = 1;
 799   namestore_iter
 800     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 801                                              NULL, /* All zones */
 802                                              &zone_iteration_error,
 803                                              NULL,
 804                                              &put_gns_record,
 805                                              NULL,
 806                                              &zone_iteration_finished,
 807                                              NULL);
 808   GNUNET_assert (NULL != namestore_iter);
 809 }
 810
 811
 812 /**
 813  * Performe zonemaster duties: watch namestore, publish records.
 814  *
 815  * @param cls closure
 816  * @param server the initialized server
 817  * @param c configuration to use
 818  */
 819 static void
 820 run (void *cls,
 821      const struct GNUNET_CONFIGURATION_Handle *c,
 822      struct GNUNET_SERVICE_Handle *service)
 823 {
 824   unsigned long long max_parallel_bg_queries = 128;
 825
 826   (void) cls;
 827   (void) service;
 828   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 829   min_relative_record_time
 830     = GNUNET_TIME_UNIT_FOREVER_REL;
 831   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 832   namestore_handle = GNUNET_NAMESTORE_connect (c);
 833   if (NULL == namestore_handle)
 834   {
 835     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 836                 _("Failed to connect to the namestore!\n"));
 837     GNUNET_SCHEDULER_shutdown ();
 838     return;
 839   }
 840   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 841                                                      "namestore",
 842                                                      "CACHE_KEYS");
 843   zone_publish_time_window_default = GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY;
 844   if (GNUNET_OK ==
 845       GNUNET_CONFIGURATION_get_value_time (c,
 846                                            "zonemaster",
 847                                            "ZONE_PUBLISH_TIME_WINDOW",
 848                                            &zone_publish_time_window_default))
 849   {
 850     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 851                 "Time window for zone iteration: %s\n",
 852                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
 853                                                         GNUNET_YES));
 854   }
 855   zone_publish_time_window = zone_publish_time_window_default;
 856   if (GNUNET_OK ==
 857       GNUNET_CONFIGURATION_get_value_number (c,
 858                                              "zonemaster",
 859                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 860                                              &max_parallel_bg_queries))
 861   {
 862     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 863                 "Number of allowed parallel background queries: %llu\n",
 864                 max_parallel_bg_queries);
 865   }
 866   if (0 == max_parallel_bg_queries)
 867     max_parallel_bg_queries = 1;
 868   dht_handle = GNUNET_DHT_connect (c,
 869                                    (unsigned int) max_parallel_bg_queries);
 870   if (NULL == dht_handle)
 871   {
 872     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 873                 _("Could not connect to DHT!\n"));
 874     GNUNET_SCHEDULER_add_now (&shutdown_task,
 875                               NULL);
 876     return;
 877   }
 878
 879   /* Schedule periodic put for our records. */
 880   first_zone_iteration = GNUNET_YES;
 881   statistics = GNUNET_STATISTICS_create ("zonemaster",
 882                                          c);
 883   GNUNET_STATISTICS_set (statistics,
 884                          "Target zone iteration velocity (μs)",
 885                          target_iteration_velocity_per_record.rel_value_us,
 886                          GNUNET_NO);
 887   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 888                                                 NULL);
 889   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 890                                  NULL);
 891 }
 892
 893
 894 /**
 895  * Define "main" method using service macro.
 896  */
 897 GNUNET_SERVICE_MAIN
 898 ("zonemaster",
 899  GNUNET_SERVICE_OPTION_NONE,
 900  &run,
 901  NULL,
 902  NULL,
 903  NULL,
 904  GNUNET_MQ_handler_end());
 905
 906
 907 /* end of gnunet-service-zonemaster.c */