src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32
  33
  34 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  35
  36
  37 /**
  38  * How often should we (re)publish each record before
  39  * it expires?
  40  */
  41 #define PUBLISH_OPS_PER_EXPIRATION 4
  42
  43 /**
  44  * How often do we measure the delta between desired zone
  45  * iteration speed and actual speed, and tell statistics
  46  * service about it?
  47  */
  48 #define DELTA_INTERVAL 100
  49
  50 /**
  51  * How many records do we fetch in one shot from the namestore?
  52  */
  53 #define NS_BLOCK_SIZE 1000
  54
  55 /**
  56  * How many pending DHT operations do we allow at most?
  57  */
  58 #define DHT_QUEUE_LIMIT 2000
  59
  60 /**
  61  * How many events may the namestore give us before it has to wait
  62  * for us to keep up?
  63  */
  64 #define NAMESTORE_QUEUE_LIMIT 50
  65
  66 /**
  67  * The initial interval in milliseconds btween puts in
  68  * a zone iteration
  69  */
  70 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  71
  72 /**
  73  * The upper bound for the zone iteration interval
  74  * (per record).
  75  */
  76 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  77
  78 /**
  79  * The factor the current zone iteration interval is divided by for each
  80  * additional new record
  81  */
  82 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  83
  84 /**
  85  * What replication level do we use for DHT PUT operations?
  86  */
  87 #define DHT_GNS_REPLICATION_LEVEL 5
  88
  89
  90 /**
  91  * Handle for DHT PUT activity triggered from the namestore monitor.
  92  */
  93 struct DhtPutActivity
  94 {
  95   /**
  96    * Kept in a DLL.
  97    */
  98   struct DhtPutActivity *next;
  99
 100   /**
 101    * Kept in a DLL.
 102    */
 103   struct DhtPutActivity *prev;
 104
 105   /**
 106    * Handle for the DHT PUT operation.
 107    */
 108   struct GNUNET_DHT_PutHandle *ph;
 109
 110   /**
 111    * When was this PUT initiated?
 112    */
 113   struct GNUNET_TIME_Absolute start_date;
 114 };
 115
 116
 117 /**
 118  * Handle to the statistics service
 119  */
 120 static struct GNUNET_STATISTICS_Handle *statistics;
 121
 122 /**
 123  * Our handle to the DHT
 124  */
 125 static struct GNUNET_DHT_Handle *dht_handle;
 126
 127 /**
 128  * Our handle to the namestore service
 129  */
 130 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 131
 132 /**
 133  * Handle to iterate over our authoritative zone in namestore
 134  */
 135 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 136
 137 /**
 138  * Head of iteration put activities; kept in a DLL.
 139  */
 140 static struct DhtPutActivity *it_head;
 141
 142 /**
 143  * Tail of iteration put activities; kept in a DLL.
 144  */
 145 static struct DhtPutActivity *it_tail;
 146
 147 /**
 148  * Number of entries in the DHT queue #it_head.
 149  */
 150 static unsigned int dht_queue_length;
 151
 152 /**
 153  * Useful for zone update for DHT put
 154  */
 155 static unsigned long long num_public_records;
 156
 157 /**
 158  * Last seen record count
 159  */
 160 static unsigned long long last_num_public_records;
 161
 162 /**
 163  * Number of successful put operations performed in the current
 164  * measurement cycle (as measured in #check_zone_namestore_next()).
 165  */
 166 static unsigned long long put_cnt;
 167
 168 /**
 169  * What is the frequency at which we currently would like
 170  * to perform DHT puts (per record)?  Calculated in
 171  * update_velocity() from the #zone_publish_time_window()
 172  * and the total number of record sets we have (so far)
 173  * observed in the zone.
 174  */
 175 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 176
 177 /**
 178  * Minimum relative expiration time of records seem during the current
 179  * zone iteration.
 180  */
 181 static struct GNUNET_TIME_Relative min_relative_record_time;
 182
 183 /**
 184  * Minimum relative expiration time of records seem during the last
 185  * zone iteration.
 186  */
 187 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 188
 189 /**
 190  * Default time window for zone iteration
 191  */
 192 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 193
 194 /**
 195  * Time window for zone iteration, adjusted based on relative record
 196  * expiration times in our zone.
 197  */
 198 static struct GNUNET_TIME_Relative zone_publish_time_window;
 199
 200 /**
 201  * When did we last start measuring the #DELTA_INTERVAL successful
 202  * DHT puts? Used for velocity calculations.
 203  */
 204 static struct GNUNET_TIME_Absolute last_put_100;
 205
 206 /**
 207  * By how much should we try to increase our per-record iteration speed
 208  * (over the desired speed calculated directly from the #put_interval)?
 209  * Basically this value corresponds to the per-record CPU time overhead
 210  * we have.
 211  */
 212 static struct GNUNET_TIME_Relative sub_delta;
 213
 214 /**
 215  * zone publish task
 216  */
 217 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 218
 219 /**
 220  * How many more values are left for the current query before we need
 221  * to explicitly ask the namestore for more?
 222  */
 223 static unsigned int ns_iteration_left;
 224
 225 /**
 226  * #GNUNET_YES if zone has never been published before
 227  */
 228 static int first_zone_iteration;
 229
 230 /**
 231  * Optimize block insertion by caching map of private keys to
 232  * public keys in memory?
 233  */
 234 static int cache_keys;
 235
 236
 237 /**
 238  * Task run during shutdown.
 239  *
 240  * @param cls unused
 241  * @param tc unused
 242  */
 243 static void
 244 shutdown_task (void *cls)
 245 {
 246   struct DhtPutActivity *ma;
 247
 248   (void) cls;
 249   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 250               "Shutting down!\n");
 251   while (NULL != (ma = it_head))
 252   {
 253     GNUNET_DHT_put_cancel (ma->ph);
 254     dht_queue_length--;
 255     GNUNET_CONTAINER_DLL_remove (it_head,
 256                                  it_tail,
 257                                  ma);
 258     dht_queue_length--;
 259     GNUNET_free (ma);
 260   }
 261   if (NULL != statistics)
 262   {
 263     GNUNET_STATISTICS_destroy (statistics,
 264                                GNUNET_NO);
 265     statistics = NULL;
 266   }
 267   if (NULL != zone_publish_task)
 268   {
 269     GNUNET_SCHEDULER_cancel (zone_publish_task);
 270     zone_publish_task = NULL;
 271   }
 272   if (NULL != namestore_iter)
 273   {
 274     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 275     namestore_iter = NULL;
 276   }
 277   if (NULL != namestore_handle)
 278   {
 279     GNUNET_NAMESTORE_disconnect (namestore_handle);
 280     namestore_handle = NULL;
 281   }
 282   if (NULL != dht_handle)
 283   {
 284     GNUNET_DHT_disconnect (dht_handle);
 285     dht_handle = NULL;
 286   }
 287 }
 288
 289
 290 /**
 291  * Method called periodically that triggers iteration over authoritative records
 292  *
 293  * @param cls NULL
 294  */
 295 static void
 296 publish_zone_namestore_next (void *cls)
 297 {
 298   (void) cls;
 299   zone_publish_task = NULL;
 300   GNUNET_assert (NULL != namestore_iter);
 301   GNUNET_assert (0 == ns_iteration_left);
 302   ns_iteration_left = NS_BLOCK_SIZE;
 303   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 304                                        NS_BLOCK_SIZE);
 305 }
 306
 307
 308 /**
 309  * Periodically iterate over our zone and store everything in dht
 310  *
 311  * @param cls NULL
 312  */
 313 static void
 314 publish_zone_dht_start (void *cls);
 315
 316
 317 /**
 318  * Calculate #target_iteration_velocity_per_record.
 319  */
 320 static void
 321 calculate_put_interval ()
 322 {
 323   if (0 == num_public_records)
 324   {
 325     /**
 326      * If no records are known (startup) or none present
 327      * we can safely set the interval to the value for a single
 328      * record
 329      */
 330     target_iteration_velocity_per_record = zone_publish_time_window;
 331     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 332                 "No records in namestore database.\n");
 333   }
 334   else
 335   {
 336     last_min_relative_record_time
 337       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 338                                   min_relative_record_time);
 339     zone_publish_time_window
 340       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 341                                                                PUBLISH_OPS_PER_EXPIRATION),
 342                                   zone_publish_time_window_default);
 343     target_iteration_velocity_per_record
 344       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 345                                      last_num_public_records);
 346   }
 347   target_iteration_velocity_per_record
 348     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 349                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 350   GNUNET_STATISTICS_set (statistics,
 351                          "Minimum relative record expiration (in μs)",
 352                          last_min_relative_record_time.rel_value_us,
 353                          GNUNET_NO);
 354   GNUNET_STATISTICS_set (statistics,
 355                          "Zone publication time window (in μs)",
 356                          zone_publish_time_window.rel_value_us,
 357                          GNUNET_NO);
 358   GNUNET_STATISTICS_set (statistics,
 359                          "Target zone iteration velocity (μs)",
 360                          target_iteration_velocity_per_record.rel_value_us,
 361                          GNUNET_NO);
 362 }
 363
 364
 365 /**
 366  * Re-calculate our velocity and the desired velocity.
 367  * We have succeeded in making #DELTA_INTERVAL puts, so
 368  * now calculate the new desired delay between puts.
 369  *
 370  * @param cnt how many records were processed since the last call?
 371  */
 372 static void
 373 update_velocity (unsigned int cnt)
 374 {
 375   struct GNUNET_TIME_Relative delta;
 376   unsigned long long pct = 0;
 377
 378   if (0 == cnt)
 379     return;
 380   /* How fast were we really? */
 381   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 382   delta.rel_value_us /= cnt;
 383   last_put_100 = GNUNET_TIME_absolute_get ();
 384
 385   /* calculate expected frequency */
 386   if ( (num_public_records > last_num_public_records) &&
 387        (GNUNET_NO == first_zone_iteration) )
 388   {
 389     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 390                 "Last record count was lower than current record count.  Reducing interval.\n");
 391     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 392     calculate_put_interval ();
 393   }
 394   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 395               "Desired global zone iteration interval is %s/record!\n",
 396               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 397                                                       GNUNET_YES));
 398
 399   /* Tell statistics actual vs. desired speed */
 400   GNUNET_STATISTICS_set (statistics,
 401                          "Current zone iteration velocity (μs/record)",
 402                          delta.rel_value_us,
 403                          GNUNET_NO);
 404   /* update "sub_delta" based on difference, taking
 405      previous sub_delta into account! */
 406   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 407   {
 408     /* We were too fast, reduce sub_delta! */
 409     struct GNUNET_TIME_Relative corr;
 410
 411     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 412                                           delta);
 413     if (sub_delta.rel_value_us > delta.rel_value_us)
 414     {
 415       /* Reduce sub_delta by corr */
 416       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 417                                                  corr);
 418     }
 419     else
 420     {
 421       /* We're doing fine with waiting the full time, this
 422          should theoretically only happen if we run at
 423          infinite speed. */
 424       sub_delta = GNUNET_TIME_UNIT_ZERO;
 425     }
 426   }
 427   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 428   {
 429     /* We were too slow, increase sub_delta! */
 430     struct GNUNET_TIME_Relative corr;
 431
 432     corr = GNUNET_TIME_relative_subtract (delta,
 433                                           target_iteration_velocity_per_record);
 434     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 435                                           corr);
 436     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 437     {
 438       /* CPU overload detected, we cannot go at desired speed,
 439          as this would mean using a negative delay. */
 440       /* compute how much faster we would want to be for
 441          the desired velocity */
 442       if (0 == target_iteration_velocity_per_record.rel_value_us)
 443         pct = UINT64_MAX; /* desired speed is infinity ... */
 444       else
 445         pct = (sub_delta.rel_value_us -
 446                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 447           / target_iteration_velocity_per_record.rel_value_us;
 448       sub_delta = target_iteration_velocity_per_record;
 449     }
 450   }
 451   GNUNET_STATISTICS_set (statistics,
 452                          "# size of the DHT queue (it)",
 453                          dht_queue_length,
 454                          GNUNET_NO);
 455   GNUNET_STATISTICS_set (statistics,
 456                          "% speed increase needed for target velocity",
 457                          pct,
 458                          GNUNET_NO);
 459   GNUNET_STATISTICS_set (statistics,
 460                          "# records processed in current iteration",
 461                          num_public_records,
 462                          GNUNET_NO);
 463 }
 464
 465
 466 /**
 467  * Check if the current zone iteration needs to be continued
 468  * by calling #publish_zone_namestore_next(), and if so with what delay.
 469  */
 470 static void
 471 check_zone_namestore_next ()
 472 {
 473   struct GNUNET_TIME_Relative delay;
 474
 475   if (0 != ns_iteration_left)
 476     return; /* current NAMESTORE iteration not yet done */
 477   update_velocity (put_cnt);
 478   put_cnt = 0;
 479   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 480                                          sub_delta);
 481   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 482      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 483   GNUNET_STATISTICS_set (statistics,
 484                          "Current artificial NAMESTORE delay (μs/record)",
 485                          delay.rel_value_us,
 486                          GNUNET_NO);
 487   delay = GNUNET_TIME_relative_multiply (delay,
 488                                          NS_BLOCK_SIZE);
 489   /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
 490   delay = GNUNET_TIME_relative_min (MAXIMUM_ZONE_ITERATION_INTERVAL,
 491                                     delay);
 492   /* no delays on first iteration */
 493   if (GNUNET_YES == first_zone_iteration)
 494     delay = GNUNET_TIME_UNIT_ZERO;
 495   GNUNET_assert (NULL == zone_publish_task);
 496   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 497                                                     &publish_zone_namestore_next,
 498                                                     NULL);
 499 }
 500
 501
 502 /**
 503  * Continuation called from DHT once the PUT operation is done.
 504  *
 505  * @param cls a `struct DhtPutActivity`
 506  */
 507 static void
 508 dht_put_continuation (void *cls)
 509 {
 510   struct DhtPutActivity *ma = cls;
 511
 512   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 513               "PUT complete\n");
 514   dht_queue_length--;
 515   GNUNET_CONTAINER_DLL_remove (it_head,
 516                                it_tail,
 517                                ma);
 518   GNUNET_free (ma);
 519 }
 520
 521
 522 /**
 523  * Convert namestore records from the internal format to that
 524  * suitable for publication (removes private records, converts
 525  * to absolute expiration time).
 526  *
 527  * @param rd input records
 528  * @param rd_count size of the @a rd and @a rd_public arrays
 529  * @param rd_public where to write the converted records
 530  * @return number of records written to @a rd_public
 531  */
 532 static unsigned int
 533 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 534                             unsigned int rd_count,
 535                             struct GNUNET_GNSRECORD_Data *rd_public)
 536 {
 537   struct GNUNET_TIME_Absolute now;
 538   unsigned int rd_public_count;
 539
 540   rd_public_count = 0;
 541   now = GNUNET_TIME_absolute_get ();
 542   for (unsigned int i=0;i<rd_count;i++)
 543   {
 544     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 545       continue;
 546     if ( (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) &&
 547          (rd[i].expiration_time < now.abs_value_us) )
 548       continue;  /* record already expired, skip it */
 549     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 550     {
 551       /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 552          we just need to adjust our iteration frequency */
 553       min_relative_record_time.rel_value_us =
 554         GNUNET_MIN (rd[i].expiration_time,
 555                     min_relative_record_time.rel_value_us);
 556     }
 557     rd_public[rd_public_count++] = rd[i];
 558   }
 559   return rd_public_count;
 560 }
 561
 562
 563 /**
 564  * Store GNS records in the DHT.
 565  *
 566  * @param key key of the zone
 567  * @param label label to store under
 568  * @param rd_public public record data
 569  * @param rd_public_count number of records in @a rd_public
 570  * @param ma handle for the put operation
 571  * @return DHT PUT handle, NULL on error
 572  */
 573 static struct GNUNET_DHT_PutHandle *
 574 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 575                  const char *label,
 576                  const struct GNUNET_GNSRECORD_Data *rd_public,
 577                  unsigned int rd_public_count,
 578                  struct DhtPutActivity *ma)
 579 {
 580   struct GNUNET_GNSRECORD_Block *block;
 581   struct GNUNET_HashCode query;
 582   struct GNUNET_TIME_Absolute expire;
 583   size_t block_size;
 584   struct GNUNET_DHT_PutHandle *ret;
 585
 586   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 587                                                         rd_public);
 588   if (cache_keys)
 589     block = GNUNET_GNSRECORD_block_create2 (key,
 590                                             expire,
 591                                             label,
 592                                             rd_public,
 593                                             rd_public_count);
 594   else
 595     block = GNUNET_GNSRECORD_block_create (key,
 596                                            expire,
 597                                            label,
 598                                            rd_public,
 599                                            rd_public_count);
 600   if (NULL == block)
 601   {
 602     GNUNET_break (0);
 603     return NULL; /* whoops */
 604   }
 605   block_size = ntohl (block->purpose.size)
 606     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 607     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 608   GNUNET_GNSRECORD_query_from_private_key (key,
 609                                            label,
 610                                            &query);
 611   GNUNET_STATISTICS_update (statistics,
 612                             "DHT put operations initiated",
 613                             1,
 614                             GNUNET_NO);
 615   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 616               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 617               rd_public_count,
 618               label,
 619               GNUNET_STRINGS_absolute_time_to_string (expire),
 620               GNUNET_h2s (&query));
 621   num_public_records++;
 622   ret = GNUNET_DHT_put (dht_handle,
 623                         &query,
 624                         DHT_GNS_REPLICATION_LEVEL,
 625                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 626                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 627                         block_size,
 628                         block,
 629                         expire,
 630                         &dht_put_continuation,
 631                         ma);
 632   GNUNET_free (block);
 633   return ret;
 634 }
 635
 636
 637 /**
 638  * We encountered an error in our zone iteration.
 639  *
 640  * @param cls NULL
 641  */
 642 static void
 643 zone_iteration_error (void *cls)
 644 {
 645   (void) cls;
 646   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 647               "Got disconnected from namestore database, retrying.\n");
 648   namestore_iter = NULL;
 649   /* We end up here on error/disconnect/shutdown, so potentially
 650      while a zone publish task or a DHT put is still running; hence
 651      we need to cancel those. */
 652   if (NULL != zone_publish_task)
 653   {
 654     GNUNET_SCHEDULER_cancel (zone_publish_task);
 655     zone_publish_task = NULL;
 656   }
 657   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 658                                                 NULL);
 659 }
 660
 661
 662 /**
 663  * Zone iteration is completed.
 664  *
 665  * @param cls NULL
 666  */
 667 static void
 668 zone_iteration_finished (void *cls)
 669 {
 670   (void) cls;
 671   /* we're done with one iteration, calculate when to do the next one */
 672   namestore_iter = NULL;
 673   last_num_public_records = num_public_records;
 674   first_zone_iteration = GNUNET_NO;
 675   last_min_relative_record_time = min_relative_record_time;
 676   calculate_put_interval ();
 677   /* reset for next iteration */
 678   min_relative_record_time
 679     = GNUNET_TIME_UNIT_FOREVER_REL;
 680   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 681               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 682               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 683                                                       GNUNET_YES));
 684   GNUNET_STATISTICS_set (statistics,
 685                          "Target zone iteration velocity (μs)",
 686                          target_iteration_velocity_per_record.rel_value_us,
 687                          GNUNET_NO);
 688   GNUNET_STATISTICS_set (statistics,
 689                          "Number of public records in DHT",
 690                          last_num_public_records,
 691                          GNUNET_NO);
 692   GNUNET_assert (NULL == zone_publish_task);
 693   if (0 == last_num_public_records)
 694   {
 695     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 696                                                       &publish_zone_dht_start,
 697                                                       NULL);
 698   }
 699   else
 700   {
 701     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 702                                                   NULL);
 703   }
 704 }
 705
 706
 707 /**
 708  * Function used to put all records successively into the DHT.
 709  *
 710  * @param cls the closure (NULL)
 711  * @param key the private key of the authority (ours)
 712  * @param label the name of the records, NULL once the iteration is done
 713  * @param rd_count the number of records in @a rd
 714  * @param rd the record data
 715  */
 716 static void
 717 put_gns_record (void *cls,
 718                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 719                 const char *label,
 720                 unsigned int rd_count,
 721                 const struct GNUNET_GNSRECORD_Data *rd)
 722 {
 723   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 724   unsigned int rd_public_count;
 725   struct DhtPutActivity *ma;
 726
 727   (void) cls;
 728   ns_iteration_left--;
 729   rd_public_count = convert_records_for_export (rd,
 730                                                 rd_count,
 731                                                 rd_public);
 732   if (0 == rd_public_count)
 733   {
 734     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 735                 "Record set empty, moving to next record set\n");
 736     check_zone_namestore_next ();
 737     return;
 738   }
 739   /* We got a set of records to publish */
 740   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 741               "Starting DHT PUT\n");
 742   ma = GNUNET_new (struct DhtPutActivity);
 743   ma->start_date = GNUNET_TIME_absolute_get ();
 744   ma->ph = perform_dht_put (key,
 745                             label,
 746                             rd_public,
 747                             rd_public_count,
 748                             ma);
 749   put_cnt++;
 750   if (0 == put_cnt % DELTA_INTERVAL)
 751     update_velocity (DELTA_INTERVAL);
 752   check_zone_namestore_next ();
 753   if (NULL == ma->ph)
 754   {
 755     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 756                 "Could not perform DHT PUT, is the DHT running?\n");
 757     GNUNET_free (ma);
 758     return;
 759   }
 760   dht_queue_length++;
 761   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 762                                     it_tail,
 763                                     ma);
 764   if (dht_queue_length > DHT_QUEUE_LIMIT)
 765   {
 766     ma = it_head;
 767     GNUNET_CONTAINER_DLL_remove (it_head,
 768                                  it_tail,
 769                                  ma);
 770     GNUNET_DHT_put_cancel (ma->ph);
 771     dht_queue_length--;
 772     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 773                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 774                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 775                                                         GNUNET_YES));
 776     GNUNET_free (ma);
 777   }
 778 }
 779
 780
 781 /**
 782  * Periodically iterate over all zones and store everything in DHT
 783  *
 784  * @param cls NULL
 785  */
 786 static void
 787 publish_zone_dht_start (void *cls)
 788 {
 789   (void) cls;
 790   zone_publish_task = NULL;
 791   GNUNET_STATISTICS_update (statistics,
 792                             "Full zone iterations launched",
 793                             1,
 794                             GNUNET_NO);
 795   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 796               "Starting DHT zone update!\n");
 797   /* start counting again */
 798   num_public_records = 0;
 799   GNUNET_assert (NULL == namestore_iter);
 800   ns_iteration_left = 1;
 801   namestore_iter
 802     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 803                                              NULL, /* All zones */
 804                                              &zone_iteration_error,
 805                                              NULL,
 806                                              &put_gns_record,
 807                                              NULL,
 808                                              &zone_iteration_finished,
 809                                              NULL);
 810   GNUNET_assert (NULL != namestore_iter);
 811 }
 812
 813
 814 /**
 815  * Performe zonemaster duties: watch namestore, publish records.
 816  *
 817  * @param cls closure
 818  * @param server the initialized server
 819  * @param c configuration to use
 820  */
 821 static void
 822 run (void *cls,
 823      const struct GNUNET_CONFIGURATION_Handle *c,
 824      struct GNUNET_SERVICE_Handle *service)
 825 {
 826   unsigned long long max_parallel_bg_queries = 128;
 827
 828   (void) cls;
 829   (void) service;
 830   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 831   min_relative_record_time
 832     = GNUNET_TIME_UNIT_FOREVER_REL;
 833   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 834   namestore_handle = GNUNET_NAMESTORE_connect (c);
 835   if (NULL == namestore_handle)
 836   {
 837     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 838                 _("Failed to connect to the namestore!\n"));
 839     GNUNET_SCHEDULER_shutdown ();
 840     return;
 841   }
 842   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 843                                                      "namestore",
 844                                                      "CACHE_KEYS");
 845   zone_publish_time_window_default = GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY;
 846   if (GNUNET_OK ==
 847       GNUNET_CONFIGURATION_get_value_time (c,
 848                                            "zonemaster",
 849                                            "ZONE_PUBLISH_TIME_WINDOW",
 850                                            &zone_publish_time_window_default))
 851   {
 852     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 853                 "Time window for zone iteration: %s\n",
 854                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
 855                                                         GNUNET_YES));
 856   }
 857   zone_publish_time_window = zone_publish_time_window_default;
 858   if (GNUNET_OK ==
 859       GNUNET_CONFIGURATION_get_value_number (c,
 860                                              "zonemaster",
 861                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 862                                              &max_parallel_bg_queries))
 863   {
 864     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 865                 "Number of allowed parallel background queries: %llu\n",
 866                 max_parallel_bg_queries);
 867   }
 868   if (0 == max_parallel_bg_queries)
 869     max_parallel_bg_queries = 1;
 870   dht_handle = GNUNET_DHT_connect (c,
 871                                    (unsigned int) max_parallel_bg_queries);
 872   if (NULL == dht_handle)
 873   {
 874     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 875                 _("Could not connect to DHT!\n"));
 876     GNUNET_SCHEDULER_add_now (&shutdown_task,
 877                               NULL);
 878     return;
 879   }
 880
 881   /* Schedule periodic put for our records. */
 882   first_zone_iteration = GNUNET_YES;
 883   statistics = GNUNET_STATISTICS_create ("zonemaster",
 884                                          c);
 885   GNUNET_STATISTICS_set (statistics,
 886                          "Target zone iteration velocity (μs)",
 887                          target_iteration_velocity_per_record.rel_value_us,
 888                          GNUNET_NO);
 889   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 890                                                 NULL);
 891   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 892                                  NULL);
 893 }
 894
 895
 896 /**
 897  * Define "main" method using service macro.
 898  */
 899 GNUNET_SERVICE_MAIN
 900 ("zonemaster",
 901  GNUNET_SERVICE_OPTION_NONE,
 902  &run,
 903  NULL,
 904  NULL,
 905  NULL,
 906  GNUNET_MQ_handler_end());
 907
 908
 909 /* end of gnunet-service-zonemaster.c */