src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software: you can redistribute it and/or modify it
   6      under the terms of the GNU Affero General Public License as published
   7      by the Free Software Foundation, either version 3 of the License,
   8      or (at your option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      Affero General Public License for more details.
  14 */
  15
  16 /**
  17  * @file zonemaster/gnunet-service-zonemaster.c
  18  * @brief publish records from namestore to GNUnet name system
  19  * @author Christian Grothoff
  20  */
  21 #include "platform.h"
  22 #include "gnunet_util_lib.h"
  23 #include "gnunet_dnsparser_lib.h"
  24 #include "gnunet_dht_service.h"
  25 #include "gnunet_namestore_service.h"
  26 #include "gnunet_statistics_service.h"
  27
  28
  29 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  30
  31
  32 /**
  33  * How often should we (re)publish each record before
  34  * it expires?
  35  */
  36 #define PUBLISH_OPS_PER_EXPIRATION 4
  37
  38 /**
  39  * How often do we measure the delta between desired zone
  40  * iteration speed and actual speed, and tell statistics
  41  * service about it?
  42  */
  43 #define DELTA_INTERVAL 100
  44
  45 /**
  46  * How many records do we fetch in one shot from the namestore?
  47  */
  48 #define NS_BLOCK_SIZE 1000
  49
  50 /**
  51  * How many pending DHT operations do we allow at most?
  52  */
  53 #define DHT_QUEUE_LIMIT 2000
  54
  55 /**
  56  * How many events may the namestore give us before it has to wait
  57  * for us to keep up?
  58  */
  59 #define NAMESTORE_QUEUE_LIMIT 50
  60
  61 /**
  62  * The initial interval in milliseconds btween puts in
  63  * a zone iteration
  64  */
  65 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  66
  67 /**
  68  * The upper bound for the zone iteration interval
  69  * (per record).
  70  */
  71 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  72
  73 /**
  74  * The factor the current zone iteration interval is divided by for each
  75  * additional new record
  76  */
  77 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  78
  79 /**
  80  * What replication level do we use for DHT PUT operations?
  81  */
  82 #define DHT_GNS_REPLICATION_LEVEL 5
  83
  84
  85 /**
  86  * Handle for DHT PUT activity triggered from the namestore monitor.
  87  */
  88 struct DhtPutActivity
  89 {
  90   /**
  91    * Kept in a DLL.
  92    */
  93   struct DhtPutActivity *next;
  94
  95   /**
  96    * Kept in a DLL.
  97    */
  98   struct DhtPutActivity *prev;
  99
 100   /**
 101    * Handle for the DHT PUT operation.
 102    */
 103   struct GNUNET_DHT_PutHandle *ph;
 104
 105   /**
 106    * When was this PUT initiated?
 107    */
 108   struct GNUNET_TIME_Absolute start_date;
 109 };
 110
 111
 112 /**
 113  * Handle to the statistics service
 114  */
 115 static struct GNUNET_STATISTICS_Handle *statistics;
 116
 117 /**
 118  * Our handle to the DHT
 119  */
 120 static struct GNUNET_DHT_Handle *dht_handle;
 121
 122 /**
 123  * Our handle to the namestore service
 124  */
 125 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 126
 127 /**
 128  * Handle to iterate over our authoritative zone in namestore
 129  */
 130 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 131
 132 /**
 133  * Head of iteration put activities; kept in a DLL.
 134  */
 135 static struct DhtPutActivity *it_head;
 136
 137 /**
 138  * Tail of iteration put activities; kept in a DLL.
 139  */
 140 static struct DhtPutActivity *it_tail;
 141
 142 /**
 143  * Number of entries in the DHT queue #it_head.
 144  */
 145 static unsigned int dht_queue_length;
 146
 147 /**
 148  * Useful for zone update for DHT put
 149  */
 150 static unsigned long long num_public_records;
 151
 152 /**
 153  * Last seen record count
 154  */
 155 static unsigned long long last_num_public_records;
 156
 157 /**
 158  * Number of successful put operations performed in the current
 159  * measurement cycle (as measured in #check_zone_namestore_next()).
 160  */
 161 static unsigned long long put_cnt;
 162
 163 /**
 164  * What is the frequency at which we currently would like
 165  * to perform DHT puts (per record)?  Calculated in
 166  * update_velocity() from the #zone_publish_time_window()
 167  * and the total number of record sets we have (so far)
 168  * observed in the zone.
 169  */
 170 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 171
 172 /**
 173  * Minimum relative expiration time of records seem during the current
 174  * zone iteration.
 175  */
 176 static struct GNUNET_TIME_Relative min_relative_record_time;
 177
 178 /**
 179  * Minimum relative expiration time of records seem during the last
 180  * zone iteration.
 181  */
 182 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 183
 184 /**
 185  * Default time window for zone iteration
 186  */
 187 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 188
 189 /**
 190  * Time window for zone iteration, adjusted based on relative record
 191  * expiration times in our zone.
 192  */
 193 static struct GNUNET_TIME_Relative zone_publish_time_window;
 194
 195 /**
 196  * When did we last start measuring the #DELTA_INTERVAL successful
 197  * DHT puts? Used for velocity calculations.
 198  */
 199 static struct GNUNET_TIME_Absolute last_put_100;
 200
 201 /**
 202  * By how much should we try to increase our per-record iteration speed
 203  * (over the desired speed calculated directly from the #put_interval)?
 204  * Basically this value corresponds to the per-record CPU time overhead
 205  * we have.
 206  */
 207 static struct GNUNET_TIME_Relative sub_delta;
 208
 209 /**
 210  * zone publish task
 211  */
 212 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 213
 214 /**
 215  * How many more values are left for the current query before we need
 216  * to explicitly ask the namestore for more?
 217  */
 218 static unsigned int ns_iteration_left;
 219
 220 /**
 221  * #GNUNET_YES if zone has never been published before
 222  */
 223 static int first_zone_iteration;
 224
 225 /**
 226  * Optimize block insertion by caching map of private keys to
 227  * public keys in memory?
 228  */
 229 static int cache_keys;
 230
 231
 232 /**
 233  * Task run during shutdown.
 234  *
 235  * @param cls unused
 236  * @param tc unused
 237  */
 238 static void
 239 shutdown_task (void *cls)
 240 {
 241   struct DhtPutActivity *ma;
 242
 243   (void) cls;
 244   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 245               "Shutting down!\n");
 246   while (NULL != (ma = it_head))
 247   {
 248     GNUNET_DHT_put_cancel (ma->ph);
 249     dht_queue_length--;
 250     GNUNET_CONTAINER_DLL_remove (it_head,
 251                                  it_tail,
 252                                  ma);
 253     dht_queue_length--;
 254     GNUNET_free (ma);
 255   }
 256   if (NULL != statistics)
 257   {
 258     GNUNET_STATISTICS_destroy (statistics,
 259                                GNUNET_NO);
 260     statistics = NULL;
 261   }
 262   if (NULL != zone_publish_task)
 263   {
 264     GNUNET_SCHEDULER_cancel (zone_publish_task);
 265     zone_publish_task = NULL;
 266   }
 267   if (NULL != namestore_iter)
 268   {
 269     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 270     namestore_iter = NULL;
 271   }
 272   if (NULL != namestore_handle)
 273   {
 274     GNUNET_NAMESTORE_disconnect (namestore_handle);
 275     namestore_handle = NULL;
 276   }
 277   if (NULL != dht_handle)
 278   {
 279     GNUNET_DHT_disconnect (dht_handle);
 280     dht_handle = NULL;
 281   }
 282 }
 283
 284
 285 /**
 286  * Method called periodically that triggers iteration over authoritative records
 287  *
 288  * @param cls NULL
 289  */
 290 static void
 291 publish_zone_namestore_next (void *cls)
 292 {
 293   (void) cls;
 294   zone_publish_task = NULL;
 295   GNUNET_assert (NULL != namestore_iter);
 296   GNUNET_assert (0 == ns_iteration_left);
 297   ns_iteration_left = NS_BLOCK_SIZE;
 298   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 299                                        NS_BLOCK_SIZE);
 300 }
 301
 302
 303 /**
 304  * Periodically iterate over our zone and store everything in dht
 305  *
 306  * @param cls NULL
 307  */
 308 static void
 309 publish_zone_dht_start (void *cls);
 310
 311
 312 /**
 313  * Calculate #target_iteration_velocity_per_record.
 314  */
 315 static void
 316 calculate_put_interval ()
 317 {
 318   if (0 == num_public_records)
 319   {
 320     /**
 321      * If no records are known (startup) or none present
 322      * we can safely set the interval to the value for a single
 323      * record
 324      */
 325     target_iteration_velocity_per_record = zone_publish_time_window;
 326     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 327                 "No records in namestore database.\n");
 328   }
 329   else
 330   {
 331     last_min_relative_record_time
 332       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 333                                   min_relative_record_time);
 334     zone_publish_time_window
 335       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 336                                                                PUBLISH_OPS_PER_EXPIRATION),
 337                                   zone_publish_time_window_default);
 338     target_iteration_velocity_per_record
 339       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 340                                      last_num_public_records);
 341   }
 342   target_iteration_velocity_per_record
 343     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 344                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 345   GNUNET_STATISTICS_set (statistics,
 346                          "Minimum relative record expiration (in μs)",
 347                          last_min_relative_record_time.rel_value_us,
 348                          GNUNET_NO);
 349   GNUNET_STATISTICS_set (statistics,
 350                          "Zone publication time window (in μs)",
 351                          zone_publish_time_window.rel_value_us,
 352                          GNUNET_NO);
 353   GNUNET_STATISTICS_set (statistics,
 354                          "Target zone iteration velocity (μs)",
 355                          target_iteration_velocity_per_record.rel_value_us,
 356                          GNUNET_NO);
 357 }
 358
 359
 360 /**
 361  * Re-calculate our velocity and the desired velocity.
 362  * We have succeeded in making #DELTA_INTERVAL puts, so
 363  * now calculate the new desired delay between puts.
 364  *
 365  * @param cnt how many records were processed since the last call?
 366  */
 367 static void
 368 update_velocity (unsigned int cnt)
 369 {
 370   struct GNUNET_TIME_Relative delta;
 371   unsigned long long pct = 0;
 372
 373   if (0 == cnt)
 374     return;
 375   /* How fast were we really? */
 376   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 377   delta.rel_value_us /= cnt;
 378   last_put_100 = GNUNET_TIME_absolute_get ();
 379
 380   /* calculate expected frequency */
 381   if ( (num_public_records > last_num_public_records) &&
 382        (GNUNET_NO == first_zone_iteration) )
 383   {
 384     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 385                 "Last record count was lower than current record count.  Reducing interval.\n");
 386     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 387     calculate_put_interval ();
 388   }
 389   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 390               "Desired global zone iteration interval is %s/record!\n",
 391               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 392                                                       GNUNET_YES));
 393
 394   /* Tell statistics actual vs. desired speed */
 395   GNUNET_STATISTICS_set (statistics,
 396                          "Current zone iteration velocity (μs/record)",
 397                          delta.rel_value_us,
 398                          GNUNET_NO);
 399   /* update "sub_delta" based on difference, taking
 400      previous sub_delta into account! */
 401   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 402   {
 403     /* We were too fast, reduce sub_delta! */
 404     struct GNUNET_TIME_Relative corr;
 405
 406     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 407                                           delta);
 408     if (sub_delta.rel_value_us > delta.rel_value_us)
 409     {
 410       /* Reduce sub_delta by corr */
 411       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 412                                                  corr);
 413     }
 414     else
 415     {
 416       /* We're doing fine with waiting the full time, this
 417          should theoretically only happen if we run at
 418          infinite speed. */
 419       sub_delta = GNUNET_TIME_UNIT_ZERO;
 420     }
 421   }
 422   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 423   {
 424     /* We were too slow, increase sub_delta! */
 425     struct GNUNET_TIME_Relative corr;
 426
 427     corr = GNUNET_TIME_relative_subtract (delta,
 428                                           target_iteration_velocity_per_record);
 429     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 430                                           corr);
 431     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 432     {
 433       /* CPU overload detected, we cannot go at desired speed,
 434          as this would mean using a negative delay. */
 435       /* compute how much faster we would want to be for
 436          the desired velocity */
 437       if (0 == target_iteration_velocity_per_record.rel_value_us)
 438         pct = UINT64_MAX; /* desired speed is infinity ... */
 439       else
 440         pct = (sub_delta.rel_value_us -
 441                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 442           / target_iteration_velocity_per_record.rel_value_us;
 443       sub_delta = target_iteration_velocity_per_record;
 444     }
 445   }
 446   GNUNET_STATISTICS_set (statistics,
 447                          "# size of the DHT queue (it)",
 448                          dht_queue_length,
 449                          GNUNET_NO);
 450   GNUNET_STATISTICS_set (statistics,
 451                          "% speed increase needed for target velocity",
 452                          pct,
 453                          GNUNET_NO);
 454   GNUNET_STATISTICS_set (statistics,
 455                          "# records processed in current iteration",
 456                          num_public_records,
 457                          GNUNET_NO);
 458 }
 459
 460
 461 /**
 462  * Check if the current zone iteration needs to be continued
 463  * by calling #publish_zone_namestore_next(), and if so with what delay.
 464  */
 465 static void
 466 check_zone_namestore_next ()
 467 {
 468   struct GNUNET_TIME_Relative delay;
 469
 470   if (0 != ns_iteration_left)
 471     return; /* current NAMESTORE iteration not yet done */
 472   update_velocity (put_cnt);
 473   put_cnt = 0;
 474   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 475                                          sub_delta);
 476   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 477      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 478   GNUNET_STATISTICS_set (statistics,
 479                          "Current artificial NAMESTORE delay (μs/record)",
 480                          delay.rel_value_us,
 481                          GNUNET_NO);
 482   delay = GNUNET_TIME_relative_multiply (delay,
 483                                          NS_BLOCK_SIZE);
 484   /* make sure we do not overshoot because of the #NS_BLOCK_SIZE factor */
 485   delay = GNUNET_TIME_relative_min (MAXIMUM_ZONE_ITERATION_INTERVAL,
 486                                     delay);
 487   /* no delays on first iteration */
 488   if (GNUNET_YES == first_zone_iteration)
 489     delay = GNUNET_TIME_UNIT_ZERO;
 490   GNUNET_assert (NULL == zone_publish_task);
 491   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 492                                                     &publish_zone_namestore_next,
 493                                                     NULL);
 494 }
 495
 496
 497 /**
 498  * Continuation called from DHT once the PUT operation is done.
 499  *
 500  * @param cls a `struct DhtPutActivity`
 501  */
 502 static void
 503 dht_put_continuation (void *cls)
 504 {
 505   struct DhtPutActivity *ma = cls;
 506
 507   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 508               "PUT complete\n");
 509   dht_queue_length--;
 510   GNUNET_CONTAINER_DLL_remove (it_head,
 511                                it_tail,
 512                                ma);
 513   GNUNET_free (ma);
 514 }
 515
 516
 517 /**
 518  * Convert namestore records from the internal format to that
 519  * suitable for publication (removes private records, converts
 520  * to absolute expiration time).
 521  *
 522  * @param rd input records
 523  * @param rd_count size of the @a rd and @a rd_public arrays
 524  * @param rd_public where to write the converted records
 525  * @return number of records written to @a rd_public
 526  */
 527 static unsigned int
 528 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 529                             unsigned int rd_count,
 530                             struct GNUNET_GNSRECORD_Data *rd_public)
 531 {
 532   struct GNUNET_TIME_Absolute now;
 533   unsigned int rd_public_count;
 534
 535   rd_public_count = 0;
 536   now = GNUNET_TIME_absolute_get ();
 537   for (unsigned int i=0;i<rd_count;i++)
 538   {
 539     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 540       continue;
 541     if ( (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION)) &&
 542          (rd[i].expiration_time < now.abs_value_us) )
 543       continue;  /* record already expired, skip it */
 544     if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 545     {
 546       /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 547          we just need to adjust our iteration frequency */
 548       min_relative_record_time.rel_value_us =
 549         GNUNET_MIN (rd[i].expiration_time,
 550                     min_relative_record_time.rel_value_us);
 551     }
 552     rd_public[rd_public_count++] = rd[i];
 553   }
 554   return rd_public_count;
 555 }
 556
 557
 558 /**
 559  * Store GNS records in the DHT.
 560  *
 561  * @param key key of the zone
 562  * @param label label to store under
 563  * @param rd_public public record data
 564  * @param rd_public_count number of records in @a rd_public
 565  * @param ma handle for the put operation
 566  * @return DHT PUT handle, NULL on error
 567  */
 568 static struct GNUNET_DHT_PutHandle *
 569 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 570                  const char *label,
 571                  const struct GNUNET_GNSRECORD_Data *rd_public,
 572                  unsigned int rd_public_count,
 573                  struct DhtPutActivity *ma)
 574 {
 575   struct GNUNET_GNSRECORD_Block *block;
 576   struct GNUNET_HashCode query;
 577   struct GNUNET_TIME_Absolute expire;
 578   size_t block_size;
 579   struct GNUNET_DHT_PutHandle *ret;
 580
 581   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 582                                                         rd_public);
 583   if (cache_keys)
 584     block = GNUNET_GNSRECORD_block_create2 (key,
 585                                             expire,
 586                                             label,
 587                                             rd_public,
 588                                             rd_public_count);
 589   else
 590     block = GNUNET_GNSRECORD_block_create (key,
 591                                            expire,
 592                                            label,
 593                                            rd_public,
 594                                            rd_public_count);
 595   if (NULL == block)
 596   {
 597     GNUNET_break (0);
 598     return NULL; /* whoops */
 599   }
 600   block_size = ntohl (block->purpose.size)
 601     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 602     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 603   GNUNET_GNSRECORD_query_from_private_key (key,
 604                                            label,
 605                                            &query);
 606   GNUNET_STATISTICS_update (statistics,
 607                             "DHT put operations initiated",
 608                             1,
 609                             GNUNET_NO);
 610   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 611               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 612               rd_public_count,
 613               label,
 614               GNUNET_STRINGS_absolute_time_to_string (expire),
 615               GNUNET_h2s (&query));
 616   num_public_records++;
 617   ret = GNUNET_DHT_put (dht_handle,
 618                         &query,
 619                         DHT_GNS_REPLICATION_LEVEL,
 620                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 621                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 622                         block_size,
 623                         block,
 624                         expire,
 625                         &dht_put_continuation,
 626                         ma);
 627   GNUNET_free (block);
 628   return ret;
 629 }
 630
 631
 632 /**
 633  * We encountered an error in our zone iteration.
 634  *
 635  * @param cls NULL
 636  */
 637 static void
 638 zone_iteration_error (void *cls)
 639 {
 640   (void) cls;
 641   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 642               "Got disconnected from namestore database, retrying.\n");
 643   namestore_iter = NULL;
 644   /* We end up here on error/disconnect/shutdown, so potentially
 645      while a zone publish task or a DHT put is still running; hence
 646      we need to cancel those. */
 647   if (NULL != zone_publish_task)
 648   {
 649     GNUNET_SCHEDULER_cancel (zone_publish_task);
 650     zone_publish_task = NULL;
 651   }
 652   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 653                                                 NULL);
 654 }
 655
 656
 657 /**
 658  * Zone iteration is completed.
 659  *
 660  * @param cls NULL
 661  */
 662 static void
 663 zone_iteration_finished (void *cls)
 664 {
 665   (void) cls;
 666   /* we're done with one iteration, calculate when to do the next one */
 667   namestore_iter = NULL;
 668   last_num_public_records = num_public_records;
 669   first_zone_iteration = GNUNET_NO;
 670   last_min_relative_record_time = min_relative_record_time;
 671   calculate_put_interval ();
 672   /* reset for next iteration */
 673   min_relative_record_time
 674     = GNUNET_TIME_UNIT_FOREVER_REL;
 675   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 676               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 677               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 678                                                       GNUNET_YES));
 679   GNUNET_STATISTICS_set (statistics,
 680                          "Target zone iteration velocity (μs)",
 681                          target_iteration_velocity_per_record.rel_value_us,
 682                          GNUNET_NO);
 683   GNUNET_STATISTICS_set (statistics,
 684                          "Number of public records in DHT",
 685                          last_num_public_records,
 686                          GNUNET_NO);
 687   GNUNET_assert (NULL == zone_publish_task);
 688   if (0 == last_num_public_records)
 689   {
 690     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 691                                                       &publish_zone_dht_start,
 692                                                       NULL);
 693   }
 694   else
 695   {
 696     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 697                                                   NULL);
 698   }
 699 }
 700
 701
 702 /**
 703  * Function used to put all records successively into the DHT.
 704  *
 705  * @param cls the closure (NULL)
 706  * @param key the private key of the authority (ours)
 707  * @param label the name of the records, NULL once the iteration is done
 708  * @param rd_count the number of records in @a rd
 709  * @param rd the record data
 710  */
 711 static void
 712 put_gns_record (void *cls,
 713                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 714                 const char *label,
 715                 unsigned int rd_count,
 716                 const struct GNUNET_GNSRECORD_Data *rd)
 717 {
 718   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 719   unsigned int rd_public_count;
 720   struct DhtPutActivity *ma;
 721
 722   (void) cls;
 723   ns_iteration_left--;
 724   rd_public_count = convert_records_for_export (rd,
 725                                                 rd_count,
 726                                                 rd_public);
 727   if (0 == rd_public_count)
 728   {
 729     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 730                 "Record set empty, moving to next record set\n");
 731     check_zone_namestore_next ();
 732     return;
 733   }
 734   /* We got a set of records to publish */
 735   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 736               "Starting DHT PUT\n");
 737   ma = GNUNET_new (struct DhtPutActivity);
 738   ma->start_date = GNUNET_TIME_absolute_get ();
 739   ma->ph = perform_dht_put (key,
 740                             label,
 741                             rd_public,
 742                             rd_public_count,
 743                             ma);
 744   put_cnt++;
 745   if (0 == put_cnt % DELTA_INTERVAL)
 746     update_velocity (DELTA_INTERVAL);
 747   check_zone_namestore_next ();
 748   if (NULL == ma->ph)
 749   {
 750     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 751                 "Could not perform DHT PUT, is the DHT running?\n");
 752     GNUNET_free (ma);
 753     return;
 754   }
 755   dht_queue_length++;
 756   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 757                                     it_tail,
 758                                     ma);
 759   if (dht_queue_length > DHT_QUEUE_LIMIT)
 760   {
 761     ma = it_head;
 762     GNUNET_CONTAINER_DLL_remove (it_head,
 763                                  it_tail,
 764                                  ma);
 765     GNUNET_DHT_put_cancel (ma->ph);
 766     dht_queue_length--;
 767     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 768                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 769                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 770                                                         GNUNET_YES));
 771     GNUNET_free (ma);
 772   }
 773 }
 774
 775
 776 /**
 777  * Periodically iterate over all zones and store everything in DHT
 778  *
 779  * @param cls NULL
 780  */
 781 static void
 782 publish_zone_dht_start (void *cls)
 783 {
 784   (void) cls;
 785   zone_publish_task = NULL;
 786   GNUNET_STATISTICS_update (statistics,
 787                             "Full zone iterations launched",
 788                             1,
 789                             GNUNET_NO);
 790   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 791               "Starting DHT zone update!\n");
 792   /* start counting again */
 793   num_public_records = 0;
 794   GNUNET_assert (NULL == namestore_iter);
 795   ns_iteration_left = 1;
 796   namestore_iter
 797     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 798                                              NULL, /* All zones */
 799                                              &zone_iteration_error,
 800                                              NULL,
 801                                              &put_gns_record,
 802                                              NULL,
 803                                              &zone_iteration_finished,
 804                                              NULL);
 805   GNUNET_assert (NULL != namestore_iter);
 806 }
 807
 808
 809 /**
 810  * Performe zonemaster duties: watch namestore, publish records.
 811  *
 812  * @param cls closure
 813  * @param server the initialized server
 814  * @param c configuration to use
 815  */
 816 static void
 817 run (void *cls,
 818      const struct GNUNET_CONFIGURATION_Handle *c,
 819      struct GNUNET_SERVICE_Handle *service)
 820 {
 821   unsigned long long max_parallel_bg_queries = 128;
 822
 823   (void) cls;
 824   (void) service;
 825   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 826   min_relative_record_time
 827     = GNUNET_TIME_UNIT_FOREVER_REL;
 828   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
 829   namestore_handle = GNUNET_NAMESTORE_connect (c);
 830   if (NULL == namestore_handle)
 831   {
 832     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 833                 _("Failed to connect to the namestore!\n"));
 834     GNUNET_SCHEDULER_shutdown ();
 835     return;
 836   }
 837   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 838                                                      "namestore",
 839                                                      "CACHE_KEYS");
 840   zone_publish_time_window_default = GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY;
 841   if (GNUNET_OK ==
 842       GNUNET_CONFIGURATION_get_value_time (c,
 843                                            "zonemaster",
 844                                            "ZONE_PUBLISH_TIME_WINDOW",
 845                                            &zone_publish_time_window_default))
 846   {
 847     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 848                 "Time window for zone iteration: %s\n",
 849                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
 850                                                         GNUNET_YES));
 851   }
 852   zone_publish_time_window = zone_publish_time_window_default;
 853   if (GNUNET_OK ==
 854       GNUNET_CONFIGURATION_get_value_number (c,
 855                                              "zonemaster",
 856                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 857                                              &max_parallel_bg_queries))
 858   {
 859     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 860                 "Number of allowed parallel background queries: %llu\n",
 861                 max_parallel_bg_queries);
 862   }
 863   if (0 == max_parallel_bg_queries)
 864     max_parallel_bg_queries = 1;
 865   dht_handle = GNUNET_DHT_connect (c,
 866                                    (unsigned int) max_parallel_bg_queries);
 867   if (NULL == dht_handle)
 868   {
 869     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 870                 _("Could not connect to DHT!\n"));
 871     GNUNET_SCHEDULER_add_now (&shutdown_task,
 872                               NULL);
 873     return;
 874   }
 875
 876   /* Schedule periodic put for our records. */
 877   first_zone_iteration = GNUNET_YES;
 878   statistics = GNUNET_STATISTICS_create ("zonemaster",
 879                                          c);
 880   GNUNET_STATISTICS_set (statistics,
 881                          "Target zone iteration velocity (μs)",
 882                          target_iteration_velocity_per_record.rel_value_us,
 883                          GNUNET_NO);
 884   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 885                                                 NULL);
 886   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 887                                  NULL);
 888 }
 889
 890
 891 /**
 892  * Define "main" method using service macro.
 893  */
 894 GNUNET_SERVICE_MAIN
 895 ("zonemaster",
 896  GNUNET_SERVICE_OPTION_NONE,
 897  &run,
 898  NULL,
 899  NULL,
 900  NULL,
 901  GNUNET_MQ_handler_end());
 902
 903
 904 /* end of gnunet-service-zonemaster.c */