src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32 #include "gnunet_namestore_plugin.h"
  33 #include "gnunet_signatures.h"
  34
  35
  36 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  37
  38
  39 /**
  40  * How often should we (re)publish each record before
  41  * it expires?
  42  */
  43 #define PUBLISH_OPS_PER_EXPIRATION 4
  44
  45 /**
  46  * How often do we measure the delta between desired zone
  47  * iteration speed and actual speed, and tell statistics
  48  * service about it?
  49  */
  50 #define DELTA_INTERVAL 100
  51
  52 /**
  53  * How many records do we fetch in one shot from the namestore?
  54  */
  55 #define NS_BLOCK_SIZE 1000
  56
  57 /**
  58  * How many pending DHT operations do we allow at most?
  59  */
  60 #define DHT_QUEUE_LIMIT 2000
  61
  62 /**
  63  * How many events may the namestore give us before it has to wait
  64  * for us to keep up?
  65  */
  66 #define NAMESTORE_QUEUE_LIMIT 50
  67
  68 /**
  69  * The initial interval in milliseconds btween puts in
  70  * a zone iteration
  71  */
  72 #define INITIAL_ZONE_ITERATION_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  73
  74 /**
  75  * The upper bound for the zone iteration interval
  76  * (per record).
  77  */
  78 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  79
  80 /**
  81  * The default put interval for the zone iteration. In case
  82  * no option is found
  83  */
  84 #define DEFAULT_ZONE_PUBLISH_TIME_WINDOW GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
  85
  86 /**
  87  * The factor the current zone iteration interval is divided by for each
  88  * additional new record
  89  */
  90 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  91
  92 /**
  93  * How long until a DHT PUT attempt should time out?
  94  */
  95 #define DHT_OPERATION_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
  96
  97 /**
  98  * What replication level do we use for DHT PUT operations?
  99  */
 100 #define DHT_GNS_REPLICATION_LEVEL 5
 101
 102
 103 /**
 104  * Handle for DHT PUT activity triggered from the namestore monitor.
 105  */
 106 struct DhtPutActivity
 107 {
 108   /**
 109    * Kept in a DLL.
 110    */
 111   struct DhtPutActivity *next;
 112
 113   /**
 114    * Kept in a DLL.
 115    */
 116   struct DhtPutActivity *prev;
 117
 118   /**
 119    * Handle for the DHT PUT operation.
 120    */
 121   struct GNUNET_DHT_PutHandle *ph;
 122
 123   /**
 124    * When was this PUT initiated?
 125    */
 126   struct GNUNET_TIME_Absolute start_date;
 127 };
 128
 129
 130 /**
 131  * Handle to the statistics service
 132  */
 133 static struct GNUNET_STATISTICS_Handle *statistics;
 134
 135 /**
 136  * Our handle to the DHT
 137  */
 138 static struct GNUNET_DHT_Handle *dht_handle;
 139
 140 /**
 141  * Our handle to the namestore service
 142  */
 143 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 144
 145 /**
 146  * Handle to iterate over our authoritative zone in namestore
 147  */
 148 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 149
 150 /**
 151  * Handle to monitor namestore changes to instant propagation.
 152  */
 153 static struct GNUNET_NAMESTORE_ZoneMonitor *zmon;
 154
 155 /**
 156  * Head of monitor activities; kept in a DLL.
 157  */
 158 static struct DhtPutActivity *ma_head;
 159
 160 /**
 161  * Tail of monitor activities; kept in a DLL.
 162  */
 163 static struct DhtPutActivity *ma_tail;
 164
 165 /**
 166  * Head of iteration put activities; kept in a DLL.
 167  */
 168 static struct DhtPutActivity *it_head;
 169
 170 /**
 171  * Tail of iteration put activities; kept in a DLL.
 172  */
 173 static struct DhtPutActivity *it_tail;
 174
 175 /**
 176  * Number of entries in the DHT queue #it_head.
 177  */
 178 static unsigned int dht_queue_length;
 179
 180 /**
 181  * Number of entries in the DHT queue #ma_head.
 182  */
 183 static unsigned int ma_queue_length;
 184
 185 /**
 186  * Useful for zone update for DHT put
 187  */
 188 static unsigned long long num_public_records;
 189
 190 /**
 191  * Last seen record count
 192  */
 193 static unsigned long long last_num_public_records;
 194
 195 /**
 196  * Number of successful put operations performed in the current
 197  * measurement cycle (as measured in #check_zone_namestore_next()).
 198  */
 199 static unsigned long long put_cnt;
 200
 201 /**
 202  * What is the frequency at which we currently would like
 203  * to perform DHT puts (per record)?  Calculated in
 204  * update_velocity() from the #zone_publish_time_window()
 205  * and the total number of record sets we have (so far)
 206  * observed in the zone.
 207  */
 208 static struct GNUNET_TIME_Relative target_iteration_velocity_per_record;
 209
 210 /**
 211  * Minimum relative expiration time of records seem during the current
 212  * zone iteration.
 213  */
 214 static struct GNUNET_TIME_Relative min_relative_record_time;
 215
 216 /**
 217  * Minimum relative expiration time of records seem during the last
 218  * zone iteration.
 219  */
 220 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 221
 222 /**
 223  * Default time window for zone iteration
 224  */
 225 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 226
 227 /**
 228  * Time window for zone iteration, adjusted based on relative record
 229  * expiration times in our zone.
 230  */
 231 static struct GNUNET_TIME_Relative zone_publish_time_window;
 232
 233 /**
 234  * When did we last start measuring the #DELTA_INTERVAL successful
 235  * DHT puts? Used for velocity calculations.
 236  */
 237 static struct GNUNET_TIME_Absolute last_put_100;
 238
 239 /**
 240  * By how much should we try to increase our per-record iteration speed
 241  * (over the desired speed calculated directly from the #put_interval)?
 242  * Basically this value corresponds to the per-record CPU time overhead
 243  * we have.
 244  */
 245 static struct GNUNET_TIME_Relative sub_delta;
 246
 247 /**
 248  * zone publish task
 249  */
 250 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 251
 252 /**
 253  * How many more values are left for the current query before we need
 254  * to explicitly ask the namestore for more?
 255  */
 256 static unsigned int ns_iteration_left;
 257
 258 /**
 259  * #GNUNET_YES if zone has never been published before
 260  */
 261 static int first_zone_iteration;
 262
 263 /**
 264  * Optimize block insertion by caching map of private keys to
 265  * public keys in memory?
 266  */
 267 static int cache_keys;
 268
 269
 270 /**
 271  * Task run during shutdown.
 272  *
 273  * @param cls unused
 274  * @param tc unused
 275  */
 276 static void
 277 shutdown_task (void *cls)
 278 {
 279   struct DhtPutActivity *ma;
 280
 281   (void) cls;
 282   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 283               "Shutting down!\n");
 284   while (NULL != (ma = ma_head))
 285   {
 286     GNUNET_DHT_put_cancel (ma->ph);
 287     ma_queue_length--;
 288     GNUNET_CONTAINER_DLL_remove (ma_head,
 289                                  ma_tail,
 290                                  ma);
 291     GNUNET_free (ma);
 292   }
 293   while (NULL != (ma = it_head))
 294   {
 295     GNUNET_DHT_put_cancel (ma->ph);
 296     dht_queue_length--;
 297     GNUNET_CONTAINER_DLL_remove (it_head,
 298                                  it_tail,
 299                                  ma);
 300     dht_queue_length--;
 301     GNUNET_free (ma);
 302   }
 303   if (NULL != statistics)
 304   {
 305     GNUNET_STATISTICS_destroy (statistics,
 306                                GNUNET_NO);
 307     statistics = NULL;
 308   }
 309   if (NULL != zone_publish_task)
 310   {
 311     GNUNET_SCHEDULER_cancel (zone_publish_task);
 312     zone_publish_task = NULL;
 313   }
 314   if (NULL != namestore_iter)
 315   {
 316     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 317     namestore_iter = NULL;
 318   }
 319   if (NULL != zmon)
 320   {
 321     GNUNET_NAMESTORE_zone_monitor_stop (zmon);
 322     zmon = NULL;
 323   }
 324   if (NULL != namestore_handle)
 325   {
 326     GNUNET_NAMESTORE_disconnect (namestore_handle);
 327     namestore_handle = NULL;
 328   }
 329   if (NULL != dht_handle)
 330   {
 331     GNUNET_DHT_disconnect (dht_handle);
 332     dht_handle = NULL;
 333   }
 334 }
 335
 336
 337 /**
 338  * Method called periodically that triggers iteration over authoritative records
 339  *
 340  * @param cls NULL
 341  */
 342 static void
 343 publish_zone_namestore_next (void *cls)
 344 {
 345   (void) cls;
 346   zone_publish_task = NULL;
 347   GNUNET_assert (NULL != namestore_iter);
 348   GNUNET_assert (0 == ns_iteration_left);
 349   ns_iteration_left = NS_BLOCK_SIZE;
 350   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 351                                        NS_BLOCK_SIZE);
 352 }
 353
 354
 355 /**
 356  * Periodically iterate over our zone and store everything in dht
 357  *
 358  * @param cls NULL
 359  */
 360 static void
 361 publish_zone_dht_start (void *cls);
 362
 363
 364 /**
 365  * Continuation called from DHT once the PUT operation triggered
 366  * by a monitor is done.
 367  *
 368  * @param cls a `struct DhtPutActivity`
 369  */
 370 static void
 371 dht_put_monitor_continuation (void *cls)
 372 {
 373   struct DhtPutActivity *ma = cls;
 374
 375   GNUNET_NAMESTORE_zone_monitor_next (zmon,
 376                                       1);
 377   ma_queue_length--;
 378   GNUNET_CONTAINER_DLL_remove (ma_head,
 379                                ma_tail,
 380                                ma);
 381   GNUNET_free (ma);
 382 }
 383
 384
 385 /**
 386  * Calculate #target_iteration_velocity_per_record.
 387  */
 388 static void
 389 calculate_put_interval ()
 390 {
 391   if (0 == num_public_records)
 392   {
 393     /**
 394      * If no records are known (startup) or none present
 395      * we can safely set the interval to the value for a single
 396      * record
 397      */
 398     target_iteration_velocity_per_record = zone_publish_time_window;
 399     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 400                 "No records in namestore database.\n");
 401   }
 402   else
 403   {
 404     last_min_relative_record_time
 405       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 406                                   min_relative_record_time);
 407     zone_publish_time_window
 408       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 409                                                                PUBLISH_OPS_PER_EXPIRATION),
 410                                   zone_publish_time_window_default);
 411     target_iteration_velocity_per_record
 412       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 413                                      last_num_public_records);
 414   }
 415   target_iteration_velocity_per_record
 416     = GNUNET_TIME_relative_min (target_iteration_velocity_per_record,
 417                                 MAXIMUM_ZONE_ITERATION_INTERVAL);
 418   GNUNET_STATISTICS_set (statistics,
 419                          "Minimum relative record expiration (in μs)",
 420                          last_min_relative_record_time.rel_value_us,
 421                          GNUNET_NO);
 422   GNUNET_STATISTICS_set (statistics,
 423                          "Zone publication time window (in μs)",
 424                          zone_publish_time_window.rel_value_us,
 425                          GNUNET_NO);
 426   GNUNET_STATISTICS_set (statistics,
 427                          "Target zone iteration velocity (μs)",
 428                          target_iteration_velocity_per_record.rel_value_us,
 429                          GNUNET_NO);
 430 }
 431
 432
 433 /**
 434  * Re-calculate our velocity and the desired velocity.
 435  * We have succeeded in making #DELTA_INTERVAL puts, so
 436  * now calculate the new desired delay between puts.
 437  *
 438  * @param cnt how many records were processed since the last call?
 439  */
 440 static void
 441 update_velocity (unsigned int cnt)
 442 {
 443   struct GNUNET_TIME_Relative delta;
 444   unsigned long long pct = 0;
 445
 446   if (0 == cnt)
 447     return;
 448   /* How fast were we really? */
 449   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 450   delta.rel_value_us /= cnt;
 451   last_put_100 = GNUNET_TIME_absolute_get ();
 452
 453   /* calculate expected frequency */
 454   if ( (num_public_records > last_num_public_records) &&
 455        (GNUNET_NO == first_zone_iteration) )
 456   {
 457     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 458                 "Last record count was lower than current record count.  Reducing interval.\n");
 459     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 460     calculate_put_interval ();
 461   }
 462   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 463               "Desired global zone iteration interval is %s/record!\n",
 464               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 465                                                       GNUNET_YES));
 466
 467   /* Tell statistics actual vs. desired speed */
 468   GNUNET_STATISTICS_set (statistics,
 469                          "Current zone iteration velocity (μs/record)",
 470                          delta.rel_value_us,
 471                          GNUNET_NO);
 472   /* update "sub_delta" based on difference, taking
 473      previous sub_delta into account! */
 474   if (target_iteration_velocity_per_record.rel_value_us > delta.rel_value_us)
 475   {
 476     /* We were too fast, reduce sub_delta! */
 477     struct GNUNET_TIME_Relative corr;
 478
 479     corr = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 480                                           delta);
 481     if (sub_delta.rel_value_us > delta.rel_value_us)
 482     {
 483       /* Reduce sub_delta by corr */
 484       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 485                                                  corr);
 486     }
 487     else
 488     {
 489       /* We're doing fine with waiting the full time, this
 490          should theoretically only happen if we run at
 491          infinite speed. */
 492       sub_delta = GNUNET_TIME_UNIT_ZERO;
 493     }
 494   }
 495   else if (target_iteration_velocity_per_record.rel_value_us < delta.rel_value_us)
 496   {
 497     /* We were too slow, increase sub_delta! */
 498     struct GNUNET_TIME_Relative corr;
 499
 500     corr = GNUNET_TIME_relative_subtract (delta,
 501                                           target_iteration_velocity_per_record);
 502     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 503                                           corr);
 504     if (sub_delta.rel_value_us > target_iteration_velocity_per_record.rel_value_us)
 505     {
 506       /* CPU overload detected, we cannot go at desired speed,
 507          as this would mean using a negative delay. */
 508       /* compute how much faster we would want to be for
 509          the desired velocity */
 510       if (0 == target_iteration_velocity_per_record.rel_value_us)
 511         pct = UINT64_MAX; /* desired speed is infinity ... */
 512       else
 513         pct = (sub_delta.rel_value_us -
 514                target_iteration_velocity_per_record.rel_value_us) * 100LLU
 515           / target_iteration_velocity_per_record.rel_value_us;
 516       sub_delta = target_iteration_velocity_per_record;
 517     }
 518   }
 519   GNUNET_STATISTICS_set (statistics,
 520                          "# size of the DHT queue (it)",
 521                          dht_queue_length,
 522                          GNUNET_NO);
 523   GNUNET_STATISTICS_set (statistics,
 524                          "# size of the DHT queue (mon)",
 525                          ma_queue_length,
 526                          GNUNET_NO);
 527   GNUNET_STATISTICS_set (statistics,
 528                          "% speed increase needed for target velocity",
 529                          pct,
 530                          GNUNET_NO);
 531   GNUNET_STATISTICS_set (statistics,
 532                          "# records processed in current iteration",
 533                          num_public_records,
 534                          GNUNET_NO);
 535 }
 536
 537
 538 /**
 539  * Check if the current zone iteration needs to be continued
 540  * by calling #publish_zone_namestore_next(), and if so with what delay.
 541  */
 542 static void
 543 check_zone_namestore_next ()
 544 {
 545   struct GNUNET_TIME_Relative delay;
 546
 547   if (0 != ns_iteration_left)
 548     return; /* current NAMESTORE iteration not yet done */
 549   update_velocity (put_cnt);
 550   put_cnt = 0;
 551   delay = GNUNET_TIME_relative_subtract (target_iteration_velocity_per_record,
 552                                          sub_delta);
 553   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 554      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 555   GNUNET_STATISTICS_set (statistics,
 556                          "Current artificial NAMESTORE delay (μs/record)",
 557                          delay.rel_value_us,
 558                          GNUNET_NO);
 559   delay = GNUNET_TIME_relative_multiply (delay,
 560                                          NS_BLOCK_SIZE);
 561   GNUNET_assert (NULL == zone_publish_task);
 562   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 563                                                     &publish_zone_namestore_next,
 564                                                     NULL);
 565 }
 566
 567
 568 /**
 569  * Continuation called from DHT once the PUT operation is done.
 570  *
 571  * @param cls a `struct DhtPutActivity`
 572  */
 573 static void
 574 dht_put_continuation (void *cls)
 575 {
 576   struct DhtPutActivity *ma = cls;
 577
 578   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 579               "PUT complete\n");
 580   dht_queue_length--;
 581   GNUNET_CONTAINER_DLL_remove (it_head,
 582                                it_tail,
 583                                ma);
 584   GNUNET_free (ma);
 585 }
 586
 587
 588 /**
 589  * Convert namestore records from the internal format to that
 590  * suitable for publication (removes private records, converts
 591  * to absolute expiration time).
 592  *
 593  * @param rd input records
 594  * @param rd_count size of the @a rd and @a rd_public arrays
 595  * @param rd_public where to write the converted records
 596  * @return number of records written to @a rd_public
 597  */
 598 static unsigned int
 599 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 600                             unsigned int rd_count,
 601                             struct GNUNET_GNSRECORD_Data *rd_public)
 602 {
 603   struct GNUNET_TIME_Absolute now;
 604   unsigned int rd_public_count;
 605
 606   rd_public_count = 0;
 607   now = GNUNET_TIME_absolute_get ();
 608   for (unsigned int i=0;i<rd_count;i++)
 609     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 610     {
 611       rd_public[rd_public_count] = rd[i];
 612       if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 613       {
 614         /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 615            we just need to adjust our iteration frequency */
 616         min_relative_record_time.rel_value_us =
 617           GNUNET_MIN (rd_public[rd_public_count].expiration_time,
 618                       min_relative_record_time.rel_value_us);
 619       }
 620       else if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
 621       {
 622         /* record already expired, skip it */
 623         continue;
 624       }
 625       rd_public_count++;
 626     }
 627   return rd_public_count;
 628 }
 629
 630
 631 /**
 632  * Store GNS records in the DHT.
 633  *
 634  * @param key key of the zone
 635  * @param label label to store under
 636  * @param rd_public public record data
 637  * @param rd_public_count number of records in @a rd_public
 638  * @param cont function to call with PUT result
 639  * @param cont_cls closure for @a cont
 640  * @return DHT PUT handle, NULL on error
 641  */
 642 static struct GNUNET_DHT_PutHandle *
 643 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 644                  const char *label,
 645                  const struct GNUNET_GNSRECORD_Data *rd_public,
 646                  unsigned int rd_public_count,
 647                  GNUNET_SCHEDULER_TaskCallback cont,
 648                  void *cont_cls)
 649 {
 650   struct GNUNET_GNSRECORD_Block *block;
 651   struct GNUNET_HashCode query;
 652   struct GNUNET_TIME_Absolute expire;
 653   size_t block_size;
 654   struct GNUNET_DHT_PutHandle *ret;
 655
 656   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 657                                                         rd_public);
 658   if (cache_keys)
 659     block = GNUNET_GNSRECORD_block_create2 (key,
 660                                             expire,
 661                                             label,
 662                                             rd_public,
 663                                             rd_public_count);
 664   else
 665     block = GNUNET_GNSRECORD_block_create (key,
 666                                            expire,
 667                                            label,
 668                                            rd_public,
 669                                            rd_public_count);
 670   if (NULL == block)
 671   {
 672     GNUNET_break (0);
 673     return NULL; /* whoops */
 674   }
 675   block_size = ntohl (block->purpose.size)
 676     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 677     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 678   GNUNET_GNSRECORD_query_from_private_key (key,
 679                                            label,
 680                                            &query);
 681   GNUNET_STATISTICS_update (statistics,
 682                             "DHT put operations initiated",
 683                             1,
 684                             GNUNET_NO);
 685   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 686               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 687               rd_public_count,
 688               label,
 689               GNUNET_STRINGS_absolute_time_to_string (expire),
 690               GNUNET_h2s (&query));
 691   num_public_records++;
 692   ret = GNUNET_DHT_put (dht_handle,
 693                         &query,
 694                         DHT_GNS_REPLICATION_LEVEL,
 695                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 696                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 697                         block_size,
 698                         block,
 699                         expire,
 700                         cont,
 701                         cont_cls);
 702   GNUNET_free (block);
 703   return ret;
 704 }
 705
 706
 707 /**
 708  * We encountered an error in our zone iteration.
 709  *
 710  * @param cls NULL
 711  */
 712 static void
 713 zone_iteration_error (void *cls)
 714 {
 715   (void) cls;
 716   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 717               "Got disconnected from namestore database, retrying.\n");
 718   namestore_iter = NULL;
 719   /* We end up here on error/disconnect/shutdown, so potentially
 720      while a zone publish task or a DHT put is still running; hence
 721      we need to cancel those. */
 722   if (NULL != zone_publish_task)
 723   {
 724     GNUNET_SCHEDULER_cancel (zone_publish_task);
 725     zone_publish_task = NULL;
 726   }
 727   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 728                                                 NULL);
 729 }
 730
 731
 732 /**
 733  * Zone iteration is completed.
 734  *
 735  * @param cls NULL
 736  */
 737 static void
 738 zone_iteration_finished (void *cls)
 739 {
 740   (void) cls;
 741   /* we're done with one iteration, calculate when to do the next one */
 742   namestore_iter = NULL;
 743   last_num_public_records = num_public_records;
 744   first_zone_iteration = GNUNET_NO;
 745   last_min_relative_record_time = min_relative_record_time;
 746   calculate_put_interval ();
 747   /* reset for next iteration */
 748   min_relative_record_time
 749     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 750                                      PUBLISH_OPS_PER_EXPIRATION);
 751   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 752               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 753               GNUNET_STRINGS_relative_time_to_string (target_iteration_velocity_per_record,
 754                                                       GNUNET_YES));
 755   GNUNET_STATISTICS_set (statistics,
 756                          "Target zone iteration velocity (μs)",
 757                          target_iteration_velocity_per_record.rel_value_us,
 758                          GNUNET_NO);
 759   GNUNET_STATISTICS_set (statistics,
 760                          "Number of public records in DHT",
 761                          last_num_public_records,
 762                          GNUNET_NO);
 763   GNUNET_assert (NULL == zone_publish_task);
 764   if (0 == last_num_public_records)
 765   {
 766     zone_publish_task = GNUNET_SCHEDULER_add_delayed (target_iteration_velocity_per_record,
 767                                                       &publish_zone_dht_start,
 768                                                       NULL);
 769   }
 770   else
 771   {
 772     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 773                                                   NULL);
 774   }
 775 }
 776
 777
 778 /**
 779  * Function used to put all records successively into the DHT.
 780  *
 781  * @param cls the closure (NULL)
 782  * @param key the private key of the authority (ours)
 783  * @param label the name of the records, NULL once the iteration is done
 784  * @param rd_count the number of records in @a rd
 785  * @param rd the record data
 786  */
 787 static void
 788 put_gns_record (void *cls,
 789                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 790                 const char *label,
 791                 unsigned int rd_count,
 792                 const struct GNUNET_GNSRECORD_Data *rd)
 793 {
 794   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 795   unsigned int rd_public_count;
 796   struct DhtPutActivity *ma;
 797
 798   (void) cls;
 799   ns_iteration_left--;
 800   rd_public_count = convert_records_for_export (rd,
 801                                                 rd_count,
 802                                                 rd_public);
 803   if (0 == rd_public_count)
 804   {
 805     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 806                 "Record set empty, moving to next record set\n");
 807     check_zone_namestore_next ();
 808     return;
 809   }
 810   /* We got a set of records to publish */
 811   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 812               "Starting DHT PUT\n");
 813   ma = GNUNET_new (struct DhtPutActivity);
 814   ma->start_date = GNUNET_TIME_absolute_get ();
 815   ma->ph = perform_dht_put (key,
 816                             label,
 817                             rd_public,
 818                             rd_public_count,
 819                             &dht_put_continuation,
 820                             ma);
 821   put_cnt++;
 822   if (0 == put_cnt % DELTA_INTERVAL)
 823     update_velocity (DELTA_INTERVAL);
 824   check_zone_namestore_next ();
 825   if (NULL == ma->ph)
 826   {
 827     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 828                 "Could not perform DHT PUT, is the DHT running?\n");
 829     GNUNET_free (ma);
 830     return;
 831   }
 832   dht_queue_length++;
 833   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 834                                     it_tail,
 835                                     ma);
 836   if (dht_queue_length > DHT_QUEUE_LIMIT)
 837   {
 838     ma = it_head;
 839     GNUNET_CONTAINER_DLL_remove (it_head,
 840                                  it_tail,
 841                                  ma);
 842     GNUNET_DHT_put_cancel (ma->ph);
 843     dht_queue_length--;
 844     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 845                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 846                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 847                                                         GNUNET_YES));
 848     GNUNET_free (ma);
 849   }
 850 }
 851
 852
 853 /**
 854  * Periodically iterate over all zones and store everything in DHT
 855  *
 856  * @param cls NULL
 857  */
 858 static void
 859 publish_zone_dht_start (void *cls)
 860 {
 861   (void) cls;
 862   zone_publish_task = NULL;
 863   GNUNET_STATISTICS_update (statistics,
 864                             "Full zone iterations launched",
 865                             1,
 866                             GNUNET_NO);
 867   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 868               "Starting DHT zone update!\n");
 869   /* start counting again */
 870   num_public_records = 0;
 871   GNUNET_assert (NULL == namestore_iter);
 872   ns_iteration_left = 1;
 873   namestore_iter
 874     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 875                                              NULL, /* All zones */
 876                                              &zone_iteration_error,
 877                                              NULL,
 878                                              &put_gns_record,
 879                                              NULL,
 880                                              &zone_iteration_finished,
 881                                              NULL);
 882   GNUNET_assert (NULL != namestore_iter);
 883 }
 884
 885
 886 /**
 887  * Process a record that was stored in the namestore
 888  * (invoked by the monitor).
 889  *
 890  * @param cls closure, NULL
 891  * @param zone private key of the zone; NULL on disconnect
 892  * @param label label of the records; NULL on disconnect
 893  * @param rd_count number of entries in @a rd array, 0 if label was deleted
 894  * @param rd array of records with data to store
 895  */
 896 static void
 897 handle_monitor_event (void *cls,
 898                       const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
 899                       const char *label,
 900                       unsigned int rd_count,
 901                       const struct GNUNET_GNSRECORD_Data *rd)
 902 {
 903   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 904   unsigned int rd_public_count;
 905   struct DhtPutActivity *ma;
 906
 907   (void) cls;
 908   GNUNET_STATISTICS_update (statistics,
 909                             "Namestore monitor events received",
 910                             1,
 911                             GNUNET_NO);
 912   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 913               "Received %u records for label `%s' via namestore monitor\n",
 914               rd_count,
 915               label);
 916   /* filter out records that are not public, and convert to
 917      absolute expiration time. */
 918   rd_public_count = convert_records_for_export (rd,
 919                                                 rd_count,
 920                                                 rd_public);
 921   if (0 == rd_public_count)
 922   {
 923     GNUNET_NAMESTORE_zone_monitor_next (zmon,
 924                                         1);
 925     return; /* nothing to do */
 926   }
 927   num_public_records++;
 928   ma = GNUNET_new (struct DhtPutActivity);
 929   ma->start_date = GNUNET_TIME_absolute_get ();
 930   ma->ph = perform_dht_put (zone,
 931                             label,
 932                             rd,
 933                             rd_count,
 934                             &dht_put_monitor_continuation,
 935                             ma);
 936   if (NULL == ma->ph)
 937   {
 938     /* PUT failed, do not remember operation */
 939     GNUNET_free (ma);
 940     GNUNET_NAMESTORE_zone_monitor_next (zmon,
 941                                         1);
 942     return;
 943   }
 944   GNUNET_CONTAINER_DLL_insert_tail (ma_head,
 945                                     ma_tail,
 946                                     ma);
 947   ma_queue_length++;
 948   if (ma_queue_length > DHT_QUEUE_LIMIT)
 949   {
 950     ma = ma_head;
 951     GNUNET_CONTAINER_DLL_remove (ma_head,
 952                                  ma_tail,
 953                                  ma);
 954     GNUNET_DHT_put_cancel (ma->ph);
 955     ma_queue_length--;
 956     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 957                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 958                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 959                                                         GNUNET_YES));
 960     GNUNET_free (ma);
 961   }
 962 }
 963
 964
 965 /**
 966  * The zone monitor is now in SYNC with the current state of the
 967  * name store.  Start to perform periodic iterations.
 968  *
 969  * @param cls NULL
 970  */
 971 static void
 972 monitor_sync_event (void *cls)
 973 {
 974   (void) cls;
 975   if ( (NULL == zone_publish_task) &&
 976        (NULL == namestore_iter) )
 977     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 978                                                   NULL);
 979 }
 980
 981
 982 /**
 983  * The zone monitor encountered an IPC error trying to to get in
 984  * sync. Restart from the beginning.
 985  *
 986  * @param cls NULL
 987  */
 988 static void
 989 handle_monitor_error (void *cls)
 990 {
 991   (void) cls;
 992   GNUNET_STATISTICS_update (statistics,
 993                             "Namestore monitor errors encountered",
 994                             1,
 995                             GNUNET_NO);
 996   if (NULL != zone_publish_task)
 997   {
 998     GNUNET_SCHEDULER_cancel (zone_publish_task);
 999     zone_publish_task = NULL;
1000   }
1001   if (NULL != namestore_iter)
1002   {
1003     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
1004     namestore_iter = NULL;
1005   }
1006   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
1007                                                 NULL);
1008 }
1009
1010
1011 /**
1012  * Performe zonemaster duties: watch namestore, publish records.
1013  *
1014  * @param cls closure
1015  * @param server the initialized server
1016  * @param c configuration to use
1017  */
1018 static void
1019 run (void *cls,
1020      const struct GNUNET_CONFIGURATION_Handle *c,
1021      struct GNUNET_SERVICE_Handle *service)
1022 {
1023   unsigned long long max_parallel_bg_queries = 128;
1024
1025   (void) cls;
1026   (void) service;
1027   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
1028   min_relative_record_time
1029     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
1030                                      PUBLISH_OPS_PER_EXPIRATION);
1031   target_iteration_velocity_per_record = INITIAL_ZONE_ITERATION_INTERVAL;
1032   namestore_handle = GNUNET_NAMESTORE_connect (c);
1033   if (NULL == namestore_handle)
1034   {
1035     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1036                 _("Failed to connect to the namestore!\n"));
1037     GNUNET_SCHEDULER_shutdown ();
1038     return;
1039   }
1040   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
1041                                                      "namestore",
1042                                                      "CACHE_KEYS");
1043   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
1044   if (GNUNET_OK ==
1045       GNUNET_CONFIGURATION_get_value_time (c,
1046                                            "zonemaster",
1047                                            "ZONE_PUBLISH_TIME_WINDOW",
1048                                            &zone_publish_time_window_default))
1049   {
1050     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1051                 "Time window for zone iteration: %s\n",
1052                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
1053                                                         GNUNET_YES));
1054   }
1055   zone_publish_time_window = zone_publish_time_window_default;
1056   if (GNUNET_OK ==
1057       GNUNET_CONFIGURATION_get_value_number (c,
1058                                              "zonemaster",
1059                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
1060                                              &max_parallel_bg_queries))
1061   {
1062     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1063                 "Number of allowed parallel background queries: %llu\n",
1064                 max_parallel_bg_queries);
1065   }
1066   if (0 == max_parallel_bg_queries)
1067     max_parallel_bg_queries = 1;
1068   dht_handle = GNUNET_DHT_connect (c,
1069                                    (unsigned int) max_parallel_bg_queries);
1070   if (NULL == dht_handle)
1071   {
1072     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1073                 _("Could not connect to DHT!\n"));
1074     GNUNET_SCHEDULER_add_now (&shutdown_task,
1075                               NULL);
1076     return;
1077   }
1078
1079   /* Schedule periodic put for our records. */
1080   first_zone_iteration = GNUNET_YES;
1081   statistics = GNUNET_STATISTICS_create ("zonemaster",
1082                                          c);
1083   GNUNET_STATISTICS_set (statistics,
1084                          "Target zone iteration velocity (μs)",
1085                          target_iteration_velocity_per_record.rel_value_us,
1086                          GNUNET_NO);
1087   zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
1088                                               NULL,
1089                                               GNUNET_NO,
1090                                               &handle_monitor_error,
1091                                               NULL,
1092                                               &handle_monitor_event,
1093                                               NULL,
1094                                               &monitor_sync_event,
1095                                               NULL);
1096   GNUNET_NAMESTORE_zone_monitor_next (zmon,
1097                                       NAMESTORE_QUEUE_LIMIT - 1);
1098   GNUNET_break (NULL != zmon);
1099   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
1100                                  NULL);
1101 }
1102
1103
1104 /**
1105  * Define "main" method using service macro.
1106  */
1107 GNUNET_SERVICE_MAIN
1108 ("zonemaster",
1109  GNUNET_SERVICE_OPTION_NONE,
1110  &run,
1111  NULL,
1112  NULL,
1113  NULL,
1114  GNUNET_MQ_handler_end());
1115
1116
1117 /* end of gnunet-service-zonemaster.c */