src/zonemaster/gnunet-service-zonemaster.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster.c
  23  * @brief publish records from namestore to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dnsparser_lib.h"
  29 #include "gnunet_dht_service.h"
  30 #include "gnunet_namestore_service.h"
  31 #include "gnunet_statistics_service.h"
  32 #include "gnunet_namestore_plugin.h"
  33 #include "gnunet_signatures.h"
  34
  35
  36 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  37
  38
  39 /**
  40  * How often should we (re)publish each record before
  41  * it expires?
  42  */
  43 #define PUBLISH_OPS_PER_EXPIRATION 4
  44
  45 /**
  46  * How often do we measure the delta between desired zone
  47  * iteration speed and actual speed, and tell statistics
  48  * service about it?
  49  */
  50 #define DELTA_INTERVAL 100
  51
  52 /**
  53  * How many records do we fetch in one shot from the namestore?
  54  */
  55 #define NS_BLOCK_SIZE 1000
  56
  57 /**
  58  * How many pending DHT operations do we allow at most?
  59  */
  60 #define DHT_QUEUE_LIMIT 2000
  61
  62 /**
  63  * The initial interval in milliseconds btween puts in
  64  * a zone iteration
  65  */
  66 #define INITIAL_PUT_INTERVAL GNUNET_TIME_UNIT_MILLISECONDS
  67
  68 /**
  69  * The upper bound for the zone iteration interval
  70  * (per record).
  71  */
  72 #define MAXIMUM_ZONE_ITERATION_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 15)
  73
  74 /**
  75  * The default put interval for the zone iteration. In case
  76  * no option is found
  77  */
  78 #define DEFAULT_ZONE_PUBLISH_TIME_WINDOW GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS, 4)
  79
  80 /**
  81  * The factor the current zone iteration interval is divided by for each
  82  * additional new record
  83  */
  84 #define LATE_ITERATION_SPEEDUP_FACTOR 2
  85
  86 /**
  87  * How long until a DHT PUT attempt should time out?
  88  */
  89 #define DHT_OPERATION_TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 60)
  90
  91 /**
  92  * What replication level do we use for DHT PUT operations?
  93  */
  94 #define DHT_GNS_REPLICATION_LEVEL 5
  95
  96
  97 /**
  98  * Handle for DHT PUT activity triggered from the namestore monitor.
  99  */
 100 struct DhtPutActivity
 101 {
 102   /**
 103    * Kept in a DLL.
 104    */
 105   struct DhtPutActivity *next;
 106
 107   /**
 108    * Kept in a DLL.
 109    */
 110   struct DhtPutActivity *prev;
 111
 112   /**
 113    * Handle for the DHT PUT operation.
 114    */
 115   struct GNUNET_DHT_PutHandle *ph;
 116
 117   /**
 118    * When was this PUT initiated?
 119    */
 120   struct GNUNET_TIME_Absolute start_date;
 121 };
 122
 123
 124 /**
 125  * Handle to the statistics service
 126  */
 127 static struct GNUNET_STATISTICS_Handle *statistics;
 128
 129 /**
 130  * Our handle to the DHT
 131  */
 132 static struct GNUNET_DHT_Handle *dht_handle;
 133
 134 /**
 135  * Active DHT put operation (or NULL)
 136  */
 137 static struct GNUNET_DHT_PutHandle *active_put;
 138
 139 /**
 140  * Our handle to the namestore service
 141  */
 142 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 143
 144 /**
 145  * Handle to iterate over our authoritative zone in namestore
 146  */
 147 static struct GNUNET_NAMESTORE_ZoneIterator *namestore_iter;
 148
 149 /**
 150  * Handle to monitor namestore changes to instant propagation.
 151  */
 152 static struct GNUNET_NAMESTORE_ZoneMonitor *zmon;
 153
 154 /**
 155  * Head of monitor activities; kept in a DLL.
 156  */
 157 static struct DhtPutActivity *ma_head;
 158
 159 /**
 160  * Tail of monitor activities; kept in a DLL.
 161  */
 162 static struct DhtPutActivity *ma_tail;
 163
 164 /**
 165  * Head of iteration put activities; kept in a DLL.
 166  */
 167 static struct DhtPutActivity *it_head;
 168
 169 /**
 170  * Tail of iteration put activities; kept in a DLL.
 171  */
 172 static struct DhtPutActivity *it_tail;
 173
 174 /**
 175  * Number of entries in the DHT queue.
 176  */
 177 static unsigned int dht_queue_length;
 178
 179 /**
 180  * Useful for zone update for DHT put
 181  */
 182 static unsigned long long num_public_records;
 183
 184 /**
 185  * Last seen record count
 186  */
 187 static unsigned long long last_num_public_records;
 188
 189 /**
 190  * Number of successful put operations performed in the current
 191  * measurement cycle (as measured in #check_zone_namestore_next()).
 192  */
 193 static unsigned long long put_cnt;
 194
 195 /**
 196  * What is the frequency at which we currently would like
 197  * to perform DHT puts (per record)?  Calculated in
 198  * update_velocity() from the #zone_publish_time_window()
 199  * and the total number of record sets we have (so far)
 200  * observed in the zone.
 201  */
 202 static struct GNUNET_TIME_Relative next_put_interval;
 203
 204 /**
 205  * Minimum relative expiration time of records seem during the current
 206  * zone iteration.
 207  */
 208 static struct GNUNET_TIME_Relative min_relative_record_time;
 209
 210 /**
 211  * Minimum relative expiration time of records seem during the last
 212  * zone iteration.
 213  */
 214 static struct GNUNET_TIME_Relative last_min_relative_record_time;
 215
 216 /**
 217  * Default time window for zone iteration
 218  */
 219 static struct GNUNET_TIME_Relative zone_publish_time_window_default;
 220
 221 /**
 222  * Time window for zone iteration, adjusted based on relative record
 223  * expiration times in our zone.
 224  */
 225 static struct GNUNET_TIME_Relative zone_publish_time_window;
 226
 227 /**
 228  * When did we last start measuring the #DELTA_INTERVAL successful
 229  * DHT puts? Used for velocity calculations.
 230  */
 231 static struct GNUNET_TIME_Absolute last_put_100;
 232
 233 /**
 234  * By how much should we try to increase our per-record iteration speed
 235  * (over the desired speed calculated directly from the #put_interval)?
 236  * Basically this value corresponds to the per-record CPU time overhead
 237  * we have.
 238  */
 239 static struct GNUNET_TIME_Relative sub_delta;
 240
 241 /**
 242  * zone publish task
 243  */
 244 static struct GNUNET_SCHEDULER_Task *zone_publish_task;
 245
 246 /**
 247  * How many more values are left for the current query before we need
 248  * to explicitly ask the namestore for more?
 249  */
 250 static unsigned int ns_iteration_left;
 251
 252 /**
 253  * #GNUNET_YES if zone has never been published before
 254  */
 255 static int first_zone_iteration;
 256
 257 /**
 258  * Optimize block insertion by caching map of private keys to
 259  * public keys in memory?
 260  */
 261 static int cache_keys;
 262
 263
 264 /**
 265  * Task run during shutdown.
 266  *
 267  * @param cls unused
 268  * @param tc unused
 269  */
 270 static void
 271 shutdown_task (void *cls)
 272 {
 273   struct DhtPutActivity *ma;
 274
 275   (void) cls;
 276   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 277               "Shutting down!\n");
 278   while (NULL != (ma = ma_head))
 279   {
 280     GNUNET_DHT_put_cancel (ma->ph);
 281     GNUNET_CONTAINER_DLL_remove (ma_head,
 282                                  ma_tail,
 283                                  ma);
 284     GNUNET_free (ma);
 285   }
 286   while (NULL != (ma = it_head))
 287   {
 288     GNUNET_DHT_put_cancel (ma->ph);
 289     GNUNET_CONTAINER_DLL_remove (it_head,
 290                                  it_tail,
 291                                  ma);
 292     dht_queue_length--;
 293     GNUNET_free (ma);
 294   }
 295   if (NULL != statistics)
 296   {
 297     GNUNET_STATISTICS_destroy (statistics,
 298                                GNUNET_NO);
 299     statistics = NULL;
 300   }
 301   if (NULL != zone_publish_task)
 302   {
 303     GNUNET_SCHEDULER_cancel (zone_publish_task);
 304     zone_publish_task = NULL;
 305   }
 306   if (NULL != namestore_iter)
 307   {
 308     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 309     namestore_iter = NULL;
 310   }
 311   if (NULL != zmon)
 312   {
 313     GNUNET_NAMESTORE_zone_monitor_stop (zmon);
 314     zmon = NULL;
 315   }
 316   if (NULL != namestore_handle)
 317   {
 318     GNUNET_NAMESTORE_disconnect (namestore_handle);
 319     namestore_handle = NULL;
 320   }
 321   if (NULL != active_put)
 322   {
 323     GNUNET_DHT_put_cancel (active_put);
 324     active_put = NULL;
 325   }
 326   if (NULL != dht_handle)
 327   {
 328     GNUNET_DHT_disconnect (dht_handle);
 329     dht_handle = NULL;
 330   }
 331 }
 332
 333
 334 /**
 335  * Method called periodically that triggers iteration over authoritative records
 336  *
 337  * @param cls NULL
 338  */
 339 static void
 340 publish_zone_namestore_next (void *cls)
 341 {
 342   (void) cls;
 343   zone_publish_task = NULL;
 344   GNUNET_assert (NULL != namestore_iter);
 345   GNUNET_assert (0 == ns_iteration_left);
 346   ns_iteration_left = NS_BLOCK_SIZE;
 347   GNUNET_NAMESTORE_zone_iterator_next (namestore_iter,
 348                                        NS_BLOCK_SIZE);
 349 }
 350
 351
 352 /**
 353  * Periodically iterate over our zone and store everything in dht
 354  *
 355  * @param cls NULL
 356  */
 357 static void
 358 publish_zone_dht_start (void *cls);
 359
 360
 361 /**
 362  * Continuation called from DHT once the PUT operation triggered
 363  * by a monitor is done.
 364  *
 365  * @param cls a `struct DhtPutActivity`
 366  */
 367 static void
 368 dht_put_monitor_continuation (void *cls)
 369 {
 370   struct DhtPutActivity *ma = cls;
 371
 372   num_public_records++;
 373   GNUNET_CONTAINER_DLL_remove (ma_head,
 374                                ma_tail,
 375                                ma);
 376   GNUNET_free (ma);
 377 }
 378
 379
 380 /**
 381  * Check if the current zone iteration needs to be continued
 382  * by calling #publish_zone_namestore_next(), and if so with what delay.
 383  */
 384 static void
 385 check_zone_namestore_next ()
 386 {
 387   struct GNUNET_TIME_Relative delay;
 388
 389   if (0 != ns_iteration_left)
 390     return; /* current NAMESTORE iteration not yet done */
 391   delay = GNUNET_TIME_relative_subtract (next_put_interval,
 392                                          sub_delta);
 393   /* We delay *once* per #NS_BLOCK_SIZE, so we need to multiply the
 394      per-record delay calculated so far with the #NS_BLOCK_SIZE */
 395   delay = GNUNET_TIME_relative_multiply (delay,
 396                                          NS_BLOCK_SIZE);
 397   GNUNET_assert (NULL == zone_publish_task);
 398   GNUNET_STATISTICS_set (statistics,
 399                          "Current artificial NAMESTORE delay (μs)",
 400                          delay.rel_value_us,
 401                          GNUNET_NO);
 402   zone_publish_task = GNUNET_SCHEDULER_add_delayed (delay,
 403                                                     &publish_zone_namestore_next,
 404                                                     NULL);
 405 }
 406
 407
 408 /**
 409  * Calculate #next_put_interval.
 410  */
 411 static void
 412 calculate_put_interval ()
 413 {
 414   if (0 == num_public_records)
 415   {
 416     /**
 417      * If no records are known (startup) or none present
 418      * we can safely set the interval to the value for a single
 419      * record
 420      */
 421     next_put_interval = zone_publish_time_window;
 422     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG | GNUNET_ERROR_TYPE_BULK,
 423                 "No records in namestore database.\n");
 424   }
 425   else
 426   {
 427     last_min_relative_record_time
 428       = GNUNET_TIME_relative_min (last_min_relative_record_time,
 429                                   min_relative_record_time);
 430     zone_publish_time_window
 431       = GNUNET_TIME_relative_min (GNUNET_TIME_relative_divide (last_min_relative_record_time,
 432                                                                PUBLISH_OPS_PER_EXPIRATION),
 433                                   zone_publish_time_window_default);
 434     next_put_interval
 435       = GNUNET_TIME_relative_divide (zone_publish_time_window,
 436                                      last_num_public_records);
 437     next_put_interval
 438       = GNUNET_TIME_relative_min (next_put_interval,
 439                                   MAXIMUM_ZONE_ITERATION_INTERVAL);
 440   }
 441   GNUNET_STATISTICS_set (statistics,
 442                          "Minimum relative record expiration (in ms)",
 443                          last_min_relative_record_time.rel_value_us / 1000LL,
 444                          GNUNET_NO);
 445   GNUNET_STATISTICS_set (statistics,
 446                          "Zone publication time window (in ms)",
 447                          zone_publish_time_window.rel_value_us / 1000LL,
 448                          GNUNET_NO);
 449 }
 450
 451
 452 /**
 453  * Re-calculate our velocity and the desired velocity.
 454  * We have succeeded in making #DELTA_INTERVAL puts, so
 455  * now calculate the new desired delay between puts.
 456  */
 457 static void
 458 update_velocity ()
 459 {
 460   struct GNUNET_TIME_Relative delta;
 461   unsigned long long pct = 0;
 462
 463   /* How fast were we really? */
 464   delta = GNUNET_TIME_absolute_get_duration (last_put_100);
 465   delta.rel_value_us /= DELTA_INTERVAL;
 466   last_put_100 = GNUNET_TIME_absolute_get ();
 467
 468   /* calculate expected frequency */
 469   if ( (num_public_records > last_num_public_records) &&
 470        (GNUNET_NO == first_zone_iteration) )
 471   {
 472     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 473                 "Last record count was lower than current record count.  Reducing interval.\n");
 474     last_num_public_records = num_public_records * LATE_ITERATION_SPEEDUP_FACTOR;
 475     calculate_put_interval ();
 476   }
 477   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 478               "Desired global zone iteration interval is %s/record!\n",
 479               GNUNET_STRINGS_relative_time_to_string (next_put_interval,
 480                                                       GNUNET_YES));
 481
 482   /* Tell statistics actual vs. desired speed */
 483   GNUNET_STATISTICS_set (statistics,
 484                          "Target zone iteration velocity (μs)",
 485                          next_put_interval.rel_value_us,
 486                          GNUNET_NO);
 487   GNUNET_STATISTICS_set (statistics,
 488                          "Current zone iteration velocity (μs)",
 489                          delta.rel_value_us,
 490                          GNUNET_NO);
 491   /* update "sub_delta" based on difference, taking
 492      previous sub_delta into account! */
 493   if (next_put_interval.rel_value_us > delta.rel_value_us)
 494   {
 495     /* We were too fast, reduce sub_delta! */
 496     struct GNUNET_TIME_Relative corr;
 497
 498     corr = GNUNET_TIME_relative_subtract (next_put_interval,
 499                                           delta);
 500     if (sub_delta.rel_value_us > delta.rel_value_us)
 501     {
 502       /* Reduce sub_delta by corr */
 503       sub_delta = GNUNET_TIME_relative_subtract (sub_delta,
 504                                                  corr);
 505     }
 506     else
 507     {
 508       /* We're doing fine with waiting the full time, this
 509          should theoretically only happen if we run at
 510          infinite speed. */
 511       sub_delta = GNUNET_TIME_UNIT_ZERO;
 512     }
 513   }
 514   else if (next_put_interval.rel_value_us < delta.rel_value_us)
 515   {
 516     /* We were too slow, increase sub_delta! */
 517     struct GNUNET_TIME_Relative corr;
 518
 519     corr = GNUNET_TIME_relative_subtract (delta,
 520                                           next_put_interval);
 521     sub_delta = GNUNET_TIME_relative_add (sub_delta,
 522                                           corr);
 523     if (sub_delta.rel_value_us > next_put_interval.rel_value_us)
 524     {
 525       /* CPU overload detected, we cannot go at desired speed,
 526          as this would mean using a negative delay. */
 527       /* compute how much faster we would want to be for
 528          the desired velocity */
 529       if (0 == next_put_interval.rel_value_us)
 530         pct = UINT64_MAX; /* desired speed is infinity ... */
 531       else
 532         pct = (sub_delta.rel_value_us - next_put_interval.rel_value_us) * 100LLU
 533           / next_put_interval.rel_value_us;
 534       sub_delta = next_put_interval;
 535     }
 536   }
 537   GNUNET_STATISTICS_set (statistics,
 538                          "# size of the DHT queue",
 539                          dht_queue_length,
 540                          GNUNET_NO);
 541   GNUNET_STATISTICS_set (statistics,
 542                          "% speed increase needed for target velocity",
 543                          pct,
 544                          GNUNET_NO);
 545   GNUNET_STATISTICS_set (statistics,
 546                          "# records processed in current iteration",
 547                          num_public_records,
 548                          GNUNET_NO);
 549 }
 550
 551
 552 /**
 553  * Continuation called from DHT once the PUT operation is done.
 554  *
 555  * @param cls a `struct DhtPutActivity`
 556  */
 557 static void
 558 dht_put_continuation (void *cls)
 559 {
 560   struct DhtPutActivity *ma = cls;
 561
 562   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 563               "PUT complete\n");
 564   dht_queue_length--;
 565   GNUNET_CONTAINER_DLL_remove (it_head,
 566                                it_tail,
 567                                ma);
 568   GNUNET_free (ma);
 569 }
 570
 571
 572 /**
 573  * Convert namestore records from the internal format to that
 574  * suitable for publication (removes private records, converts
 575  * to absolute expiration time).
 576  *
 577  * @param rd input records
 578  * @param rd_count size of the @a rd and @a rd_public arrays
 579  * @param rd_public where to write the converted records
 580  * @return number of records written to @a rd_public
 581  */
 582 static unsigned int
 583 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 584                             unsigned int rd_count,
 585                             struct GNUNET_GNSRECORD_Data *rd_public)
 586 {
 587   struct GNUNET_TIME_Absolute now;
 588   unsigned int rd_public_count;
 589
 590   rd_public_count = 0;
 591   now = GNUNET_TIME_absolute_get ();
 592   for (unsigned int i=0;i<rd_count;i++)
 593     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 594     {
 595       rd_public[rd_public_count] = rd[i];
 596       if (0 != (rd[i].flags & GNUNET_GNSRECORD_RF_RELATIVE_EXPIRATION))
 597       {
 598         /* GNUNET_GNSRECORD_block_create will convert to absolute time;
 599            we just need to adjust our iteration frequency */
 600         min_relative_record_time.rel_value_us =
 601           GNUNET_MIN (rd_public[rd_public_count].expiration_time,
 602                       min_relative_record_time.rel_value_us);
 603       }
 604       else if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
 605       {
 606         /* record already expired, skip it */
 607         continue;
 608       }
 609       rd_public_count++;
 610     }
 611   return rd_public_count;
 612 }
 613
 614
 615 /**
 616  * Store GNS records in the DHT.
 617  *
 618  * @param key key of the zone
 619  * @param label label to store under
 620  * @param rd_public public record data
 621  * @param rd_public_count number of records in @a rd_public
 622  * @param cont function to call with PUT result
 623  * @param cont_cls closure for @a cont
 624  * @return DHT PUT handle, NULL on error
 625  */
 626 static struct GNUNET_DHT_PutHandle *
 627 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 628                  const char *label,
 629                  const struct GNUNET_GNSRECORD_Data *rd_public,
 630                  unsigned int rd_public_count,
 631                  GNUNET_SCHEDULER_TaskCallback cont,
 632                  void *cont_cls)
 633 {
 634   struct GNUNET_GNSRECORD_Block *block;
 635   struct GNUNET_HashCode query;
 636   struct GNUNET_TIME_Absolute expire;
 637   size_t block_size;
 638   struct GNUNET_DHT_PutHandle *ret;
 639
 640   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 641                                                         rd_public);
 642   if (cache_keys)
 643     block = GNUNET_GNSRECORD_block_create2 (key,
 644                                             expire,
 645                                             label,
 646                                             rd_public,
 647                                             rd_public_count);
 648   else
 649     block = GNUNET_GNSRECORD_block_create (key,
 650                                            expire,
 651                                            label,
 652                                            rd_public,
 653                                            rd_public_count);
 654   if (NULL == block)
 655   {
 656     GNUNET_break (0);
 657     return NULL; /* whoops */
 658   }
 659   block_size = ntohl (block->purpose.size)
 660     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 661     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 662   GNUNET_GNSRECORD_query_from_private_key (key,
 663                                            label,
 664                                            &query);
 665   GNUNET_STATISTICS_update (statistics,
 666                             "DHT put operations initiated",
 667                             1,
 668                             GNUNET_NO);
 669   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 670               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 671               rd_public_count,
 672               label,
 673               GNUNET_STRINGS_absolute_time_to_string (expire),
 674               GNUNET_h2s (&query));
 675   num_public_records++;
 676   ret = GNUNET_DHT_put (dht_handle,
 677                         &query,
 678                         DHT_GNS_REPLICATION_LEVEL,
 679                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 680                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 681                         block_size,
 682                         block,
 683                         expire,
 684                         cont,
 685                         cont_cls);
 686   GNUNET_free (block);
 687   return ret;
 688 }
 689
 690
 691 /**
 692  * We encountered an error in our zone iteration.
 693  *
 694  * @param cls NULL
 695  */
 696 static void
 697 zone_iteration_error (void *cls)
 698 {
 699   (void) cls;
 700   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 701               "Got disconnected from namestore database, retrying.\n");
 702   namestore_iter = NULL;
 703   /* We end up here on error/disconnect/shutdown, so potentially
 704      while a zone publish task or a DHT put is still running; hence
 705      we need to cancel those. */
 706   if (NULL != zone_publish_task)
 707   {
 708     GNUNET_SCHEDULER_cancel (zone_publish_task);
 709     zone_publish_task = NULL;
 710   }
 711   if (NULL != active_put)
 712   {
 713     GNUNET_DHT_put_cancel (active_put);
 714     active_put = NULL;
 715   }
 716   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 717                                                 NULL);
 718 }
 719
 720
 721 /**
 722  * Zone iteration is completed.
 723  *
 724  * @param cls NULL
 725  */
 726 static void
 727 zone_iteration_finished (void *cls)
 728 {
 729   (void) cls;
 730   /* we're done with one iteration, calculate when to do the next one */
 731   namestore_iter = NULL;
 732   last_num_public_records = num_public_records;
 733   first_zone_iteration = GNUNET_NO;
 734   last_min_relative_record_time = min_relative_record_time;
 735   calculate_put_interval ();
 736   /* reset for next iteration */
 737   min_relative_record_time
 738     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 739                                      PUBLISH_OPS_PER_EXPIRATION);
 740   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 741               "Zone iteration finished. Adjusted zone iteration interval to %s\n",
 742               GNUNET_STRINGS_relative_time_to_string (next_put_interval,
 743                                                       GNUNET_YES));
 744   GNUNET_STATISTICS_set (statistics,
 745                          "Current zone iteration interval (in ms)",
 746                          next_put_interval.rel_value_us / 1000LL,
 747                          GNUNET_NO);
 748   GNUNET_STATISTICS_set (statistics,
 749                          "Number of public records in DHT",
 750                          last_num_public_records,
 751                          GNUNET_NO);
 752   GNUNET_assert (NULL == zone_publish_task);
 753   if (0 == last_num_public_records)
 754     zone_publish_task = GNUNET_SCHEDULER_add_delayed (next_put_interval,
 755                                                       &publish_zone_dht_start,
 756                                                       NULL);
 757   else
 758     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 759                                                   NULL);
 760 }
 761
 762
 763 /**
 764  * Function used to put all records successively into the DHT.
 765  *
 766  * @param cls the closure (NULL)
 767  * @param key the private key of the authority (ours)
 768  * @param label the name of the records, NULL once the iteration is done
 769  * @param rd_count the number of records in @a rd
 770  * @param rd the record data
 771  */
 772 static void
 773 put_gns_record (void *cls,
 774                 const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 775                 const char *label,
 776                 unsigned int rd_count,
 777                 const struct GNUNET_GNSRECORD_Data *rd)
 778 {
 779   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 780   unsigned int rd_public_count;
 781   struct DhtPutActivity *ma;
 782
 783   (void) cls;
 784   ns_iteration_left--;
 785   rd_public_count = convert_records_for_export (rd,
 786                                                 rd_count,
 787                                                 rd_public);
 788   if (0 == rd_public_count)
 789   {
 790     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 791                 "Record set empty, moving to next record set\n");
 792     check_zone_namestore_next ();
 793     return;
 794   }
 795   /* We got a set of records to publish */
 796   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 797               "Starting DHT PUT\n");
 798   ma = GNUNET_new (struct DhtPutActivity);
 799   ma->start_date = GNUNET_TIME_absolute_get ();
 800   ma->ph = perform_dht_put (key,
 801                             label,
 802                             rd_public,
 803                             rd_public_count,
 804                             &dht_put_continuation,
 805                             ma);
 806   put_cnt++;
 807   if (0 == put_cnt % DELTA_INTERVAL)
 808     update_velocity ();
 809   check_zone_namestore_next ();
 810   if (NULL == ma->ph)
 811   {
 812     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 813                 "Could not perform DHT PUT, is the DHT running?\n");
 814     GNUNET_free (ma);
 815     return;
 816   }
 817   dht_queue_length++;
 818   GNUNET_CONTAINER_DLL_insert_tail (it_head,
 819                                     it_tail,
 820                                     ma);
 821   if (dht_queue_length > DHT_QUEUE_LIMIT)
 822   {
 823     ma = it_head;
 824     GNUNET_CONTAINER_DLL_remove (it_head,
 825                                  it_tail,
 826                                  ma);
 827     GNUNET_DHT_put_cancel (ma->ph);
 828     dht_queue_length--;
 829     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 830                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 831                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 832                                                         GNUNET_YES));
 833     GNUNET_free (ma);
 834   }
 835 }
 836
 837
 838 /**
 839  * Periodically iterate over all zones and store everything in DHT
 840  *
 841  * @param cls NULL
 842  */
 843 static void
 844 publish_zone_dht_start (void *cls)
 845 {
 846   (void) cls;
 847   zone_publish_task = NULL;
 848   GNUNET_STATISTICS_update (statistics,
 849                             "Full zone iterations launched",
 850                             1,
 851                             GNUNET_NO);
 852   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 853               "Starting DHT zone update!\n");
 854   /* start counting again */
 855   num_public_records = 0;
 856   GNUNET_assert (NULL == namestore_iter);
 857   ns_iteration_left = 1;
 858   namestore_iter
 859     = GNUNET_NAMESTORE_zone_iteration_start (namestore_handle,
 860                                              NULL, /* All zones */
 861                                              &zone_iteration_error,
 862                                              NULL,
 863                                              &put_gns_record,
 864                                              NULL,
 865                                              &zone_iteration_finished,
 866                                              NULL);
 867   GNUNET_assert (NULL != namestore_iter);
 868 }
 869
 870
 871 /**
 872  * Process a record that was stored in the namestore
 873  * (invoked by the monitor).
 874  *
 875  * @param cls closure, NULL
 876  * @param zone private key of the zone; NULL on disconnect
 877  * @param label label of the records; NULL on disconnect
 878  * @param rd_count number of entries in @a rd array, 0 if label was deleted
 879  * @param rd array of records with data to store
 880  */
 881 static void
 882 handle_monitor_event (void *cls,
 883                       const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
 884                       const char *label,
 885                       unsigned int rd_count,
 886                       const struct GNUNET_GNSRECORD_Data *rd)
 887 {
 888   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 889   unsigned int rd_public_count;
 890   struct DhtPutActivity *ma;
 891
 892   (void) cls;
 893   GNUNET_STATISTICS_update (statistics,
 894                             "Namestore monitor events received",
 895                             1,
 896                             GNUNET_NO);
 897   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 898               "Received %u records for label `%s' via namestore monitor\n",
 899               rd_count,
 900               label);
 901   /* filter out records that are not public, and convert to
 902      absolute expiration time. */
 903   rd_public_count = convert_records_for_export (rd,
 904                                                 rd_count,
 905                                                 rd_public);
 906   if (0 == rd_public_count)
 907     return; /* nothing to do */
 908   ma = GNUNET_new (struct DhtPutActivity);
 909   ma->start_date = GNUNET_TIME_absolute_get ();
 910   ma->ph = perform_dht_put (zone,
 911                             label,
 912                             rd,
 913                             rd_count,
 914                             &dht_put_monitor_continuation,
 915                             ma);
 916   if (NULL == ma->ph)
 917   {
 918     /* PUT failed, do not remember operation */
 919     GNUNET_free (ma);
 920     return;
 921   }
 922   GNUNET_CONTAINER_DLL_insert (ma_head,
 923                                ma_tail,
 924                                ma);
 925 }
 926
 927
 928 /**
 929  * The zone monitor is now in SYNC with the current state of the
 930  * name store.  Start to perform periodic iterations.
 931  *
 932  * @param cls NULL
 933  */
 934 static void
 935 monitor_sync_event (void *cls)
 936 {
 937   (void) cls;
 938   if ( (NULL == zone_publish_task) &&
 939        (NULL == namestore_iter) )
 940     zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 941                                                   NULL);
 942 }
 943
 944
 945 /**
 946  * The zone monitor encountered an IPC error trying to to get in
 947  * sync. Restart from the beginning.
 948  *
 949  * @param cls NULL
 950  */
 951 static void
 952 handle_monitor_error (void *cls)
 953 {
 954   (void) cls;
 955   GNUNET_STATISTICS_update (statistics,
 956                             "Namestore monitor errors encountered",
 957                             1,
 958                             GNUNET_NO);
 959   if (NULL != zone_publish_task)
 960   {
 961     GNUNET_SCHEDULER_cancel (zone_publish_task);
 962     zone_publish_task = NULL;
 963   }
 964   if (NULL != namestore_iter)
 965   {
 966     GNUNET_NAMESTORE_zone_iteration_stop (namestore_iter);
 967     namestore_iter = NULL;
 968   }
 969   if (NULL != active_put)
 970   {
 971     GNUNET_DHT_put_cancel (active_put);
 972     active_put = NULL;
 973   }
 974   zone_publish_task = GNUNET_SCHEDULER_add_now (&publish_zone_dht_start,
 975                                                 NULL);
 976 }
 977
 978
 979 /**
 980  * Performe zonemaster duties: watch namestore, publish records.
 981  *
 982  * @param cls closure
 983  * @param server the initialized server
 984  * @param c configuration to use
 985  */
 986 static void
 987 run (void *cls,
 988      const struct GNUNET_CONFIGURATION_Handle *c,
 989      struct GNUNET_SERVICE_Handle *service)
 990 {
 991   unsigned long long max_parallel_bg_queries = 128;
 992
 993   (void) cls;
 994   (void) service;
 995   last_put_100 = GNUNET_TIME_absolute_get (); /* first time! */
 996   min_relative_record_time
 997     = GNUNET_TIME_relative_multiply (GNUNET_DHT_DEFAULT_REPUBLISH_FREQUENCY,
 998                                      PUBLISH_OPS_PER_EXPIRATION);
 999   next_put_interval = INITIAL_PUT_INTERVAL;
1000   namestore_handle = GNUNET_NAMESTORE_connect (c);
1001   if (NULL == namestore_handle)
1002   {
1003     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1004                 _("Failed to connect to the namestore!\n"));
1005     GNUNET_SCHEDULER_shutdown ();
1006     return;
1007   }
1008   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
1009                                                      "namestore",
1010                                                      "CACHE_KEYS");
1011   zone_publish_time_window_default = DEFAULT_ZONE_PUBLISH_TIME_WINDOW;
1012   if (GNUNET_OK ==
1013       GNUNET_CONFIGURATION_get_value_time (c,
1014                                            "zonemaster",
1015                                            "ZONE_PUBLISH_TIME_WINDOW",
1016                                            &zone_publish_time_window_default))
1017   {
1018     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1019                 "Time window for zone iteration: %s\n",
1020                 GNUNET_STRINGS_relative_time_to_string (zone_publish_time_window,
1021                                                         GNUNET_YES));
1022   }
1023   zone_publish_time_window = zone_publish_time_window_default;
1024   if (GNUNET_OK ==
1025       GNUNET_CONFIGURATION_get_value_number (c,
1026                                              "zonemaster",
1027                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
1028                                              &max_parallel_bg_queries))
1029   {
1030     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1031                 "Number of allowed parallel background queries: %llu\n",
1032                 max_parallel_bg_queries);
1033   }
1034   if (0 == max_parallel_bg_queries)
1035     max_parallel_bg_queries = 1;
1036   dht_handle = GNUNET_DHT_connect (c,
1037                                    (unsigned int) max_parallel_bg_queries);
1038   if (NULL == dht_handle)
1039   {
1040     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1041                 _("Could not connect to DHT!\n"));
1042     GNUNET_SCHEDULER_add_now (&shutdown_task, NULL);
1043     return;
1044   }
1045
1046   /* Schedule periodic put for our records. */
1047   first_zone_iteration = GNUNET_YES;\
1048   statistics = GNUNET_STATISTICS_create ("zonemaster",
1049                                          c);
1050   zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
1051                                               NULL,
1052                                               GNUNET_NO,
1053                                               &handle_monitor_error,
1054                                               NULL,
1055                                               &handle_monitor_event,
1056                                               NULL,
1057                                               &monitor_sync_event,
1058                                               NULL);
1059   GNUNET_break (NULL != zmon);
1060   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
1061                                  NULL);
1062 }
1063
1064
1065 /**
1066  * Define "main" method using service macro.
1067  */
1068 GNUNET_SERVICE_MAIN
1069 ("zonemaster",
1070  GNUNET_SERVICE_OPTION_NONE,
1071  &run,
1072  NULL,
1073  NULL,
1074  NULL,
1075  GNUNET_MQ_handler_end());
1076
1077
1078 /* end of gnunet-service-zonemaster.c */