src/zonemaster/gnunet-service-zonemaster-monitor.c

   1 /*
   2      This file is part of GNUnet.
   3      Copyright (C) 2012, 2013, 2014, 2017, 2018 GNUnet e.V.
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 3, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
  18      Boston, MA 02110-1301, USA.
  19 */
  20
  21 /**
  22  * @file zonemaster/gnunet-service-zonemaster-monitor.c
  23  * @brief monitor namestore changes and publish them immediately to GNUnet name system
  24  * @author Christian Grothoff
  25  */
  26 #include "platform.h"
  27 #include "gnunet_util_lib.h"
  28 #include "gnunet_dht_service.h"
  29 #include "gnunet_namestore_service.h"
  30 #include "gnunet_statistics_service.h"
  31
  32
  33 #define LOG_STRERROR_FILE(kind,syscall,filename) GNUNET_log_from_strerror_file (kind, "util", syscall, filename)
  34
  35
  36 /**
  37  * How often should we (re)publish each record before
  38  * it expires?
  39  */
  40 #define PUBLISH_OPS_PER_EXPIRATION 4
  41
  42 /**
  43  * How many pending DHT operations do we allow at most?
  44  */
  45 #define DHT_QUEUE_LIMIT 2000
  46
  47 /**
  48  * How many events may the namestore give us before it has to wait
  49  * for us to keep up?
  50  */
  51 #define NAMESTORE_QUEUE_LIMIT 50
  52
  53 /**
  54  * What replication level do we use for DHT PUT operations?
  55  */
  56 #define DHT_GNS_REPLICATION_LEVEL 5
  57
  58
  59 /**
  60  * Handle for DHT PUT activity triggered from the namestore monitor.
  61  */
  62 struct DhtPutActivity
  63 {
  64   /**
  65    * Kept in a DLL.
  66    */
  67   struct DhtPutActivity *next;
  68
  69   /**
  70    * Kept in a DLL.
  71    */
  72   struct DhtPutActivity *prev;
  73
  74   /**
  75    * Handle for the DHT PUT operation.
  76    */
  77   struct GNUNET_DHT_PutHandle *ph;
  78
  79   /**
  80    * When was this PUT initiated?
  81    */
  82   struct GNUNET_TIME_Absolute start_date;
  83 };
  84
  85
  86 /**
  87  * Handle to the statistics service
  88  */
  89 static struct GNUNET_STATISTICS_Handle *statistics;
  90
  91 /**
  92  * Our handle to the DHT
  93  */
  94 static struct GNUNET_DHT_Handle *dht_handle;
  95
  96 /**
  97  * Our handle to the namestore service
  98  */
  99 static struct GNUNET_NAMESTORE_Handle *namestore_handle;
 100
 101 /**
 102  * Handle to monitor namestore changes to instant propagation.
 103  */
 104 static struct GNUNET_NAMESTORE_ZoneMonitor *zmon;
 105
 106 /**
 107  * Head of monitor activities; kept in a DLL.
 108  */
 109 static struct DhtPutActivity *ma_head;
 110
 111 /**
 112  * Tail of monitor activities; kept in a DLL.
 113  */
 114 static struct DhtPutActivity *ma_tail;
 115
 116 /**
 117  * Number of entries in the DHT queue #ma_head.
 118  */
 119 static unsigned int ma_queue_length;
 120
 121 /**
 122  * Optimize block insertion by caching map of private keys to
 123  * public keys in memory?
 124  */
 125 static int cache_keys;
 126
 127
 128 /**
 129  * Task run during shutdown.
 130  *
 131  * @param cls unused
 132  * @param tc unused
 133  */
 134 static void
 135 shutdown_task (void *cls)
 136 {
 137   struct DhtPutActivity *ma;
 138
 139   (void) cls;
 140   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 141               "Shutting down!\n");
 142   while (NULL != (ma = ma_head))
 143   {
 144     GNUNET_DHT_put_cancel (ma->ph);
 145     ma_queue_length--;
 146     GNUNET_CONTAINER_DLL_remove (ma_head,
 147                                  ma_tail,
 148                                  ma);
 149     GNUNET_free (ma);
 150   }
 151   if (NULL != statistics)
 152   {
 153     GNUNET_STATISTICS_destroy (statistics,
 154                                GNUNET_NO);
 155     statistics = NULL;
 156   }
 157   if (NULL != zmon)
 158   {
 159     GNUNET_NAMESTORE_zone_monitor_stop (zmon);
 160     zmon = NULL;
 161   }
 162   if (NULL != namestore_handle)
 163   {
 164     GNUNET_NAMESTORE_disconnect (namestore_handle);
 165     namestore_handle = NULL;
 166   }
 167   if (NULL != dht_handle)
 168   {
 169     GNUNET_DHT_disconnect (dht_handle);
 170     dht_handle = NULL;
 171   }
 172 }
 173
 174
 175 /**
 176  * Continuation called from DHT once the PUT operation triggered
 177  * by a monitor is done.
 178  *
 179  * @param cls a `struct DhtPutActivity`
 180  */
 181 static void
 182 dht_put_monitor_continuation (void *cls)
 183 {
 184   struct DhtPutActivity *ma = cls;
 185
 186   GNUNET_NAMESTORE_zone_monitor_next (zmon,
 187                                       1);
 188   ma_queue_length--;
 189   GNUNET_CONTAINER_DLL_remove (ma_head,
 190                                ma_tail,
 191                                ma);
 192   GNUNET_free (ma);
 193 }
 194
 195
 196 /**
 197  * Convert namestore records from the internal format to that
 198  * suitable for publication (removes private records, converts
 199  * to absolute expiration time).
 200  *
 201  * @param rd input records
 202  * @param rd_count size of the @a rd and @a rd_public arrays
 203  * @param rd_public where to write the converted records
 204  * @return number of records written to @a rd_public
 205  */
 206 static unsigned int
 207 convert_records_for_export (const struct GNUNET_GNSRECORD_Data *rd,
 208                             unsigned int rd_count,
 209                             struct GNUNET_GNSRECORD_Data *rd_public)
 210 {
 211   struct GNUNET_TIME_Absolute now;
 212   unsigned int rd_public_count;
 213
 214   rd_public_count = 0;
 215   now = GNUNET_TIME_absolute_get ();
 216   for (unsigned int i=0;i<rd_count;i++)
 217     if (0 == (rd[i].flags & GNUNET_GNSRECORD_RF_PRIVATE))
 218     {
 219       rd_public[rd_public_count] = rd[i];
 220       if (rd_public[rd_public_count].expiration_time < now.abs_value_us)
 221       {
 222         /* record already expired, skip it */
 223         continue;
 224       }
 225       rd_public_count++;
 226     }
 227   return rd_public_count;
 228 }
 229
 230
 231 /**
 232  * Store GNS records in the DHT.
 233  *
 234  * @param key key of the zone
 235  * @param label label to store under
 236  * @param rd_public public record data
 237  * @param rd_public_count number of records in @a rd_public
 238  * @param cont function to call with PUT result
 239  * @param cont_cls closure for @a cont
 240  * @return DHT PUT handle, NULL on error
 241  */
 242 static struct GNUNET_DHT_PutHandle *
 243 perform_dht_put (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key,
 244                  const char *label,
 245                  const struct GNUNET_GNSRECORD_Data *rd_public,
 246                  unsigned int rd_public_count,
 247                  GNUNET_SCHEDULER_TaskCallback cont,
 248                  void *cont_cls)
 249 {
 250   struct GNUNET_GNSRECORD_Block *block;
 251   struct GNUNET_HashCode query;
 252   struct GNUNET_TIME_Absolute expire;
 253   size_t block_size;
 254   struct GNUNET_DHT_PutHandle *ret;
 255
 256   expire = GNUNET_GNSRECORD_record_get_expiration_time (rd_public_count,
 257                                                         rd_public);
 258   if (cache_keys)
 259     block = GNUNET_GNSRECORD_block_create2 (key,
 260                                             expire,
 261                                             label,
 262                                             rd_public,
 263                                             rd_public_count);
 264   else
 265     block = GNUNET_GNSRECORD_block_create (key,
 266                                            expire,
 267                                            label,
 268                                            rd_public,
 269                                            rd_public_count);
 270   if (NULL == block)
 271   {
 272     GNUNET_break (0);
 273     return NULL; /* whoops */
 274   }
 275   block_size = ntohl (block->purpose.size)
 276     + sizeof (struct GNUNET_CRYPTO_EcdsaSignature)
 277     + sizeof (struct GNUNET_CRYPTO_EcdsaPublicKey);
 278   GNUNET_GNSRECORD_query_from_private_key (key,
 279                                            label,
 280                                            &query);
 281   GNUNET_STATISTICS_update (statistics,
 282                             "DHT put operations initiated",
 283                             1,
 284                             GNUNET_NO);
 285   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 286               "Storing %u record(s) for label `%s' in DHT with expiration `%s' under key %s\n",
 287               rd_public_count,
 288               label,
 289               GNUNET_STRINGS_absolute_time_to_string (expire),
 290               GNUNET_h2s (&query));
 291   ret = GNUNET_DHT_put (dht_handle,
 292                         &query,
 293                         DHT_GNS_REPLICATION_LEVEL,
 294                         GNUNET_DHT_RO_DEMULTIPLEX_EVERYWHERE,
 295                         GNUNET_BLOCK_TYPE_GNS_NAMERECORD,
 296                         block_size,
 297                         block,
 298                         expire,
 299                         cont,
 300                         cont_cls);
 301   GNUNET_free (block);
 302   return ret;
 303 }
 304
 305
 306 /**
 307  * Process a record that was stored in the namestore
 308  * (invoked by the monitor).
 309  *
 310  * @param cls closure, NULL
 311  * @param zone private key of the zone; NULL on disconnect
 312  * @param label label of the records; NULL on disconnect
 313  * @param rd_count number of entries in @a rd array, 0 if label was deleted
 314  * @param rd array of records with data to store
 315  */
 316 static void
 317 handle_monitor_event (void *cls,
 318                       const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone,
 319                       const char *label,
 320                       unsigned int rd_count,
 321                       const struct GNUNET_GNSRECORD_Data *rd)
 322 {
 323   struct GNUNET_GNSRECORD_Data rd_public[rd_count];
 324   unsigned int rd_public_count;
 325   struct DhtPutActivity *ma;
 326
 327   (void) cls;
 328   GNUNET_STATISTICS_update (statistics,
 329                             "Namestore monitor events received",
 330                             1,
 331                             GNUNET_NO);
 332   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 333               "Received %u records for label `%s' via namestore monitor\n",
 334               rd_count,
 335               label);
 336   /* filter out records that are not public, and convert to
 337      absolute expiration time. */
 338   rd_public_count = convert_records_for_export (rd,
 339                                                 rd_count,
 340                                                 rd_public);
 341   if (0 == rd_public_count)
 342   {
 343     GNUNET_NAMESTORE_zone_monitor_next (zmon,
 344                                         1);
 345     return; /* nothing to do */
 346   }
 347   ma = GNUNET_new (struct DhtPutActivity);
 348   ma->start_date = GNUNET_TIME_absolute_get ();
 349   ma->ph = perform_dht_put (zone,
 350                             label,
 351                             rd,
 352                             rd_count,
 353                             &dht_put_monitor_continuation,
 354                             ma);
 355   if (NULL == ma->ph)
 356   {
 357     /* PUT failed, do not remember operation */
 358     GNUNET_free (ma);
 359     GNUNET_NAMESTORE_zone_monitor_next (zmon,
 360                                         1);
 361     return;
 362   }
 363   GNUNET_CONTAINER_DLL_insert_tail (ma_head,
 364                                     ma_tail,
 365                                     ma);
 366   ma_queue_length++;
 367   if (ma_queue_length > DHT_QUEUE_LIMIT)
 368   {
 369     ma = ma_head;
 370     GNUNET_CONTAINER_DLL_remove (ma_head,
 371                                  ma_tail,
 372                                  ma);
 373     GNUNET_DHT_put_cancel (ma->ph);
 374     ma_queue_length--;
 375     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 376                 "DHT PUT unconfirmed after %s, aborting PUT\n",
 377                 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_duration (ma->start_date),
 378                                                         GNUNET_YES));
 379     GNUNET_free (ma);
 380   }
 381 }
 382
 383
 384 /**
 385  * The zone monitor encountered an IPC error trying to to get in
 386  * sync. Restart from the beginning.
 387  *
 388  * @param cls NULL
 389  */
 390 static void
 391 handle_monitor_error (void *cls)
 392 {
 393   (void) cls;
 394   GNUNET_STATISTICS_update (statistics,
 395                             "Namestore monitor errors encountered",
 396                             1,
 397                             GNUNET_NO);
 398 }
 399
 400
 401 /**
 402  * Performe zonemaster duties: watch namestore, publish records.
 403  *
 404  * @param cls closure
 405  * @param server the initialized server
 406  * @param c configuration to use
 407  */
 408 static void
 409 run (void *cls,
 410      const struct GNUNET_CONFIGURATION_Handle *c,
 411      struct GNUNET_SERVICE_Handle *service)
 412 {
 413   unsigned long long max_parallel_bg_queries = 128;
 414
 415   (void) cls;
 416   (void) service;
 417   namestore_handle = GNUNET_NAMESTORE_connect (c);
 418   if (NULL == namestore_handle)
 419   {
 420     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 421                 _("Failed to connect to the namestore!\n"));
 422     GNUNET_SCHEDULER_shutdown ();
 423     return;
 424   }
 425   cache_keys = GNUNET_CONFIGURATION_get_value_yesno (c,
 426                                                      "namestore",
 427                                                      "CACHE_KEYS");
 428   if (GNUNET_OK ==
 429       GNUNET_CONFIGURATION_get_value_number (c,
 430                                              "zonemaster",
 431                                              "MAX_PARALLEL_BACKGROUND_QUERIES",
 432                                              &max_parallel_bg_queries))
 433   {
 434     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 435                 "Number of allowed parallel background queries: %llu\n",
 436                 max_parallel_bg_queries);
 437   }
 438   if (0 == max_parallel_bg_queries)
 439     max_parallel_bg_queries = 1;
 440   dht_handle = GNUNET_DHT_connect (c,
 441                                    (unsigned int) max_parallel_bg_queries);
 442   if (NULL == dht_handle)
 443   {
 444     GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 445                 _("Could not connect to DHT!\n"));
 446     GNUNET_SCHEDULER_add_now (&shutdown_task,
 447                               NULL);
 448     return;
 449   }
 450
 451   /* Schedule periodic put for our records. */
 452   statistics = GNUNET_STATISTICS_create ("zonemaster-mon",
 453                                          c);
 454   zmon = GNUNET_NAMESTORE_zone_monitor_start (c,
 455                                               NULL,
 456                                               GNUNET_NO,
 457                                               &handle_monitor_error,
 458                                               NULL,
 459                                               &handle_monitor_event,
 460                                               NULL,
 461                                               NULL /* sync_cb */,
 462                                               NULL);
 463   GNUNET_NAMESTORE_zone_monitor_next (zmon,
 464                                       NAMESTORE_QUEUE_LIMIT - 1);
 465   GNUNET_break (NULL != zmon);
 466   GNUNET_SCHEDULER_add_shutdown (&shutdown_task,
 467                                  NULL);
 468 }
 469
 470
 471 /**
 472  * Define "main" method using service macro.
 473  */
 474 GNUNET_SERVICE_MAIN
 475 ("zonemaster-monitor",
 476  GNUNET_SERVICE_OPTION_NONE,
 477  &run,
 478  NULL,
 479  NULL,
 480  NULL,
 481  GNUNET_MQ_handler_end());
 482
 483
 484 /* end of gnunet-service-zonemaster-monitor.c */