adding support for detection quota changes / missing bloomfilter files and reconstruc...

diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c
index 327244ffd9b10398df62970bfda64e6987c72faa..8555fad3aeb0c7e3acec35ee97d3d387f736c52c 100644 (file)
--- a/src/datastore/gnunet-service-datastore.c
+++ b/src/datastore/gnunet-service-datastore.c
@@ -1443,6 +1443,24 @@ cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client)
 }
 
 
+/**
+ * Adds a given key to the bloomfilter 'count' times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
+ */
+static void
+add_key_to_bloomfilter (void *cls,
+                       const GNUNET_HashCode *key,
+                       unsigned int count)
+{
+  struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+  while (0 < count--)
+    GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
 /**
  * Process datastore requests.
  *
@@ -1477,6 +1495,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
   };
   char *fn;
   unsigned int bf_size;
+  int refresh_bf;
 
   cfg = c;
   if (GNUNET_OK !=
@@ -1509,9 +1528,58 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
     fn = NULL;
   }
   if (fn != NULL)
-    filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+  {
+    if (GNUNET_YES == GNUNET_DISK_file_test (fn))
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+      if (NULL == filter)
+      {
+       /* file exists but not valid, remove and try again, but refresh */
+       if (0 != UNLINK (fn))
+       {
+         /* failed to remove, run without file */
+         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                     _("Failed to remove bogus bloomfilter file `%s'\n"),
+                     fn);
+         GNUNET_free (fn);
+         fn = NULL;
+         filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);        /* approx. 3% false positives at max use */
+         refresh_bf = GNUNET_YES;
+       }
+       else
+       {
+         /* try again after remove */
+         filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+         refresh_bf = GNUNET_YES;
+         if (NULL == filter)
+         {
+           /* failed yet again, give up on using file */
+           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                       _("Failed to remove bogus bloomfilter file `%s'\n"),
+                       fn);
+           GNUNET_free (fn);
+           fn = NULL;
+           filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5);        /* approx. 3% false positives at max use */
+         }
+       }
+      }
+      else
+      {
+       /* normal case: have an existing valid bf file, no need to refresh */
+       refresh_bf = GNUNET_NO;
+      }
+    }
+    else
+    {
+      filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5);        /* approx. 3% false positives at max use */
+      refresh_bf = GNUNET_YES;
+    }
+  }
   else
+  {
     filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5);      /* approx. 3% false positives at max use */
+    refresh_bf = GNUNET_YES;
+  }
   GNUNET_free_non_null (fn);
   if (filter == NULL)
   {
@@ -1542,6 +1610,19 @@ run (void *cls, struct GNUNET_SERVER_Handle *server,
                              &process_stat_in, plugin);
   GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
   GNUNET_SERVER_add_handlers (server, handlers);
+  if (GNUNET_YES == refresh_bf)
+  {
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Rebuilding bloomfilter.  Please be patient.\n"));
+    if (NULL != plugin->api->get_keys)
+      plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);  
+    else
+      GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+                 _("Plugin does not support get_keys function. Please fix!\n"));
+
+    GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+               _("Bloomfilter construction complete.\n"));
+  }
   expired_kill_task =
       GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
                                           &delete_expired, NULL);

diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h
index fb1dff51d217b167b4fa36661a72f18b683e1d30..bbf0ce2aac0b3b7f201d47ab7b6c5abef79fb9a0 100644 (file)
--- a/src/include/gnunet_datastore_plugin.h
+++ b/src/include/gnunet_datastore_plugin.h
@@ -134,6 +134,29 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
                           struct GNUNET_TIME_Absolute expiration, char **msg);
 
 
+/**
+ * An processor over a set of keys stored in the datastore.
+ *
+ * @param cls closure
+ * @param key key in the data store
+ * @param count how many values are stored under this key in the datastore
+ */
+typedef void (*PluginKeyProcessor) (void *cls, 
+                                  const GNUNET_HashCode *key,
+                                  unsigned int count);
+
+
+/**
+ * Get all of the keys in the datastore.
+ *
+ * @param cls closure
+ * @param proc function to call on each key
+ * @param proc_cls closure for proc
+ */
+typedef void (*PluginGetKeys) (void *cls,
+                              PluginKeyProcessor proc, void *proc_cls);
+
+
 /**
  * Get one of the results for a particular key in the datastore.
  *
@@ -148,6 +171,8 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size,
  *        there may be!
  * @param type entries of which type are relevant?
  *     Use 0 for any type.
+ * @param min find the smallest key that is larger than the given min,
+ *            NULL for no minimum (return smallest key)
  * @param proc function to call on the matching value;
  *        proc should be called with NULL if there is no result
  * @param proc_cls closure for proc
@@ -159,7 +184,6 @@ typedef void (*PluginGetKey) (void *cls, uint64_t offset,
                               PluginDatumProcessor proc, void *proc_cls);
 
 
-
 /**
  * Get a random item (additional constraints may apply depending on
  * the specific implementation).  Calls 'proc' with all values ZERO or
@@ -174,6 +198,8 @@ typedef void (*PluginGetRandom) (void *cls, PluginDatumProcessor proc,
                                  void *proc_cls);
 
 
+
+
 /**
  * Update the priority for a particular key in the datastore.  If
  * the expiration time in value is different than the time found in
@@ -296,6 +322,11 @@ struct GNUNET_DATASTORE_PluginFunctions
    */
   PluginDrop drop;
 
+  /**
+   * Iterate over all keys in the database.
+   */
+  PluginGetKeys get_keys;
+  
 };