From: Christian Grothoff Date: Thu, 15 Dec 2011 14:55:51 +0000 (+0000) Subject: adding support for detection quota changes / missing bloomfilter files and reconstruc... X-Git-Tag: initial-import-from-subversion-38251~15630 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=f04c8750cca2dcfd35aceae3379b83637b8aadfc;p=oweals%2Fgnunet.git adding support for detection quota changes / missing bloomfilter files and reconstruction/recovery code --- diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index 327244ffd..8555fad3a 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c @@ -1443,6 +1443,24 @@ cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client) } +/** + * Adds a given key to the bloomfilter 'count' times. + * + * @param cls the bloomfilter + * @param key key to add + * @param count number of times to add key + */ +static void +add_key_to_bloomfilter (void *cls, + const GNUNET_HashCode *key, + unsigned int count) +{ + struct GNUNET_CONTAINER_BloomFilter *bf = cls; + while (0 < count--) + GNUNET_CONTAINER_bloomfilter_add (bf, key); +} + + /** * Process datastore requests. * @@ -1477,6 +1495,7 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, }; char *fn; unsigned int bf_size; + int refresh_bf; cfg = c; if (GNUNET_OK != @@ -1509,9 +1528,58 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, fn = NULL; } if (fn != NULL) - filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ + { + if (GNUNET_YES == GNUNET_DISK_file_test (fn)) + { + filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ + if (NULL == filter) + { + /* file exists but not valid, remove and try again, but refresh */ + if (0 != UNLINK (fn)) + { + /* failed to remove, run without file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + fn); + GNUNET_free (fn); + fn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + else + { + /* try again after remove */ + filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + if (NULL == filter) + { + /* failed yet again, give up on using file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + fn); + GNUNET_free (fn); + fn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + } + } + } + else + { + /* normal case: have an existing valid bf file, no need to refresh */ + refresh_bf = GNUNET_NO; + } + } + else + { + filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + } else + { filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } GNUNET_free_non_null (fn); if (filter == NULL) { @@ -1542,6 +1610,19 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, &process_stat_in, plugin); GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL); GNUNET_SERVER_add_handlers (server, handlers); + if (GNUNET_YES == refresh_bf) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Rebuilding bloomfilter. Please be patient.\n")); + if (NULL != plugin->api->get_keys) + plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter); + else + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Plugin does not support get_keys function. Please fix!\n")); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Bloomfilter construction complete.\n")); + } expired_kill_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, &delete_expired, NULL); diff --git a/src/include/gnunet_datastore_plugin.h b/src/include/gnunet_datastore_plugin.h index fb1dff51d..bbf0ce2aa 100644 --- a/src/include/gnunet_datastore_plugin.h +++ b/src/include/gnunet_datastore_plugin.h @@ -134,6 +134,29 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size, struct GNUNET_TIME_Absolute expiration, char **msg); +/** + * An processor over a set of keys stored in the datastore. + * + * @param cls closure + * @param key key in the data store + * @param count how many values are stored under this key in the datastore + */ +typedef void (*PluginKeyProcessor) (void *cls, + const GNUNET_HashCode *key, + unsigned int count); + + +/** + * Get all of the keys in the datastore. + * + * @param cls closure + * @param proc function to call on each key + * @param proc_cls closure for proc + */ +typedef void (*PluginGetKeys) (void *cls, + PluginKeyProcessor proc, void *proc_cls); + + /** * Get one of the results for a particular key in the datastore. * @@ -148,6 +171,8 @@ typedef int (*PluginPut) (void *cls, const GNUNET_HashCode * key, uint32_t size, * there may be! * @param type entries of which type are relevant? * Use 0 for any type. + * @param min find the smallest key that is larger than the given min, + * NULL for no minimum (return smallest key) * @param proc function to call on the matching value; * proc should be called with NULL if there is no result * @param proc_cls closure for proc @@ -159,7 +184,6 @@ typedef void (*PluginGetKey) (void *cls, uint64_t offset, PluginDatumProcessor proc, void *proc_cls); - /** * Get a random item (additional constraints may apply depending on * the specific implementation). Calls 'proc' with all values ZERO or @@ -174,6 +198,8 @@ typedef void (*PluginGetRandom) (void *cls, PluginDatumProcessor proc, void *proc_cls); + + /** * Update the priority for a particular key in the datastore. If * the expiration time in value is different than the time found in @@ -296,6 +322,11 @@ struct GNUNET_DATASTORE_PluginFunctions */ PluginDrop drop; + /** + * Iterate over all keys in the database. + */ + PluginGetKeys get_keys; + };