X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fdatastore%2Fgnunet-service-datastore.c;h=1d7e8cd2beb68e185c8820c0d929f8df4fe54f72;hb=354a6b2d5484d52bc1dfc25bc176857034a2d3e4;hp=71379d5db3bb1fe6bc50b03dde6c7d843f661b15;hpb=41d0acfe4f85509cb61f20a172a074304d962484;p=oweals%2Fgnunet.git diff --git a/src/datastore/gnunet-service-datastore.c b/src/datastore/gnunet-service-datastore.c index 71379d5db..1d7e8cd2b 100644 --- a/src/datastore/gnunet-service-datastore.c +++ b/src/datastore/gnunet-service-datastore.c @@ -48,8 +48,10 @@ */ #define MIN_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1) - -#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore") +/** + * Name under which we store current space consumption. + */ +static char *quota_stat_name; /** * After how many payload-changing operations @@ -149,6 +151,11 @@ static unsigned long long quota; */ static int do_drop; +/** + * Name of our plugin. + */ +static char *plugin_name; + /** * How much space are we using for the cache? (space available for * insertions that will be instantly reclaimed by discarding less @@ -191,6 +198,13 @@ static GNUNET_SCHEDULER_TaskIdentifier expired_kill_task; */ const struct GNUNET_CONFIGURATION_Handle *cfg; +/** + * Minimum time that content should have to not be discarded instantly + * (time stamp of any content that we've been discarding recently to + * stay below the quota). FOREVER if we had to expire content with + * non-zero priority. + */ +static struct GNUNET_TIME_Absolute min_expiration; /** * Handle for reporting statistics. @@ -205,7 +219,8 @@ static struct GNUNET_STATISTICS_Handle *stats; static void sync_stats () { - GNUNET_STATISTICS_set (stats, QUOTA_STAT_NAME, payload, GNUNET_YES); + GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES); + GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload, GNUNET_NO); lastSync = 0; } @@ -311,7 +326,9 @@ expired_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, if (key == NULL) { expired_kill_task = - GNUNET_SCHEDULER_add_delayed (MAX_EXPIRE_DELAY, &delete_expired, NULL); + GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); return GNUNET_SYSERR; } now = GNUNET_TIME_absolute_get (); @@ -319,7 +336,9 @@ expired_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, { /* finished processing */ expired_kill_task = - GNUNET_SCHEDULER_add_delayed (MAX_EXPIRE_DELAY, &delete_expired, NULL); + GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); return GNUNET_SYSERR; } #if DEBUG_DATASTORE @@ -328,11 +347,14 @@ expired_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, GNUNET_h2s (key), type, (unsigned long long) (now.abs_value - expiration.abs_value)); #endif + min_expiration = now; GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes expired"), size, GNUNET_YES); GNUNET_CONTAINER_bloomfilter_remove (filter, key); expired_kill_task = - GNUNET_SCHEDULER_add_delayed (MIN_EXPIRE_DELAY, &delete_expired, NULL); + GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY, + GNUNET_SCHEDULER_PRIORITY_IDLE, + &delete_expired, NULL); return GNUNET_NO; } @@ -385,14 +407,21 @@ quota_processor (void *cls, const GNUNET_HashCode * key, uint32_t size, return GNUNET_SYSERR; #if DEBUG_DATASTORE GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Deleting %llu bytes of low-priority content `%s' of type %u (still trying to free another %llu bytes)\n", + "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %llu ms prior to expiration (still trying to free another %llu bytes)\n", (unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD), - GNUNET_h2s (key), type, *need); + (unsigned int) priority, + GNUNET_h2s (key), type, + (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).rel_value, + *need); #endif if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need) *need = 0; else *need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD; + if (priority > 0) + min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS; + else + min_expiration = expiration; GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes purged (low-priority)"), size, GNUNET_YES); @@ -535,6 +564,7 @@ transmit_status (struct GNUNET_SERVER_Client *client, int code, const char *msg) sm->header.size = htons (sizeof (struct StatusMessage) + slen); sm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS); sm->status = htonl (code); + sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration); if (slen > 0) memcpy (&sm[1], msg, slen); transmit (client, &sm->header); @@ -1285,31 +1315,21 @@ load_plugin () { struct DatastorePlugin *ret; char *libname; - char *name; - if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", - &name)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", - "DATASTORE"); - return NULL; - } ret = GNUNET_malloc (sizeof (struct DatastorePlugin)); ret->env.cfg = cfg; ret->env.duc = &disk_utilization_change_cb; ret->env.cls = NULL; GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), - name); - GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); - ret->short_name = name; + plugin_name); + GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name); + ret->short_name = GNUNET_strdup (plugin_name); ret->lib_name = libname; ret->api = GNUNET_PLUGIN_load (libname, &ret->env); if (ret->api == NULL) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("Failed to load datastore plugin for `%s'\n"), name); + _("Failed to load datastore plugin for `%s'\n"), plugin_name); GNUNET_free (ret->short_name); GNUNET_free (libname); GNUNET_free (ret); @@ -1336,6 +1356,8 @@ unload_plugin (struct DatastorePlugin *plug) GNUNET_free (plug->lib_name); GNUNET_free (plug->short_name); GNUNET_free (plug); + GNUNET_free (quota_stat_name); + quota_stat_name = NULL; } @@ -1346,6 +1368,8 @@ unload_plugin (struct DatastorePlugin *plug) static void unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { + if (lastSync > 0) + sync_stats (); if (GNUNET_YES == do_drop) plugin->api->drop (plugin->api->cls); unload_plugin (plugin); @@ -1355,8 +1379,6 @@ unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_CONTAINER_bloomfilter_free (filter); filter = NULL; } - if (lastSync > 0) - sync_stats (); if (stat_get != NULL) { GNUNET_STATISTICS_get_cancel (stat_get); @@ -1367,6 +1389,8 @@ unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) GNUNET_STATISTICS_destroy (stats, GNUNET_YES); stats = NULL; } + GNUNET_free_non_null (plugin_name); + plugin_name = NULL; } @@ -1443,6 +1467,24 @@ cleanup_reservations (void *cls, struct GNUNET_SERVER_Client *client) } +/** + * Adds a given key to the bloomfilter 'count' times. + * + * @param cls the bloomfilter + * @param key key to add + * @param count number of times to add key + */ +static void +add_key_to_bloomfilter (void *cls, + const GNUNET_HashCode *key, + unsigned int count) +{ + struct GNUNET_CONTAINER_BloomFilter *bf = cls; + while (0 < count--) + GNUNET_CONTAINER_bloomfilter_add (bf, key); +} + + /** * Process datastore requests. * @@ -1475,12 +1517,26 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, sizeof (struct GNUNET_MessageHeader)}, {NULL, NULL, 0, 0} }; - char *fn; + char *fn; + char *pfn; unsigned int bf_size; + int refresh_bf; cfg = c; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_number (cfg, "DATASTORE", "QUOTA", "a)) + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &plugin_name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } + GNUNET_asprintf ("a_stat_name, + _("# bytes used in file-sharing datastore `%s'"), + plugin_name); + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", "a)) { GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("No `%s' specified for `%s' in configuration!\n"), "QUOTA", @@ -1492,7 +1548,10 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, cache_size = quota / 8; /* Or should we make this an option? */ GNUNET_STATISTICS_set (stats, gettext_noop ("# cache size"), cache_size, GNUNET_NO); - bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */ + if (quota / (32 * 1024LL) > (1 << 31)) + bf_size = (1 << 31); /* absolute limit: ~2 GB, beyond that BF just won't help anyway */ + else + bf_size = quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */ fn = NULL; if ((GNUNET_OK != GNUNET_CONFIGURATION_get_value_filename (cfg, "DATASTORE", "BLOOMFILTER", @@ -1506,9 +1565,60 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, fn = NULL; } if (fn != NULL) - filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */ + { + GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name); + if (GNUNET_YES == GNUNET_DISK_file_test (pfn)) + { + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + if (NULL == filter) + { + /* file exists but not valid, remove and try again, but refresh */ + if (0 != UNLINK (pfn)) + { + /* failed to remove, run without file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + pfn); + GNUNET_free (pfn); + pfn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + else + { + /* try again after remove */ + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + if (NULL == filter) + { + /* failed yet again, give up on using file */ + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Failed to remove bogus bloomfilter file `%s'\n"), + pfn); + GNUNET_free (pfn); + pfn = NULL; + filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + } + } + } + else + { + /* normal case: have an existing valid bf file, no need to refresh */ + refresh_bf = GNUNET_NO; + } + } + else + { + filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } + GNUNET_free (pfn); + } else + { filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */ + refresh_bf = GNUNET_YES; + } GNUNET_free_non_null (fn); if (filter == NULL) { @@ -1534,11 +1644,24 @@ run (void *cls, struct GNUNET_SERVER_Handle *server, return; } stat_get = - GNUNET_STATISTICS_get (stats, "datastore", QUOTA_STAT_NAME, + GNUNET_STATISTICS_get (stats, "datastore", quota_stat_name, GNUNET_TIME_UNIT_SECONDS, &process_stat_done, &process_stat_in, plugin); GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL); GNUNET_SERVER_add_handlers (server, handlers); + if (GNUNET_YES == refresh_bf) + { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Rebuilding bloomfilter. Please be patient.\n")); + if (NULL != plugin->api->get_keys) + plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter); + else + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("Plugin does not support get_keys function. Please fix!\n")); + + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Bloomfilter construction complete.\n")); + } expired_kill_task = GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, &delete_expired, NULL);