*/
#define MIN_EXPIRE_DELAY GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
-
-#define QUOTA_STAT_NAME gettext_noop ("# bytes used in file-sharing datastore")
+/**
+ * Name under which we store current space consumption.
+ */
+static char *quota_stat_name;
/**
* After how many payload-changing operations
*/
static int do_drop;
+/**
+ * Name of our plugin.
+ */
+static char *plugin_name;
+
/**
* How much space are we using for the cache? (space available for
* insertions that will be instantly reclaimed by discarding less
*/
const struct GNUNET_CONFIGURATION_Handle *cfg;
+/**
+ * Minimum time that content should have to not be discarded instantly
+ * (time stamp of any content that we've been discarding recently to
+ * stay below the quota). FOREVER if we had to expire content with
+ * non-zero priority.
+ */
+static struct GNUNET_TIME_Absolute min_expiration;
/**
* Handle for reporting statistics.
static void
sync_stats ()
{
- GNUNET_STATISTICS_set (stats, QUOTA_STAT_NAME, payload, GNUNET_YES);
+ GNUNET_STATISTICS_set (stats, quota_stat_name, payload, GNUNET_YES);
+ GNUNET_STATISTICS_set (stats, "# utilization by current datastore", payload, GNUNET_NO);
lastSync = 0;
}
if (key == NULL)
{
expired_kill_task =
- GNUNET_SCHEDULER_add_delayed (MAX_EXPIRE_DELAY, &delete_expired, NULL);
+ GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
+ GNUNET_SCHEDULER_PRIORITY_IDLE,
+ &delete_expired, NULL);
return GNUNET_SYSERR;
}
now = GNUNET_TIME_absolute_get ();
{
/* finished processing */
expired_kill_task =
- GNUNET_SCHEDULER_add_delayed (MAX_EXPIRE_DELAY, &delete_expired, NULL);
+ GNUNET_SCHEDULER_add_delayed_with_priority (MAX_EXPIRE_DELAY,
+ GNUNET_SCHEDULER_PRIORITY_IDLE,
+ &delete_expired, NULL);
return GNUNET_SYSERR;
}
#if DEBUG_DATASTORE
GNUNET_h2s (key), type,
(unsigned long long) (now.abs_value - expiration.abs_value));
#endif
+ min_expiration = now;
GNUNET_STATISTICS_update (stats, gettext_noop ("# bytes expired"), size,
GNUNET_YES);
GNUNET_CONTAINER_bloomfilter_remove (filter, key);
expired_kill_task =
- GNUNET_SCHEDULER_add_delayed (MIN_EXPIRE_DELAY, &delete_expired, NULL);
+ GNUNET_SCHEDULER_add_delayed_with_priority (MIN_EXPIRE_DELAY,
+ GNUNET_SCHEDULER_PRIORITY_IDLE,
+ &delete_expired, NULL);
return GNUNET_NO;
}
return GNUNET_SYSERR;
#if DEBUG_DATASTORE
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
- "Deleting %llu bytes of low-priority content `%s' of type %u (still trying to free another %llu bytes)\n",
+ "Deleting %llu bytes of low-priority (%u) content `%s' of type %u at %llu ms prior to expiration (still trying to free another %llu bytes)\n",
(unsigned long long) (size + GNUNET_DATASTORE_ENTRY_OVERHEAD),
- GNUNET_h2s (key), type, *need);
+ (unsigned int) priority,
+ GNUNET_h2s (key), type,
+ (unsigned long long) GNUNET_TIME_absolute_get_remaining (expiration).rel_value,
+ *need);
#endif
if (size + GNUNET_DATASTORE_ENTRY_OVERHEAD > *need)
*need = 0;
else
*need -= size + GNUNET_DATASTORE_ENTRY_OVERHEAD;
+ if (priority > 0)
+ min_expiration = GNUNET_TIME_UNIT_FOREVER_ABS;
+ else
+ min_expiration = expiration;
GNUNET_STATISTICS_update (stats,
gettext_noop ("# bytes purged (low-priority)"),
size, GNUNET_YES);
sm->header.size = htons (sizeof (struct StatusMessage) + slen);
sm->header.type = htons (GNUNET_MESSAGE_TYPE_DATASTORE_STATUS);
sm->status = htonl (code);
+ sm->min_expiration = GNUNET_TIME_absolute_hton (min_expiration);
if (slen > 0)
memcpy (&sm[1], msg, slen);
transmit (client, &sm->header);
{
struct DatastorePlugin *ret;
char *libname;
- char *name;
- if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
- &name))
- {
- GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
- "DATASTORE");
- return NULL;
- }
ret = GNUNET_malloc (sizeof (struct DatastorePlugin));
ret->env.cfg = cfg;
ret->env.duc = &disk_utilization_change_cb;
ret->env.cls = NULL;
GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"),
- name);
- GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
- ret->short_name = name;
+ plugin_name);
+ GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", plugin_name);
+ ret->short_name = GNUNET_strdup (plugin_name);
ret->lib_name = libname;
ret->api = GNUNET_PLUGIN_load (libname, &ret->env);
if (ret->api == NULL)
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
- _("Failed to load datastore plugin for `%s'\n"), name);
+ _("Failed to load datastore plugin for `%s'\n"), plugin_name);
GNUNET_free (ret->short_name);
GNUNET_free (libname);
GNUNET_free (ret);
GNUNET_free (plug->lib_name);
GNUNET_free (plug->short_name);
GNUNET_free (plug);
+ GNUNET_free (quota_stat_name);
+ quota_stat_name = NULL;
}
static void
unload_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
{
+ if (lastSync > 0)
+ sync_stats ();
if (GNUNET_YES == do_drop)
plugin->api->drop (plugin->api->cls);
unload_plugin (plugin);
GNUNET_CONTAINER_bloomfilter_free (filter);
filter = NULL;
}
- if (lastSync > 0)
- sync_stats ();
if (stat_get != NULL)
{
GNUNET_STATISTICS_get_cancel (stat_get);
GNUNET_STATISTICS_destroy (stats, GNUNET_YES);
stats = NULL;
}
+ GNUNET_free_non_null (plugin_name);
+ plugin_name = NULL;
}
}
+/**
+ * Adds a given key to the bloomfilter 'count' times.
+ *
+ * @param cls the bloomfilter
+ * @param key key to add
+ * @param count number of times to add key
+ */
+static void
+add_key_to_bloomfilter (void *cls,
+ const GNUNET_HashCode *key,
+ unsigned int count)
+{
+ struct GNUNET_CONTAINER_BloomFilter *bf = cls;
+ while (0 < count--)
+ GNUNET_CONTAINER_bloomfilter_add (bf, key);
+}
+
+
/**
* Process datastore requests.
*
sizeof (struct GNUNET_MessageHeader)},
{NULL, NULL, 0, 0}
};
- char *fn;
+ char *fn;
+ char *pfn;
unsigned int bf_size;
+ int refresh_bf;
cfg = c;
if (GNUNET_OK !=
- GNUNET_CONFIGURATION_get_value_number (cfg, "DATASTORE", "QUOTA", "a))
+ GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
+ &plugin_name))
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
+ "DATASTORE");
+ return;
+ }
+ GNUNET_asprintf ("a_stat_name,
+ _("# bytes used in file-sharing datastore `%s'"),
+ plugin_name);
+ if (GNUNET_OK !=
+ GNUNET_CONFIGURATION_get_value_size (cfg, "DATASTORE", "QUOTA", "a))
{
GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
_("No `%s' specified for `%s' in configuration!\n"), "QUOTA",
cache_size = quota / 8; /* Or should we make this an option? */
GNUNET_STATISTICS_set (stats, gettext_noop ("# cache size"), cache_size,
GNUNET_NO);
- bf_size = quota / 32; /* 8 bit per entry, 1 bit per 32 kb in DB */
+ if (quota / (32 * 1024LL) > (1 << 31))
+ bf_size = (1 << 31); /* absolute limit: ~2 GB, beyond that BF just won't help anyway */
+ else
+ bf_size = quota / (32 * 1024LL); /* 8 bit per entry, 1 bit per 32 kb in DB */
fn = NULL;
if ((GNUNET_OK !=
GNUNET_CONFIGURATION_get_value_filename (cfg, "DATASTORE", "BLOOMFILTER",
fn = NULL;
}
if (fn != NULL)
- filter = GNUNET_CONTAINER_bloomfilter_load (fn, bf_size, 5); /* approx. 3% false positives at max use */
+ {
+ GNUNET_asprintf (&pfn, "%s.%s", fn, plugin_name);
+ if (GNUNET_YES == GNUNET_DISK_file_test (pfn))
+ {
+ filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */
+ if (NULL == filter)
+ {
+ /* file exists but not valid, remove and try again, but refresh */
+ if (0 != UNLINK (pfn))
+ {
+ /* failed to remove, run without file */
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ _("Failed to remove bogus bloomfilter file `%s'\n"),
+ pfn);
+ GNUNET_free (pfn);
+ pfn = NULL;
+ filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */
+ refresh_bf = GNUNET_YES;
+ }
+ else
+ {
+ /* try again after remove */
+ filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */
+ refresh_bf = GNUNET_YES;
+ if (NULL == filter)
+ {
+ /* failed yet again, give up on using file */
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ _("Failed to remove bogus bloomfilter file `%s'\n"),
+ pfn);
+ GNUNET_free (pfn);
+ pfn = NULL;
+ filter = GNUNET_CONTAINER_bloomfilter_load (NULL, bf_size, 5); /* approx. 3% false positives at max use */
+ }
+ }
+ }
+ else
+ {
+ /* normal case: have an existing valid bf file, no need to refresh */
+ refresh_bf = GNUNET_NO;
+ }
+ }
+ else
+ {
+ filter = GNUNET_CONTAINER_bloomfilter_load (pfn, bf_size, 5); /* approx. 3% false positives at max use */
+ refresh_bf = GNUNET_YES;
+ }
+ GNUNET_free (pfn);
+ }
else
+ {
filter = GNUNET_CONTAINER_bloomfilter_init (NULL, bf_size, 5); /* approx. 3% false positives at max use */
+ refresh_bf = GNUNET_YES;
+ }
GNUNET_free_non_null (fn);
if (filter == NULL)
{
return;
}
stat_get =
- GNUNET_STATISTICS_get (stats, "datastore", QUOTA_STAT_NAME,
+ GNUNET_STATISTICS_get (stats, "datastore", quota_stat_name,
GNUNET_TIME_UNIT_SECONDS, &process_stat_done,
&process_stat_in, plugin);
GNUNET_SERVER_disconnect_notify (server, &cleanup_reservations, NULL);
GNUNET_SERVER_add_handlers (server, handlers);
+ if (GNUNET_YES == refresh_bf)
+ {
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Rebuilding bloomfilter. Please be patient.\n"));
+ if (NULL != plugin->api->get_keys)
+ plugin->api->get_keys (plugin->api->cls, &add_key_to_bloomfilter, filter);
+ else
+ GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
+ _("Plugin does not support get_keys function. Please fix!\n"));
+
+ GNUNET_log (GNUNET_ERROR_TYPE_INFO,
+ _("Bloomfilter construction complete.\n"));
+ }
expired_kill_task =
GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
&delete_expired, NULL);
projects / oweals / gnunet.git / blobdiff