X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fdatastore%2Fperf_plugin_datastore.c;h=908a7ee5564294cff8a39ef7246cd634dbadcaf2;hb=9a50e92d4a43f917b0cd1ed34d4932470cb3a3da;hp=85659f06a980194faac90a871827c5b46f5dc208;hpb=a1f0c51bc626efeab9d842765fd1aede65035f93;p=oweals%2Fgnunet.git diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c index 85659f06a..908a7ee55 100644 --- a/src/datastore/perf_plugin_datastore.c +++ b/src/datastore/perf_plugin_datastore.c @@ -1,10 +1,10 @@ /* This file is part of GNUnet. - (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your + by the Free Software Foundation; either version 3, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but @@ -26,9 +26,9 @@ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_protocols.h" -#include "plugin_datastore.h" - -#define VERBOSE GNUNET_NO +#include "gnunet_datastore_plugin.h" +#include "gnunet_testing_lib.h" +#include /** * Target datastore size (in bytes). Realistic sizes are @@ -36,33 +36,38 @@ * those take too long to run them in the usual "make check" * sequence. Hence the value used for shipping is tiny. */ -#define MAX_SIZE 1024LL * 1024 * 128 +#define MAX_SIZE 1024LL * 1024 * 16 * 1 -#define ITERATIONS 10 +#define ITERATIONS 2 /** * Number of put operations equivalent to 1/10th of MAX_SIZE */ #define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS) +static char category[256]; + +static unsigned int hits[PUT_10 / 8 + 1]; + static unsigned long long stored_bytes; static unsigned long long stored_entries; static unsigned long long stored_ops; +static const char *plugin_name; + static int ok; enum RunPhase - { - RP_DONE = 0, - RP_PUT, - RP_LP_GET, - RP_AE_GET, - RP_ZA_GET, - RP_MO_GET, - RP_AN_GET - }; +{ + RP_ERROR = 0, + RP_PUT, + RP_REP_GET, + RP_ZA_GET, + RP_EXP_GET, + RP_DONE +}; struct CpsRunContext @@ -70,57 +75,69 @@ struct CpsRunContext unsigned int i; struct GNUNET_TIME_Absolute start; struct GNUNET_TIME_Absolute end; - struct GNUNET_SCHEDULER_Handle *sched; - struct GNUNET_CONFIGURATION_Handle *cfg; - struct GNUNET_DATASTORE_PluginFunctions * api; - const char *msg; + const struct GNUNET_CONFIGURATION_Handle *cfg; + struct GNUNET_DATASTORE_PluginFunctions *api; enum RunPhase phase; unsigned int cnt; + unsigned int iter; + uint64_t offset; }; - +/** + * Function called by plugins to notify us about a + * change in their disk utilization. + * + * @param cls closure (NULL) + * @param delta change in disk utilization, + * 0 for "reset to empty" + */ static void -putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) +disk_utilization_change_cb (void *cls, int delta) +{ +} + + +static void +putValue (struct GNUNET_DATASTORE_PluginFunctions *api, int i, int k) { char value[65536]; size_t size; - static GNUNET_HashCode key; + static struct GNUNET_HashCode key; static int ic; char *msg; + unsigned int prio; /* most content is 32k */ size = 32 * 1024; - - if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ - size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); size = size - (size & 7); /* always multiple of 8 */ /* generate random key */ - key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().value; - GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); + key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us; + GNUNET_CRYPTO_hash (&key, sizeof (struct GNUNET_HashCode), &key); memset (value, i, size); if (i > 255) memset (value, i - 255, size / 2); value[0] = k; + memcpy (&value[4], &i, sizeof (i)); msg = NULL; - if (GNUNET_OK != api->put (api->cls, - &key, - size, - value, - i, - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100), - i, - GNUNET_TIME_relative_to_absolute - (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, - 60 * 60 * 60 * 1000 + - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), - &msg)) - { - fprintf (stderr, "ERROR: `%s'\n", msg); - GNUNET_free_non_null (msg); - return; - } + prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); + if (GNUNET_OK != api->put (api->cls, &key, size, value, 1 + i % 4 /* type */ , + prio, i % 4 /* anonymity */ , + 0 /* replication */ , + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_MILLISECONDS, + 60 * 60 * 60 * 1000 + + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), &msg)) + { + FPRINTF (stderr, "ERROR: `%s'\n", msg); + GNUNET_free_non_null (msg); + return; + } ic++; stored_bytes += size; stored_ops++; @@ -128,81 +145,177 @@ putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) } static void -test (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); static int -iterateDummy (void *cls, - void *next_cls, - const GNUNET_HashCode * key, - uint32_t size, - const void *data, - uint32_t type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, - uint64_t uid) +iterate_zeros (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) { struct CpsRunContext *crc = cls; - - if (key == NULL) - { - crc->end = GNUNET_TIME_absolute_get(); - printf (crc->msg, - crc->i, - (unsigned long long) (crc->end.value - crc->start.value), - crc->cnt); - if (crc->phase != RP_AN_GET) - { - crc->phase++; - } - else - { - if (crc->i == ITERATIONS) - crc->phase = RP_DONE; - else - crc->phase = RP_PUT; - } - GNUNET_SCHEDULER_add_after (crc->sched, - GNUNET_NO, - GNUNET_SCHEDULER_PRIORITY_KEEP, - GNUNET_SCHEDULER_NO_PREREQUISITE_TASK, - &test, crc); - return GNUNET_OK; - } + int i; + const char *cdata = data; + + GNUNET_assert (key != NULL); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found result %d type=%u, priority=%u, size=%u, expire=%s\n", + i, + type, priority, size, + GNUNET_STRINGS_absolute_time_to_string (expiration)); crc->cnt++; - crc->api->next_request (next_cls, - GNUNET_NO); + if (crc->cnt == PUT_10 / 4 - 1) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Select random zero-anonymity item", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, crc->cnt); + if (crc->cnt > 0) + GAUGER (category, "Select random zero-anonymity item", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); return GNUNET_OK; } +static int +expiration_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + crc->cnt++; + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Selecting and deleting by expiration", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting and deleting by expiration", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + if (++crc->iter == ITERATIONS) + crc->phase++; + else + crc->phase = RP_PUT; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_NO; +} + + +static int +replication_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (NULL != key); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + crc->cnt++; + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Selecting random item for replication", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting random item for replication", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->offset = 0; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_OK; +} + /** * Function called when the service shuts * down. Unloads our datastore plugin. * * @param api api to unload + * @param cfg configuration to use */ static void -unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api, - struct GNUNET_CONFIGURATION_Handle *cfg) +unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, + const struct GNUNET_CONFIGURATION_Handle *cfg) { char *name; char *libname; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "DATASTORE", "DATABASE", &name)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), - "DATABASE", - "DATASTORE"); - return; - } + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); GNUNET_free (libname); @@ -226,81 +339,65 @@ cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) static void -test (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ struct CpsRunContext *crc = cls; int j; + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + { + GNUNET_break (0); + crc->phase = RP_ERROR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "In phase %d, iteration %u\n", crc->phase, crc->cnt); switch (crc->phase) + { + case RP_ERROR: + GNUNET_break (0); + crc->api->drop (crc->api->cls); + ok = 1; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + case RP_PUT: + crc->start = GNUNET_TIME_absolute_get (); + for (j = 0; j < PUT_10; j++) + putValue (crc->api, j, crc->i); + crc->end = GNUNET_TIME_absolute_get (); { - case RP_PUT: - crc->start = GNUNET_TIME_absolute_get (); - for (j=0;japi, j, crc->i); - crc->end = GNUNET_TIME_absolute_get (); - printf ("%3u insertion took %20llums for %u\n", - crc->i, - (unsigned long long) (crc->end.value - crc->start.value), - (unsigned int) PUT_10); - crc->i++; - crc->phase = RP_LP_GET; - GNUNET_SCHEDULER_add_after (crc->sched, - GNUNET_NO, - GNUNET_SCHEDULER_PRIORITY_KEEP, - GNUNET_SCHEDULER_NO_PREREQUISITE_TASK, - &test, crc); - break; - case RP_LP_GET: - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - crc->msg = "%3u low priority iteration took %20llums for %u\n"; - crc->api->iter_low_priority (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_AE_GET: - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - crc->msg = "%3u ascending expiration iteration took %20llums for %u\n"; - crc->api->iter_ascending_expiration (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_ZA_GET: - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - crc->msg = "%3u zero anonymity iteration took %20llums for %u\n"; - crc->api->iter_zero_anonymity (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_MO_GET: - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - crc->msg = "%3u migration order iteration took %20llums for %u\n"; - crc->api->iter_migration_order (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_AN_GET: - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - crc->msg = "%3u all now iteration took %20llums for %u\n"; - crc->api->iter_all_now (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_DONE: - crc->api->drop (crc->api->cls); - GNUNET_SCHEDULER_add_delayed (crc->sched, - GNUNET_YES, - GNUNET_SCHEDULER_PRIORITY_IDLE, - GNUNET_SCHEDULER_NO_PREREQUISITE_TASK, - GNUNET_TIME_UNIT_ZERO, - &cleaning_task, crc); - break; + printf ("%s took %s for %llu items\n", "Storing an item", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + PUT_10); + if (PUT_10 > 0) + GAUGER (category, "Storing an item", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / PUT_10, + "ms/item"); } + crc->i++; + crc->start = GNUNET_TIME_absolute_get (); + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + case RP_REP_GET: + crc->api->get_replication (crc->api->cls, &replication_get, crc); + break; + case RP_ZA_GET: + crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1, + &iterate_zeros, crc); + break; + case RP_EXP_GET: + crc->api->get_expiration (crc->api->cls, &expiration_get, crc); + break; + case RP_DONE: + crc->api->drop (crc->api->cls); + ok = 0; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + } } @@ -308,30 +405,35 @@ test (void *cls, * Load the datastore plugin. */ static struct GNUNET_DATASTORE_PluginFunctions * -load_plugin (struct GNUNET_CONFIGURATION_Handle *cfg, - struct GNUNET_SCHEDULER_Handle *sched) +load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) { static struct GNUNET_DATASTORE_PluginEnvironment env; - struct GNUNET_DATASTORE_PluginFunctions * ret; + struct GNUNET_DATASTORE_PluginFunctions *ret; char *name; char *libname; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "DATASTORE", "DATABASE", &name)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), - "DATABASE", - "DATASTORE"); - return NULL; - } + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return NULL; + } env.cfg = cfg; - env.sched = sched; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _("Loading `%s' datastore plugin\n"), name); + env.duc = &disk_utilization_change_cb; + env.cls = NULL; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), + name); GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); - GNUNET_assert (NULL != (ret = GNUNET_PLUGIN_load (libname, &env))); + if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) + { + FPRINTF (stderr, "Failed to load plugin `%s'!\n", name); + GNUNET_free (name); + GNUNET_free (libname); + return NULL; + } GNUNET_free (libname); GNUNET_free (name); return ret; @@ -339,72 +441,65 @@ load_plugin (struct GNUNET_CONFIGURATION_Handle *cfg, static void -run (void *cls, - struct GNUNET_SCHEDULER_Handle *s, - char *const *args, - const char *cfgfile, - struct GNUNET_CONFIGURATION_Handle *c) +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *c) { struct GNUNET_DATASTORE_PluginFunctions *api; struct CpsRunContext *crc; - api = load_plugin (c, s); - GNUNET_assert (api != NULL); - crc = GNUNET_malloc(sizeof(struct CpsRunContext)); + if (NULL == c) + { + GNUNET_break (0); + return; + } + api = load_plugin (c); + if (api == NULL) + { + FPRINTF (stderr, + "%s", "Could not initialize plugin, assuming database not configured. Test not run!\n"); + return; + } + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); crc->api = api; - crc->sched = s; crc->cfg = c; crc->phase = RP_PUT; - GNUNET_SCHEDULER_add_after (s, - GNUNET_YES, - GNUNET_SCHEDULER_PRIORITY_KEEP, - GNUNET_SCHEDULER_NO_PREREQUISITE_TASK, - &test, crc); + ok = 2; + GNUNET_SCHEDULER_add_now (&test, crc); } -static int -check () +int +main (int argc, char *argv[]) { - char *const argv[] = { + char dir_name[128]; + char cfg_name[128]; + char *const xargv[] = { "perf-plugin-datastore", "-c", - "perf_plugin_datastore_data.conf", -#if VERBOSE - "-L", "DEBUG", -#endif + cfg_name, NULL }; struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_OPTION_END }; - GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, - argv, "perf-plugin-datastore", "nohelp", - options, &run, NULL); - if (ok != 0) - fprintf (stderr, "Missed some testcases: %u\n", ok); - return ok; -} - - -int -main (int argc, char *argv[]) -{ - int ret; + plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/perf-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); GNUNET_log_setup ("perf-plugin-datastore", -#if VERBOSE - "DEBUG", -#else "WARNING", -#endif NULL); - ret = check (); + GNUNET_snprintf (category, sizeof (category), "DATASTORE-%s", plugin_name); + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "perf_plugin_datastore_data_%s.conf", plugin_name); + GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv, + "perf-plugin-datastore", "nohelp", options, &run, NULL); + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); + GNUNET_DISK_directory_remove (dir_name); - return ret; + return ok; } - /* end of perf_plugin_datastore.c */ - -