X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;f=src%2Fdatastore%2Fperf_plugin_datastore.c;h=908a7ee5564294cff8a39ef7246cd634dbadcaf2;hb=9a50e92d4a43f917b0cd1ed34d4932470cb3a3da;hp=c21d9551b6e9f0fb735e13b91960c9b106aba6c3;hpb=70ec14559f335b5d433fb39ebefcbefdb659194a;p=oweals%2Fgnunet.git diff --git a/src/datastore/perf_plugin_datastore.c b/src/datastore/perf_plugin_datastore.c index c21d9551b..908a7ee55 100644 --- a/src/datastore/perf_plugin_datastore.c +++ b/src/datastore/perf_plugin_datastore.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet. - (C) 2004, 2005, 2006, 2007, 2009 Christian Grothoff (and other contributing authors) + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -27,8 +27,8 @@ #include "gnunet_util_lib.h" #include "gnunet_protocols.h" #include "gnunet_datastore_plugin.h" - -#define VERBOSE GNUNET_NO +#include "gnunet_testing_lib.h" +#include /** * Target datastore size (in bytes). Realistic sizes are @@ -36,15 +36,19 @@ * those take too long to run them in the usual "make check" * sequence. Hence the value used for shipping is tiny. */ -#define MAX_SIZE 1024LL * 1024 * 128 +#define MAX_SIZE 1024LL * 1024 * 16 * 1 -#define ITERATIONS 10 +#define ITERATIONS 2 /** * Number of put operations equivalent to 1/10th of MAX_SIZE */ #define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS) +static char category[256]; + +static unsigned int hits[PUT_10 / 8 + 1]; + static unsigned long long stored_bytes; static unsigned long long stored_entries; @@ -56,13 +60,14 @@ static const char *plugin_name; static int ok; enum RunPhase - { - RP_DONE = 0, - RP_PUT, - RP_LP_GET, - RP_AE_GET, - RP_ZA_GET - }; +{ + RP_ERROR = 0, + RP_PUT, + RP_REP_GET, + RP_ZA_GET, + RP_EXP_GET, + RP_DONE +}; struct CpsRunContext @@ -71,10 +76,11 @@ struct CpsRunContext struct GNUNET_TIME_Absolute start; struct GNUNET_TIME_Absolute end; const struct GNUNET_CONFIGURATION_Handle *cfg; - struct GNUNET_DATASTORE_PluginFunctions * api; - const char *msg; + struct GNUNET_DATASTORE_PluginFunctions *api; enum RunPhase phase; unsigned int cnt; + unsigned int iter; + uint64_t offset; }; @@ -83,60 +89,55 @@ struct CpsRunContext * change in their disk utilization. * * @param cls closure (NULL) - * @param delta change in disk utilization, + * @param delta change in disk utilization, * 0 for "reset to empty" */ static void -disk_utilization_change_cb (void *cls, - int delta) +disk_utilization_change_cb (void *cls, int delta) { } - + static void -putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) +putValue (struct GNUNET_DATASTORE_PluginFunctions *api, int i, int k) { char value[65536]; size_t size; - static GNUNET_HashCode key; + static struct GNUNET_HashCode key; static int ic; char *msg; unsigned int prio; /* most content is 32k */ size = 32 * 1024; - - if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ - size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); size = size - (size & 7); /* always multiple of 8 */ /* generate random key */ - key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value; - GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); + key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us; + GNUNET_CRYPTO_hash (&key, sizeof (struct GNUNET_HashCode), &key); memset (value, i, size); if (i > 255) memset (value, i - 255, size / 2); value[0] = k; + memcpy (&value[4], &i, sizeof (i)); msg = NULL; prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100); - if (GNUNET_OK != api->put (api->cls, - &key, - size, - value, - i /* type */, - prio, - i /* anonymity */, - 0 /* replication */, - GNUNET_TIME_relative_to_absolute - (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS, - 60 * 60 * 60 * 1000 + - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), - &msg)) - { - fprintf (stderr, "ERROR: `%s'\n", msg); - GNUNET_free_non_null (msg); - return; - } + if (GNUNET_OK != api->put (api->cls, &key, size, value, 1 + i % 4 /* type */ , + prio, i % 4 /* anonymity */ , + 0 /* replication */ , + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_MILLISECONDS, + 60 * 60 * 60 * 1000 + + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), &msg)) + { + FPRINTF (stderr, "ERROR: `%s'\n", msg); + GNUNET_free_non_null (msg); + return; + } ic++; stored_bytes += size; stored_ops++; @@ -144,92 +145,154 @@ putValue (struct GNUNET_DATASTORE_PluginFunctions * api, int i, int k) } static void -test (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc); +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc); static int -iterateDummy (void *cls, - void *next_cls, - const GNUNET_HashCode * key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, - uint64_t uid) +iterate_zeros (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) { struct CpsRunContext *crc = cls; - - if (key == NULL) - { - crc->end = GNUNET_TIME_absolute_get(); - printf (crc->msg, - crc->i, - (unsigned long long) (crc->end.abs_value - crc->start.abs_value), - crc->cnt); - if (crc->phase != RP_ZA_GET) - { - crc->phase++; - } - else - { - if (crc->i == ITERATIONS) - crc->phase = RP_DONE; - else - crc->phase = RP_PUT; - } - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - GNUNET_SCHEDULER_add_now (&test, crc); - return GNUNET_OK; - } -#if VERBOSE - fprintf (stderr, "Found result type=%u, priority=%u, size=%u, expire=%llu\n", - type, priority, size, - (unsigned long long) expiration.abs_value); -#endif + int i; + const char *cdata = data; + + GNUNET_assert (key != NULL); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Found result %d type=%u, priority=%u, size=%u, expire=%s\n", + i, + type, priority, size, + GNUNET_STRINGS_absolute_time_to_string (expiration)); crc->cnt++; - crc->api->next_request (next_cls, - GNUNET_NO); + if (crc->cnt == PUT_10 / 4 - 1) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Select random zero-anonymity item", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, crc->cnt); + if (crc->cnt > 0) + GAUGER (category, "Select random zero-anonymity item", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); return GNUNET_OK; } - static int -dummy_get (void *cls, - void *next_cls, - const GNUNET_HashCode * key, - uint32_t size, - const void *data, - enum GNUNET_BLOCK_Type type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, - uint64_t uid) +expiration_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) { struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); crc->cnt++; - if (1000 == crc->cnt) - { - crc->end = GNUNET_TIME_absolute_get(); - printf (crc->msg, - crc->i, - (unsigned long long) (crc->end.abs_value - crc->start.abs_value), - crc->cnt); + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Selecting and deleting by expiration", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting and deleting by expiration", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + if (++crc->iter == ITERATIONS) crc->phase++; - crc->cnt = 0; - crc->start = GNUNET_TIME_absolute_get (); - } + else + crc->phase = RP_PUT; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); + return GNUNET_NO; +} + + +static int +replication_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size, + const void *data, enum GNUNET_BLOCK_Type type, + uint32_t priority, uint32_t anonymity, + struct GNUNET_TIME_Absolute expiration, uint64_t uid) +{ + struct CpsRunContext *crc = cls; + int i; + const char *cdata = data; + + GNUNET_assert (NULL != key); + GNUNET_assert (size >= 8); + memcpy (&i, &cdata[4], sizeof (i)); + hits[i / 8] |= (1 << (i % 8)); + crc->cnt++; + if (PUT_10 <= crc->cnt) + { + unsigned int bc; + + bc = 0; + for (i = 0; i < PUT_10; i++) + if (0 != (hits[i / 8] & (1 << (i % 8)))) + bc++; + + crc->end = GNUNET_TIME_absolute_get (); + printf ("%s took %s yielding %u/%u items\n", + "Selecting random item for replication", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + bc, (unsigned int) PUT_10); + if (crc->cnt > 0) + GAUGER (category, "Selecting random item for replication", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt, + "ms/item"); + memset (hits, 0, sizeof (hits)); + crc->phase++; + crc->offset = 0; + crc->cnt = 0; + crc->start = GNUNET_TIME_absolute_get (); + } + GNUNET_SCHEDULER_add_now (&test, crc); return GNUNET_OK; } + /** * Function called when the service shuts * down. Unloads our datastore plugin. @@ -238,22 +301,21 @@ dummy_get (void *cls, * @param cfg configuration to use */ static void -unload_plugin (struct GNUNET_DATASTORE_PluginFunctions * api, - const struct GNUNET_CONFIGURATION_Handle *cfg) +unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api, + const struct GNUNET_CONFIGURATION_Handle *cfg) { char *name; char *libname; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "DATASTORE", "DATABASE", &name)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), - "DATABASE", - "DATASTORE"); - return; - } + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return; + } GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api)); GNUNET_free (libname); @@ -277,58 +339,65 @@ cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) static void -test (void *cls, - const struct GNUNET_SCHEDULER_TaskContext *tc) -{ +test (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ struct CpsRunContext *crc = cls; int j; if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) - { - crc->phase = RP_DONE; - ok = 1; - } + { + GNUNET_break (0); + crc->phase = RP_ERROR; + } + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "In phase %d, iteration %u\n", crc->phase, crc->cnt); switch (crc->phase) + { + case RP_ERROR: + GNUNET_break (0); + crc->api->drop (crc->api->cls); + ok = 1; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + case RP_PUT: + crc->start = GNUNET_TIME_absolute_get (); + for (j = 0; j < PUT_10; j++) + putValue (crc->api, j, crc->i); + crc->end = GNUNET_TIME_absolute_get (); { - case RP_PUT: - crc->start = GNUNET_TIME_absolute_get (); - for (j=0;japi, j, crc->i); - crc->end = GNUNET_TIME_absolute_get (); - printf ("%3u insertion took %20llums for %u\n", - crc->i, - (unsigned long long) (crc->end.abs_value - crc->start.abs_value), - (unsigned int) PUT_10); - crc->i++; - crc->start = GNUNET_TIME_absolute_get (); - crc->phase = RP_LP_GET; - GNUNET_SCHEDULER_add_after (GNUNET_SCHEDULER_NO_TASK, - &test, crc); - break; - case RP_LP_GET: - crc->msg = "%3u replication iteration took %20llums for %u\n"; - crc->api->replication_get (crc->api->cls, - &dummy_get, - crc); - break; - case RP_AE_GET: - crc->msg = "%3u expiration iteration took %20llums for %u\n"; - crc->api->expiration_get (crc->api->cls, - &dummy_get, - crc); - break; - case RP_ZA_GET: - crc->msg = "%3u zero anonymity iteration took %20llums for %u\n"; - crc->api->iter_zero_anonymity (crc->api->cls, 0, - &iterateDummy, - crc); - break; - case RP_DONE: - crc->api->drop (crc->api->cls); - GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, - &cleaning_task, crc); - break; + printf ("%s took %s for %llu items\n", "Storing an item", + GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start, + crc->end), + GNUNET_YES), + PUT_10); + if (PUT_10 > 0) + GAUGER (category, "Storing an item", + (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / PUT_10, + "ms/item"); } + crc->i++; + crc->start = GNUNET_TIME_absolute_get (); + crc->phase++; + GNUNET_SCHEDULER_add_now (&test, crc); + break; + case RP_REP_GET: + crc->api->get_replication (crc->api->cls, &replication_get, crc); + break; + case RP_ZA_GET: + crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1, + &iterate_zeros, crc); + break; + case RP_EXP_GET: + crc->api->get_expiration (crc->api->cls, &expiration_get, crc); + break; + case RP_DONE: + crc->api->drop (crc->api->cls); + ok = 0; + GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE, + &cleaning_task, crc); + break; + } } @@ -339,33 +408,32 @@ static struct GNUNET_DATASTORE_PluginFunctions * load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) { static struct GNUNET_DATASTORE_PluginEnvironment env; - struct GNUNET_DATASTORE_PluginFunctions * ret; + struct GNUNET_DATASTORE_PluginFunctions *ret; char *name; char *libname; if (GNUNET_OK != - GNUNET_CONFIGURATION_get_value_string (cfg, - "DATASTORE", "DATABASE", &name)) - { - GNUNET_log (GNUNET_ERROR_TYPE_ERROR, - _("No `%s' specified for `%s' in configuration!\n"), - "DATABASE", - "DATASTORE"); - return NULL; - } + GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE", + &name)) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + _("No `%s' specified for `%s' in configuration!\n"), "DATABASE", + "DATASTORE"); + return NULL; + } env.cfg = cfg; env.duc = &disk_utilization_change_cb; env.cls = NULL; - GNUNET_log (GNUNET_ERROR_TYPE_INFO, - _("Loading `%s' datastore plugin\n"), name); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"), + name); GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name); if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env))) - { - fprintf (stderr, - "Failed to load plugin `%s'!\n", - name); - return NULL; - } + { + FPRINTF (stderr, "Failed to load plugin `%s'!\n", name); + GNUNET_free (name); + GNUNET_free (libname); + return NULL; + } GNUNET_free (libname); GNUNET_free (name); return ret; @@ -373,95 +441,65 @@ load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg) static void -run (void *cls, - char *const *args, - const char *cfgfile, +run (void *cls, char *const *args, const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *c) { struct GNUNET_DATASTORE_PluginFunctions *api; struct CpsRunContext *crc; + if (NULL == c) + { + GNUNET_break (0); + return; + } api = load_plugin (c); if (api == NULL) - { - fprintf (stderr, - "Could not initialize plugin, assuming database not configured. Test not run!\n"); - return; - } - crc = GNUNET_malloc(sizeof(struct CpsRunContext)); + { + FPRINTF (stderr, + "%s", "Could not initialize plugin, assuming database not configured. Test not run!\n"); + return; + } + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); crc->api = api; crc->cfg = c; crc->phase = RP_PUT; + ok = 2; GNUNET_SCHEDULER_add_now (&test, crc); } -static int -check () +int +main (int argc, char *argv[]) { + char dir_name[128]; char cfg_name[128]; - char *const argv[] = { + char *const xargv[] = { "perf-plugin-datastore", "-c", cfg_name, -#if VERBOSE - "-L", "DEBUG", -#endif NULL }; struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_OPTION_END }; - GNUNET_snprintf (cfg_name, - sizeof (cfg_name), - "perf_plugin_datastore_data_%s.conf", - plugin_name); - GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, - argv, "perf-plugin-datastore", "nohelp", - options, &run, NULL); - if (ok != 0) - fprintf (stderr, "Missed some testcases: %u\n", ok); - return ok; -} - - -int -main (int argc, char *argv[]) -{ - int ret; - char *pos; - char dir_name[128]; - - /* determine name of plugin to use */ - plugin_name = argv[0]; - while (NULL != (pos = strstr(plugin_name, "_"))) - plugin_name = pos+1; - if (NULL != (pos = strstr(plugin_name, "."))) - pos[0] = 0; - else - pos = (char *) plugin_name; - - GNUNET_snprintf (dir_name, - sizeof (dir_name), - "/tmp/perf-gnunet-datastore-%s", - plugin_name); + plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/perf-gnunet-datastore-%s", + plugin_name); GNUNET_DISK_directory_remove (dir_name); GNUNET_log_setup ("perf-plugin-datastore", -#if VERBOSE - "DEBUG", -#else "WARNING", -#endif NULL); - ret = check (); - if (pos != plugin_name) - pos[0] = '.'; + GNUNET_snprintf (category, sizeof (category), "DATASTORE-%s", plugin_name); + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "perf_plugin_datastore_data_%s.conf", plugin_name); + GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv, + "perf-plugin-datastore", "nohelp", options, &run, NULL); + if (ok != 0) + FPRINTF (stderr, "Missed some testcases: %u\n", ok); GNUNET_DISK_directory_remove (dir_name); - return ret; + return ok; } /* end of perf_plugin_datastore.c */ - -