X-Git-Url: https://git.librecmc.org/?a=blobdiff_plain;ds=sidebyside;f=src%2Fdatastore%2Fperf_datastore_api.c;h=cf9911329eb1d0607317aa45c8163b99e72cec59;hb=5746309cb4be2073d550ad7a6885e918631dbc38;hp=95ee18dad40fac030ea9e57df4dc57cdf1585625;hpb=440bab102fc945dd51bd98f07e25f9375f58fa21;p=oweals%2Fgnunet.git diff --git a/src/datastore/perf_datastore_api.c b/src/datastore/perf_datastore_api.c index 95ee18dad..cf9911329 100644 --- a/src/datastore/perf_datastore_api.c +++ b/src/datastore/perf_datastore_api.c @@ -1,10 +1,10 @@ /* This file is part of GNUnet. - (C) 2004, 2005, 2006, 2007 Christian Grothoff (and other contributing authors) + (C) 2004, 2005, 2006, 2007, 2009, 2011 Christian Grothoff (and other contributing authors) GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published - by the Free Software Foundation; either version 2, or (at your + by the Free Software Foundation; either version 3, or (at your option) any later version. GNUnet is distributed in the hope that it will be useful, but @@ -22,78 +22,55 @@ * @brief performance measurement for the datastore implementation * @author Christian Grothoff * - * This testcase inserts a bunch of (variable size) data and then deletes - * data until the (reported) database size drops below a given threshold. - * This is iterated 10 times, with the actual size of the content stored, - * the database size reported and the file size on disk being printed for - * each iteration. The code also prints a "I" for every 40 blocks + * This testcase inserts a bunch of (variable size) data and then + * deletes data until the (reported) database size drops below a given + * threshold. This is iterated 10 times, with the actual size of the + * content stored and the number of operations performed being printed + * for each iteration. The code also prints a "I" for every 40 blocks * inserted and a "D" for every 40 blocks deleted. The deletion - * strategy alternates between "lowest priority" and "earliest expiration". - * Priorities and expiration dates are set using a pseudo-random value - * within a realistic range. + * strategy uses the "random" iterator. Priorities and expiration + * dates are set using a pseudo-random value within a realistic range. */ #include "platform.h" #include "gnunet_util_lib.h" #include "gnunet_protocols.h" #include "gnunet_datastore_service.h" +#include + +#define VERBOSE GNUNET_NO + +/** + * How long until we give up on transmitting the message? + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 15) + +static const char *plugin_name; static struct GNUNET_DATASTORE_Handle *datastore; /** * Target datastore size (in bytes). - *

- * Example impact of total size on the reported number - * of operations (insert and delete) per second (once - * roughly stabilized -- this is not "sound" experimental - * data but just a rough idea) for a particular machine: - * 

- *    4: 60   at   7k ops total
- *    8: 50   at   3k ops total
- *   16: 48   at   8k ops total
- *   32: 46   at   8k ops total
- *   64: 61   at   9k ops total
- *  128: 89   at   9k ops total
- * 4092: 11   at 383k ops total (12 GB stored, 14.8 GB DB size on disk, 2.5 GB reported)
- *
- * Pure insertion performance into an empty DB initially peaks - * at about 400 ops. The performance seems to drop especially - * once the existing (fragmented) ISAM space is filled up and - * the DB needs to grow on disk. This could be explained with - * ISAM looking more carefully for defragmentation opportunities. - *

- * MySQL disk space overheads (for otherwise unused database when - * run with 128 MB target data size; actual size 651 MB, useful - * data stored 520 MB) are quite large in the range of 25-30%. - *

- * This kind of processing seems to be IO bound (system is roughly - * at 90% wait, 10% CPU). This is with MySQL 5.0. - * */ -#define MAX_SIZE 1024LL * 1024 * 16 +#define MAX_SIZE 1024LL * 1024 * 4 /** * Report progress outside of major reports? Should probably be GNUNET_YES if * size is > 16 MB. */ -#define REPORT_ID GNUNET_NO - -/** - * Number of put operations equivalent to 1/10th of MAX_SIZE - */ -#define PUT_10 MAX_SIZE / 32 / 1024 / 10 +#define REPORT_ID GNUNET_YES /** - * Progress report frequency. 1/10th of a put operation block. + * Number of put operations equivalent to 1/3rd of MAX_SIZE */ -#define REP_FREQ PUT_10 / 10 +#define PUT_10 MAX_SIZE / 32 / 1024 / 3 /** * Total number of iterations (each iteration doing * PUT_10 put operations); we report full status every * 10 iterations. Abort with CTRL-C. */ -#define ITERATIONS 100 +#define ITERATIONS 8 static unsigned long long stored_bytes; @@ -104,113 +81,254 @@ static unsigned long long stored_ops; static struct GNUNET_TIME_Absolute start_time; -static int -putValue (int i, int k) +static int ok; + +enum RunPhase { - size_t size; - static GNUNET_HashCode key; - static int ic; - static char data[65536]; + RP_DONE = 0, + RP_PUT, + RP_CUT, + RP_REPORT, + RP_ERROR +}; + - /* most content is 32k */ - size = 32 * 1024; - if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ - size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); - size = size - (size & 7); /* always multiple of 8 */ - - GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); - memset (data, i, size); - if (i > 255) - memset (data, i - 255, size / 2); - data[0] = k; - GNUNET_DATASTORE_put (datastore, - 0, - &key, - size, - data, - i, - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100), - i, - GNUNET_TIME_relative_to_absolute - (GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, - GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), - NULL, NULL); - ic++; +struct CpsRunContext +{ + const struct GNUNET_CONFIGURATION_Handle *cfg; + enum RunPhase phase; + int j; + unsigned long long size; + int i; +}; + + + +static void run_continuation (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc); + + + + +static void +check_success (void *cls, int success, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "Check success failed: `%s'\n", msg); + crc->phase = RP_ERROR; + GNUNET_SCHEDULER_add_now (&run_continuation, crc); + return; + } #if REPORT_ID - if (ic % REP_FREQ == 0) - fprintf (stderr, "I"); + fprintf (stderr, "I"); #endif - stored_bytes += size; + stored_bytes += crc->size; stored_ops++; stored_entries++; - return GNUNET_OK; + crc->j++; + if (crc->j >= PUT_10) + { + crc->j = 0; + crc->i++; + if (crc->i == ITERATIONS) + crc->phase = RP_DONE; + else + crc->phase = RP_CUT; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); } +/** + * Continuation called to notify client about result of the + * operation. + * + * @param cls closure + * @param success GNUNET_SYSERR on failure + * @param msg NULL on success, otherwise an error message + */ static void -iterate_delete (void *cls, - const GNUNET_HashCode * key, - uint32_t size, - const void *data, - uint32_t type, - uint32_t priority, - uint32_t anonymity, - struct GNUNET_TIME_Absolute - expiration, uint64_t uid) +remove_next (void *cls, int success, const char *msg) { - GNUNET_DATASTORE_remove (datastore, key, size, data, NULL, NULL); + struct CpsRunContext *crc = cls; + + if (GNUNET_OK != success) + { + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, "remove_next failed: `%s'\n", msg); + crc->phase = RP_ERROR; + GNUNET_SCHEDULER_add_now (&run_continuation, crc); + return; + } +#if REPORT_ID + fprintf (stderr, "D"); +#endif + GNUNET_assert (GNUNET_OK == success); + GNUNET_SCHEDULER_add_now (&run_continuation, crc); } +static void +delete_value (void *cls, const GNUNET_HashCode * key, size_t size, + const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority, + uint32_t anonymity, struct GNUNET_TIME_Absolute expiration, + uint64_t uid) +{ + struct CpsRunContext *crc = cls; + + GNUNET_assert (NULL != key); + stored_ops++; + stored_bytes -= size; + stored_entries--; + stored_ops++; + if (stored_bytes < MAX_SIZE) + crc->phase = RP_PUT; + GNUNET_assert (NULL != + GNUNET_DATASTORE_remove (datastore, key, size, data, 1, 1, + TIMEOUT, &remove_next, crc)); +} + static void -run (void *cls, - struct GNUNET_SCHEDULER_Handle *sched, - char *const *args, - const char *cfgfile, struct GNUNET_CONFIGURATION_Handle *cfg) +run_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { - int j; - unsigned long long size; + struct CpsRunContext *crc = cls; + size_t size; + static GNUNET_HashCode key; + static char data[65536]; int i; - - datastore = GNUNET_DATASTORE_connect (cfg, sched); - /* FIXME: change loop to use CPS; current - datastore API will likely react negative to - us ignoring the callbacks... */ - for (i = 0; i < ITERATIONS; i++) - { + int k; + char gstr[128]; + + ok = (int) crc->phase; + switch (crc->phase) + { + case RP_PUT: + memset (&key, 256 - crc->i, sizeof (GNUNET_HashCode)); + i = crc->j; + k = crc->i; + /* most content is 32k */ + size = 32 * 1024; + if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */ + size = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024); + crc->size = size = size - (size & 7); /* always multiple of 8 */ + GNUNET_CRYPTO_hash (&key, sizeof (GNUNET_HashCode), &key); + memset (data, i, size); + if (i > 255) + memset (data, i - 255, size / 2); + data[0] = k; + GNUNET_assert (NULL != + GNUNET_DATASTORE_put (datastore, 0, &key, size, data, i + 1, + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 100), i, + 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_relative_multiply + (GNUNET_TIME_UNIT_SECONDS, + GNUNET_CRYPTO_random_u32 + (GNUNET_CRYPTO_QUALITY_WEAK, 1000))), + 1, 1, TIMEOUT, &check_success, crc)); + break; + case RP_CUT: + /* trim down below MAX_SIZE again */ + GNUNET_assert (NULL != + GNUNET_DATASTORE_get_for_replication (datastore, 1, 1, + TIMEOUT, &delete_value, + crc)); + break; + case RP_REPORT: + printf ( #if REPORT_ID - fprintf (stderr, "."); + "\n" #endif - /* insert data equivalent to 1/10th of MAX_SIZE */ - for (j = 0; j < PUT_10; j++) - GNUNET_assert (GNUNET_OK == putValue (j, i)); - - /* trim down below MAX_SIZE again */ - if ((i % 2) == 0) - GNUNET_DATASTORE_get_random (datastore, &iterate_delete, NULL); - size = 0; - printf ( -#if REPORT_ID - "\n" -#endif - "Stored %llu kB / %lluk ops / %llu ops/s\n", - stored_bytes / 1024, /* used size in k */ - (stored_ops * 2 - stored_entries) / 1024, /* total operations (in k) */ - 1000 * (stored_ops * 2 - stored_entries) / (1 + GNUNET_TIME_absolute_get_duration(start_time).value)); /* operations per second */ - } - GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + "Stored %llu kB / %lluk ops / %llu ops/s\n", stored_bytes / 1024, /* used size in k */ + stored_ops / 1024, /* total operations (in k) */ + 1000 * stored_ops / (1 + + GNUNET_TIME_absolute_get_duration + (start_time).rel_value)); + crc->phase = RP_PUT; + crc->j = 0; + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); + break; + case RP_DONE: + GNUNET_snprintf (gstr, sizeof (gstr), "DATASTORE-%s", plugin_name); + if ((crc->i == ITERATIONS) && (stored_ops > 0)) + GAUGER (gstr, "PUT operation duration", + GNUNET_TIME_absolute_get_duration (start_time).rel_value / + stored_ops, "ms/operation"); + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 0; + break; + case RP_ERROR: + GNUNET_DATASTORE_disconnect (datastore, GNUNET_YES); + GNUNET_free (crc); + ok = 1; + break; + default: + GNUNET_assert (0); + } +} + + +static void +run_tests (void *cls, int success, const char *msg) +{ + struct CpsRunContext *crc = cls; + + if (success != GNUNET_YES) + { + fprintf (stderr, + "Test 'put' operation failed with error `%s' database likely not setup, skipping test.", + msg); + GNUNET_free (crc); + return; + } + GNUNET_SCHEDULER_add_continuation (&run_continuation, crc, + GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + + +static void +run (void *cls, char *const *args, const char *cfgfile, + const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + struct CpsRunContext *crc; + static GNUNET_HashCode zkey; + + datastore = GNUNET_DATASTORE_connect (cfg); + start_time = GNUNET_TIME_absolute_get (); + crc = GNUNET_malloc (sizeof (struct CpsRunContext)); + crc->cfg = cfg; + crc->phase = RP_PUT; + if (NULL == + GNUNET_DATASTORE_put (datastore, 0, &zkey, 4, "TEST", + GNUNET_BLOCK_TYPE_TEST, 0, 0, 0, + GNUNET_TIME_relative_to_absolute + (GNUNET_TIME_UNIT_SECONDS), 0, 1, + GNUNET_TIME_UNIT_MINUTES, &run_tests, crc)) + { + fprintf (stderr, "Test 'put' operation failed.\n"); + ok = 1; + GNUNET_free (crc); + } } static int check () { - int ok = 1 + 2 + 4 + 8; - pid_t pid; - char *const argv[] = { "perf-datastore-api", + struct GNUNET_OS_Process *proc; + char cfg_name[128]; + + char *const argv[] = { + "perf-datastore-api", "-c", - "test_datastore_api_data.conf", + cfg_name, #if VERBOSE "-L", "DEBUG", #endif @@ -219,24 +337,28 @@ check () struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_OPTION_END }; - pid = GNUNET_OS_start_process ("gnunet-service-datastore", - "gnunet-service-datastore", + + GNUNET_snprintf (cfg_name, sizeof (cfg_name), + "test_datastore_api_data_%s.conf", plugin_name); + proc = + GNUNET_OS_start_process (NULL, NULL, "gnunet-service-arm", + "gnunet-service-arm", #if VERBOSE - "-L", "DEBUG", + "-L", "DEBUG", #endif - "-c", "test_datastore_api_data.conf", NULL); - sleep (1); - GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, - argv, "perf-datastore-api", "nohelp", - options, &run, &ok); - if (0 != PLIBC_KILL (pid, SIGTERM)) - { - GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); - ok = 1; - } - GNUNET_OS_process_wait(pid); - if (ok != 0) - fprintf (stderr, "Missed some testcases: %u\n", ok); + "-c", cfg_name, NULL); + GNUNET_assert (NULL != proc); + GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, + "perf-datastore-api", "nohelp", options, &run, NULL); + sleep (1); /* give datastore chance to process 'DROP' */ + if (0 != GNUNET_OS_process_kill (proc, SIGTERM)) + { + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + ok = 1; + } + GNUNET_OS_process_wait (proc); + GNUNET_OS_process_close (proc); + proc = NULL; return ok; } @@ -245,7 +367,22 @@ int main (int argc, char *argv[]) { int ret; + char *pos; + char dir_name[128]; + sleep (1); + /* determine name of plugin to use */ + plugin_name = argv[0]; + while (NULL != (pos = strstr (plugin_name, "_"))) + plugin_name = pos + 1; + if (NULL != (pos = strstr (plugin_name, "."))) + pos[0] = 0; + else + pos = (char *) plugin_name; + + GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/test-gnunet-datastore-%s", + plugin_name); + GNUNET_DISK_directory_remove (dir_name); GNUNET_log_setup ("perf-datastore-api", #if VERBOSE "DEBUG", @@ -254,9 +391,13 @@ main (int argc, char *argv[]) #endif NULL); ret = check (); - + if (pos != plugin_name) + pos[0] = '.'; +#if REPORT_ID + fprintf (stderr, "\n"); +#endif + GNUNET_DISK_directory_remove (dir_name); return ret; } - /* end of perf_datastore_api.c */