2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
21 * @file perf_plugin_datastore.c
22 * @brief Profile database plugin directly, focusing on iterators.
23 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_protocols.h"
29 #include "gnunet_datastore_plugin.h"
30 #include "gnunet_testing_lib.h"
34 * Target datastore size (in bytes). Realistic sizes are
35 * more like 16 GB (not the default of 16 MB); however,
36 * those take too long to run them in the usual "make check"
37 * sequence. Hence the value used for shipping is tiny.
39 #define MAX_SIZE 1024LL * 1024 * 16 * 1
44 * Number of put operations equivalent to 1/10th of MAX_SIZE
46 #define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
48 static char category[256];
50 static unsigned int hits[PUT_10 / 8 + 1];
52 static unsigned long long stored_bytes;
54 static unsigned long long stored_entries;
56 static unsigned long long stored_ops;
58 static const char *plugin_name;
76 struct GNUNET_TIME_Absolute start;
77 struct GNUNET_TIME_Absolute end;
78 const struct GNUNET_CONFIGURATION_Handle *cfg;
79 struct GNUNET_DATASTORE_PluginFunctions *api;
88 * Function called by plugins to notify us about a
89 * change in their disk utilization.
91 * @param cls closure (NULL)
92 * @param delta change in disk utilization,
93 * 0 for "reset to empty"
96 disk_utilization_change_cb (void *cls, int delta)
109 * @param key key for the item stored
110 * @param size size of the item stored
111 * @param status #GNUNET_OK or #GNUNET_SYSERROR
112 * @param msg error message on error
115 put_continuation (void *cls,
116 const struct GNUNET_HashCode *key,
121 struct CpsRunContext *crc = cls;
123 if (GNUNET_OK != status)
125 FPRINTF (stderr, "ERROR: `%s'\n", msg);
129 stored_bytes += size;
133 GNUNET_SCHEDULER_add_now (&test, crc);
137 do_put (struct CpsRunContext *crc)
141 static struct GNUNET_HashCode key;
146 crc->start = GNUNET_TIME_absolute_get ();
150 crc->end = GNUNET_TIME_absolute_get ();
152 printf ("%s took %s for %llu items\n", "Storing an item",
153 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start,
158 GAUGER (category, "Storing an item",
159 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / PUT_10,
163 crc->start = GNUNET_TIME_absolute_get ();
165 GNUNET_SCHEDULER_add_now (&test, crc);
168 /* most content is 32k */
170 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
171 size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
172 size = size - (size & 7); /* always multiple of 8 */
174 /* generate random key */
175 key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us;
176 GNUNET_CRYPTO_hash (&key, sizeof (struct GNUNET_HashCode), &key);
177 memset (value, i, size);
179 memset (value, i - 255, size / 2);
181 memcpy (&value[4], &i, sizeof (i));
182 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
183 crc->api->put (crc->api->cls, &key, size, value, 1 + i % 4 /* type */ ,
184 prio, i % 4 /* anonymity */ ,
185 0 /* replication */ ,
186 GNUNET_TIME_relative_to_absolute
187 (GNUNET_TIME_relative_multiply
188 (GNUNET_TIME_UNIT_MILLISECONDS,
189 60 * 60 * 60 * 1000 +
190 GNUNET_CRYPTO_random_u32
191 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
192 put_continuation, crc);
198 iterate_zeros (void *cls, const struct GNUNET_HashCode * key, uint32_t size,
199 const void *data, enum GNUNET_BLOCK_Type type, uint32_t priority,
200 uint32_t anonymity, struct GNUNET_TIME_Absolute expiration,
203 struct CpsRunContext *crc = cls;
205 const char *cdata = data;
207 GNUNET_assert (key != NULL);
208 GNUNET_assert (size >= 8);
209 memcpy (&i, &cdata[4], sizeof (i));
210 hits[i / 8] |= (1 << (i % 8));
212 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
213 "Found result %d type=%u, priority=%u, size=%u, expire=%s\n",
215 type, priority, size,
216 GNUNET_STRINGS_absolute_time_to_string (expiration));
218 if (crc->cnt == PUT_10 / 4 - 1)
223 for (i = 0; i < PUT_10; i++)
224 if (0 != (hits[i / 8] & (1 << (i % 8))))
227 crc->end = GNUNET_TIME_absolute_get ();
228 printf ("%s took %s yielding %u/%u items\n",
229 "Select random zero-anonymity item",
230 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start,
235 GAUGER (category, "Select random zero-anonymity item",
236 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt,
238 memset (hits, 0, sizeof (hits));
241 crc->start = GNUNET_TIME_absolute_get ();
243 GNUNET_SCHEDULER_add_now (&test, crc);
249 expiration_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size,
250 const void *data, enum GNUNET_BLOCK_Type type,
251 uint32_t priority, uint32_t anonymity,
252 struct GNUNET_TIME_Absolute expiration, uint64_t uid)
254 struct CpsRunContext *crc = cls;
256 const char *cdata = data;
258 GNUNET_assert (size >= 8);
259 memcpy (&i, &cdata[4], sizeof (i));
260 hits[i / 8] |= (1 << (i % 8));
262 if (PUT_10 <= crc->cnt)
267 for (i = 0; i < PUT_10; i++)
268 if (0 != (hits[i / 8] & (1 << (i % 8))))
271 crc->end = GNUNET_TIME_absolute_get ();
272 printf ("%s took %s yielding %u/%u items\n",
273 "Selecting and deleting by expiration",
274 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start,
277 bc, (unsigned int) PUT_10);
279 GAUGER (category, "Selecting and deleting by expiration",
280 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt,
282 memset (hits, 0, sizeof (hits));
283 if (++crc->iter == ITERATIONS)
288 crc->start = GNUNET_TIME_absolute_get ();
290 GNUNET_SCHEDULER_add_now (&test, crc);
296 replication_get (void *cls, const struct GNUNET_HashCode * key, uint32_t size,
297 const void *data, enum GNUNET_BLOCK_Type type,
298 uint32_t priority, uint32_t anonymity,
299 struct GNUNET_TIME_Absolute expiration, uint64_t uid)
301 struct CpsRunContext *crc = cls;
303 const char *cdata = data;
305 GNUNET_assert (NULL != key);
306 GNUNET_assert (size >= 8);
307 memcpy (&i, &cdata[4], sizeof (i));
308 hits[i / 8] |= (1 << (i % 8));
310 if (PUT_10 <= crc->cnt)
315 for (i = 0; i < PUT_10; i++)
316 if (0 != (hits[i / 8] & (1 << (i % 8))))
319 crc->end = GNUNET_TIME_absolute_get ();
320 printf ("%s took %s yielding %u/%u items\n",
321 "Selecting random item for replication",
322 GNUNET_STRINGS_relative_time_to_string (GNUNET_TIME_absolute_get_difference (crc->start,
325 bc, (unsigned int) PUT_10);
327 GAUGER (category, "Selecting random item for replication",
328 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL / crc->cnt,
330 memset (hits, 0, sizeof (hits));
334 crc->start = GNUNET_TIME_absolute_get ();
337 GNUNET_SCHEDULER_add_now (&test, crc);
343 * Function called when the service shuts
344 * down. Unloads our datastore plugin.
346 * @param api api to unload
347 * @param cfg configuration to use
350 unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
351 const struct GNUNET_CONFIGURATION_Handle *cfg)
357 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
360 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
361 _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
365 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
366 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
367 GNUNET_free (libname);
374 * Last task run during shutdown. Disconnects us from
375 * the transport and core.
378 cleaning_task (void *cls)
380 struct CpsRunContext *crc = cls;
382 unload_plugin (crc->api, crc->cfg);
390 struct CpsRunContext *crc = cls;
391 const struct GNUNET_SCHEDULER_TaskContext *tc;
393 tc = GNUNET_SCHEDULER_get_task_context ();
394 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
397 crc->phase = RP_ERROR;
399 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
400 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
405 crc->api->drop (crc->api->cls);
407 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
408 &cleaning_task, crc);
414 crc->api->get_replication (crc->api->cls, &replication_get, crc);
417 crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1,
418 &iterate_zeros, crc);
421 crc->api->get_expiration (crc->api->cls, &expiration_get, crc);
424 crc->api->drop (crc->api->cls);
426 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
427 &cleaning_task, crc);
434 * Load the datastore plugin.
436 static struct GNUNET_DATASTORE_PluginFunctions *
437 load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
439 static struct GNUNET_DATASTORE_PluginEnvironment env;
440 struct GNUNET_DATASTORE_PluginFunctions *ret;
445 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
448 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
449 _("No `%s' specified for `%s' in configuration!\n"), "DATABASE",
454 env.duc = &disk_utilization_change_cb;
456 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Loading `%s' datastore plugin\n"),
458 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
459 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
461 FPRINTF (stderr, "Failed to load plugin `%s'!\n", name);
463 GNUNET_free (libname);
466 GNUNET_free (libname);
473 run (void *cls, char *const *args, const char *cfgfile,
474 const struct GNUNET_CONFIGURATION_Handle *c)
476 struct GNUNET_DATASTORE_PluginFunctions *api;
477 struct CpsRunContext *crc;
484 api = load_plugin (c);
488 "%s", "Could not initialize plugin, assuming database not configured. Test not run!\n");
491 crc = GNUNET_new (struct CpsRunContext);
496 GNUNET_SCHEDULER_add_now (&test, crc);
501 main (int argc, char *argv[])
505 char *const xargv[] = {
506 "perf-plugin-datastore",
511 struct GNUNET_GETOPT_CommandLineOption options[] = {
512 GNUNET_GETOPT_OPTION_END
515 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
516 GNUNET_snprintf (dir_name, sizeof (dir_name), "/tmp/perf-gnunet-datastore-%s",
518 GNUNET_DISK_directory_remove (dir_name);
519 GNUNET_log_setup ("perf-plugin-datastore",
522 GNUNET_snprintf (category, sizeof (category), "DATASTORE-%s", plugin_name);
523 GNUNET_snprintf (cfg_name, sizeof (cfg_name),
524 "perf_plugin_datastore_data_%s.conf", plugin_name);
525 GNUNET_PROGRAM_run ((sizeof (xargv) / sizeof (char *)) - 1, xargv,
526 "perf-plugin-datastore", "nohelp", options, &run, NULL);
528 FPRINTF (stderr, "Missed some testcases: %u\n", ok);
529 GNUNET_DISK_directory_remove (dir_name);
534 /* end of perf_plugin_datastore.c */