2 This file is part of GNUnet.
3 Copyright (C) 2004, 2005, 2006, 2007, 2009, 2011 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file perf_plugin_datastore.c
22 * @brief Profile database plugin directly, focusing on iterators.
23 * @author Christian Grothoff
27 #include "gnunet_util_lib.h"
28 #include "gnunet_protocols.h"
29 #include "gnunet_datastore_plugin.h"
30 #include "gnunet_testing_lib.h"
34 * Target datastore size (in bytes). Realistic sizes are
35 * more like 16 GB (not the default of 16 MB); however,
36 * those take too long to run them in the usual "make check"
37 * sequence. Hence the value used for shipping is tiny.
39 #define MAX_SIZE 1024LL * 1024 * 16 * 1
44 * Number of put operations equivalent to 1/10th of MAX_SIZE
46 #define PUT_10 (MAX_SIZE / 32 / 1024 / ITERATIONS)
48 static char category[256];
50 static unsigned int hits[PUT_10 / 8 + 1];
52 static unsigned long long stored_bytes;
54 static unsigned long long stored_entries;
56 static unsigned long long stored_ops;
58 static const char *plugin_name;
76 struct GNUNET_TIME_Absolute start;
77 struct GNUNET_TIME_Absolute end;
78 const struct GNUNET_CONFIGURATION_Handle *cfg;
79 struct GNUNET_DATASTORE_PluginFunctions *api;
88 * Function called by plugins to notify us about a
89 * change in their disk utilization.
91 * @param cls closure (NULL)
92 * @param delta change in disk utilization,
93 * 0 for "reset to empty"
96 disk_utilization_change_cb (void *cls, int delta)
109 * @param key key for the item stored
110 * @param size size of the item stored
111 * @param status #GNUNET_OK or #GNUNET_SYSERROR
112 * @param msg error message on error
115 put_continuation (void *cls,
116 const struct GNUNET_HashCode *key,
121 struct CpsRunContext *crc = cls;
123 if (GNUNET_OK != status)
125 fprintf (stderr, "ERROR: `%s'\n", msg);
129 stored_bytes += size;
133 GNUNET_SCHEDULER_add_now (&test, crc);
138 do_put (struct CpsRunContext *crc)
142 static struct GNUNET_HashCode key;
147 crc->start = GNUNET_TIME_absolute_get ();
151 crc->end = GNUNET_TIME_absolute_get ();
153 printf ("%s took %s for %llu items\n", "Storing an item",
154 GNUNET_STRINGS_relative_time_to_string (
155 GNUNET_TIME_absolute_get_difference (crc->start,
161 GAUGER (category, "Storing an item",
162 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
167 crc->start = GNUNET_TIME_absolute_get ();
169 GNUNET_SCHEDULER_add_now (&test, crc);
172 /* most content is 32k */
174 if (GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 16) == 0) /* but some of it is less! */
175 size = 8 + GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 32 * 1024);
176 size = size - (size & 7); /* always multiple of 8 */
178 /* generate random key */
179 key.bits[0] = (unsigned int) GNUNET_TIME_absolute_get ().abs_value_us;
180 GNUNET_CRYPTO_hash (&key, sizeof(struct GNUNET_HashCode), &key);
181 memset (value, i, size);
183 memset (value, i - 255, size / 2);
185 GNUNET_memcpy (&value[4], &i, sizeof(i));
186 prio = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, 100);
187 crc->api->put (crc->api->cls,
192 1 + i % 4 /* type */,
194 i % 4 /* anonymity */,
196 GNUNET_TIME_relative_to_absolute
197 (GNUNET_TIME_relative_multiply
198 (GNUNET_TIME_UNIT_MILLISECONDS,
200 + GNUNET_CRYPTO_random_u32
201 (GNUNET_CRYPTO_QUALITY_WEAK, 1000))),
209 iterate_zeros (void *cls,
210 const struct GNUNET_HashCode *key,
213 enum GNUNET_BLOCK_Type type,
216 uint32_t replication,
217 struct GNUNET_TIME_Absolute expiration,
220 struct CpsRunContext *crc = cls;
222 const char *cdata = data;
224 GNUNET_assert (key != NULL);
225 GNUNET_assert (size >= 8);
226 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
227 hits[i / 8] |= (1 << (i % 8));
229 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
230 "Found result %d type=%u, priority=%u, size=%u, expire=%s\n",
232 type, priority, size,
233 GNUNET_STRINGS_absolute_time_to_string (expiration));
235 if (crc->cnt == PUT_10 / 4 - 1)
240 for (i = 0; i < PUT_10; i++)
241 if (0 != (hits[i / 8] & (1 << (i % 8))))
244 crc->end = GNUNET_TIME_absolute_get ();
245 printf ("%s took %s yielding %u/%u items\n",
246 "Select random zero-anonymity item",
247 GNUNET_STRINGS_relative_time_to_string (
248 GNUNET_TIME_absolute_get_difference (crc->start,
254 GAUGER (category, "Select random zero-anonymity item",
255 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
258 memset (hits, 0, sizeof(hits));
261 crc->start = GNUNET_TIME_absolute_get ();
263 GNUNET_SCHEDULER_add_now (&test, crc);
269 expiration_get (void *cls,
270 const struct GNUNET_HashCode *key,
273 enum GNUNET_BLOCK_Type type,
276 uint32_t replication,
277 struct GNUNET_TIME_Absolute expiration,
280 struct CpsRunContext *crc = cls;
282 const char *cdata = data;
284 GNUNET_assert (size >= 8);
285 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
286 hits[i / 8] |= (1 << (i % 8));
288 if (PUT_10 <= crc->cnt)
293 for (i = 0; i < PUT_10; i++)
294 if (0 != (hits[i / 8] & (1 << (i % 8))))
297 crc->end = GNUNET_TIME_absolute_get ();
298 printf ("%s took %s yielding %u/%u items\n",
299 "Selecting and deleting by expiration",
300 GNUNET_STRINGS_relative_time_to_string (
301 GNUNET_TIME_absolute_get_difference (crc->start,
305 bc, (unsigned int) PUT_10);
307 GAUGER (category, "Selecting and deleting by expiration",
308 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
311 memset (hits, 0, sizeof(hits));
312 if (++crc->iter == ITERATIONS)
317 crc->start = GNUNET_TIME_absolute_get ();
319 GNUNET_SCHEDULER_add_now (&test, crc);
325 replication_get (void *cls,
326 const struct GNUNET_HashCode *key,
329 enum GNUNET_BLOCK_Type type,
332 uint32_t replication,
333 struct GNUNET_TIME_Absolute expiration,
336 struct CpsRunContext *crc = cls;
338 const char *cdata = data;
340 GNUNET_assert (NULL != key);
341 GNUNET_assert (size >= 8);
342 GNUNET_memcpy (&i, &cdata[4], sizeof(i));
343 hits[i / 8] |= (1 << (i % 8));
345 if (PUT_10 <= crc->cnt)
350 for (i = 0; i < PUT_10; i++)
351 if (0 != (hits[i / 8] & (1 << (i % 8))))
354 crc->end = GNUNET_TIME_absolute_get ();
355 printf ("%s took %s yielding %u/%u items\n",
356 "Selecting random item for replication",
357 GNUNET_STRINGS_relative_time_to_string (
358 GNUNET_TIME_absolute_get_difference (crc->start,
362 bc, (unsigned int) PUT_10);
364 GAUGER (category, "Selecting random item for replication",
365 (crc->end.abs_value_us - crc->start.abs_value_us) / 1000LL
368 memset (hits, 0, sizeof(hits));
372 crc->start = GNUNET_TIME_absolute_get ();
375 GNUNET_SCHEDULER_add_now (&test, crc);
381 * Function called when the service shuts
382 * down. Unloads our datastore plugin.
384 * @param api api to unload
385 * @param cfg configuration to use
388 unload_plugin (struct GNUNET_DATASTORE_PluginFunctions *api,
389 const struct GNUNET_CONFIGURATION_Handle *cfg)
395 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
398 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
399 _ ("No `%s' specified for `%s' in configuration!\n"),
404 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
405 GNUNET_break (NULL == GNUNET_PLUGIN_unload (libname, api));
406 GNUNET_free (libname);
412 * Last task run during shutdown. Disconnects us from
413 * the transport and core.
416 cleaning_task (void *cls)
418 struct CpsRunContext *crc = cls;
420 unload_plugin (crc->api, crc->cfg);
428 struct CpsRunContext *crc = cls;
430 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
431 "In phase %d, iteration %u\n", crc->phase, crc->cnt);
436 crc->api->drop (crc->api->cls);
438 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
439 &cleaning_task, crc);
447 crc->api->get_replication (crc->api->cls, &replication_get, crc);
451 crc->api->get_zero_anonymity (crc->api->cls, crc->offset++, 1,
452 &iterate_zeros, crc);
456 crc->api->get_expiration (crc->api->cls, &expiration_get, crc);
460 crc->api->drop (crc->api->cls);
462 GNUNET_SCHEDULER_add_with_priority (GNUNET_SCHEDULER_PRIORITY_IDLE,
463 &cleaning_task, crc);
470 * Load the datastore plugin.
472 static struct GNUNET_DATASTORE_PluginFunctions *
473 load_plugin (const struct GNUNET_CONFIGURATION_Handle *cfg)
475 static struct GNUNET_DATASTORE_PluginEnvironment env;
476 struct GNUNET_DATASTORE_PluginFunctions *ret;
481 GNUNET_CONFIGURATION_get_value_string (cfg, "DATASTORE", "DATABASE",
484 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
485 _ ("No `%s' specified for `%s' in configuration!\n"),
491 env.duc = &disk_utilization_change_cb;
493 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _ ("Loading `%s' datastore plugin\n"),
495 GNUNET_asprintf (&libname, "libgnunet_plugin_datastore_%s", name);
496 if (NULL == (ret = GNUNET_PLUGIN_load (libname, &env)))
498 fprintf (stderr, "Failed to load plugin `%s'!\n", name);
500 GNUNET_free (libname);
503 GNUNET_free (libname);
510 run (void *cls, char *const *args, const char *cfgfile,
511 const struct GNUNET_CONFIGURATION_Handle *c)
513 struct GNUNET_DATASTORE_PluginFunctions *api;
514 struct CpsRunContext *crc;
521 api = load_plugin (c);
526 "Could not initialize plugin, assuming database not configured. Test not run!\n");
529 crc = GNUNET_new (struct CpsRunContext);
534 GNUNET_SCHEDULER_add_now (&test, crc);
539 main (int argc, char *argv[])
541 char dir_name[PATH_MAX];
542 char cfg_name[PATH_MAX];
543 char *const xargv[] = {
544 "perf-plugin-datastore",
549 struct GNUNET_GETOPT_CommandLineOption options[] = {
550 GNUNET_GETOPT_OPTION_END
553 plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]);
554 GNUNET_snprintf (dir_name, sizeof(dir_name), "/tmp/perf-gnunet-datastore-%s",
556 GNUNET_DISK_directory_remove (dir_name);
557 GNUNET_log_setup ("perf-plugin-datastore",
560 GNUNET_snprintf (category, sizeof(category), "DATASTORE-%s", plugin_name);
561 GNUNET_snprintf (cfg_name, sizeof(cfg_name),
562 "perf_plugin_datastore_data_%s.conf", plugin_name);
563 GNUNET_PROGRAM_run ((sizeof(xargv) / sizeof(char *)) - 1, xargv,
564 "perf-plugin-datastore", "nohelp", options, &run, NULL);
566 fprintf (stderr, "Missed some testcases: %u\n", ok);
567 GNUNET_DISK_directory_remove (dir_name);
573 /* end of perf_plugin_datastore.c */