From 8f0e0c9dfd1020c5a202cd1927054291c034ca20 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Tue, 1 May 2018 11:53:00 +0200 Subject: [PATCH] add perf_ logic for namestore iterations, improve namestore insertion performance by 30 percent --- .gitignore | 3 + src/namestore/Makefile.am | 41 +- src/namestore/gnunet-service-namestore.c | 21 +- .../perf_namestore_api_zone_iteration.c | 389 ++++++++++++++++++ src/namestore/test_namestore_api_flat.conf | 7 + .../test_namestore_api_postgres.conf | 7 + src/namestore/test_namestore_api_sqlite.conf | 4 + src/namestore/test_plugin_namestore.c | 2 - 8 files changed, 454 insertions(+), 20 deletions(-) create mode 100644 src/namestore/perf_namestore_api_zone_iteration.c create mode 100644 src/namestore/test_namestore_api_flat.conf create mode 100644 src/namestore/test_namestore_api_postgres.conf create mode 100644 src/namestore/test_namestore_api_sqlite.conf diff --git a/.gitignore b/.gitignore index 3c739546f..b7dd58739 100644 --- a/.gitignore +++ b/.gitignore @@ -46,3 +46,6 @@ src/credential/gnunet-credential src/credential/gnunet-service-credential src/identity-provider/gnunet-idp *.patch +src/namestore/perf_namestore_api_zone_iteration_flat +src/namestore/perf_namestore_api_zone_iteration_postgres +src/namestore/perf_namestore_api_zone_iteration_sqlite diff --git a/src/namestore/Makefile.am b/src/namestore/Makefile.am index 5f18506a7..b925261b7 100644 --- a/src/namestore/Makefile.am +++ b/src/namestore/Makefile.am @@ -20,29 +20,28 @@ if USE_COVERAGE XLIBS = -lgcov endif -if HAVE_EXPERIMENTAL FLAT_PLUGIN = libgnunet_plugin_namestore_flat.la if HAVE_TESTING -FLAT_TESTS = test_plugin_namestore_flat -endif +FLAT_TESTS = test_plugin_namestore_flat \ + perf_namestore_api_zone_iteration_flat endif if HAVE_SQLITE SQLITE_PLUGIN = libgnunet_plugin_namestore_sqlite.la if HAVE_TESTING -SQLITE_TESTS = test_plugin_namestore_sqlite +SQLITE_TESTS = test_plugin_namestore_sqlite \ + perf_namestore_api_zone_iteration_sqlite endif endif if HAVE_POSTGRESQL -# postgres doesn't even build yet; thus: experimental! POSTGRES_PLUGIN = libgnunet_plugin_namestore_postgres.la if HAVE_TESTING -POSTGRES_TESTS = test_plugin_namestore_postgres +POSTGRES_TESTS = test_plugin_namestore_postgres \ + perf_namestore_api_zone_iteration_postgres endif endif -# testcases do not even build yet; thus: experimental! if HAVE_TESTING TESTING_TESTS = \ test_namestore_api_store.nc \ @@ -81,7 +80,6 @@ test_namestore_api_zone_iteration_specific_zone.log: test_namestore_api_zone_ite test_namestore_api_zone_iteration_stop.log: test_namestore_api_monitoring.log test_namestore_api_monitoring.log: test_namestore_api_monitoring_existing.log - if HAVE_SQLITE check_PROGRAMS = \ $(SQLITE_TESTS) \ @@ -341,6 +339,30 @@ test_namestore_api_zone_iteration_nc_LDADD = \ $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ libgnunetnamestore.la +perf_namestore_api_zone_iteration_postgres_SOURCES = \ + perf_namestore_api_zone_iteration.c +perf_namestore_api_zone_iteration_postgres_LDADD = \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + libgnunetnamestore.la + +perf_namestore_api_zone_iteration_sqlite_SOURCES = \ + perf_namestore_api_zone_iteration.c +perf_namestore_api_zone_iteration_sqlite_LDADD = \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + libgnunetnamestore.la + +perf_namestore_api_zone_iteration_flat_SOURCES = \ + perf_namestore_api_zone_iteration.c +perf_namestore_api_zone_iteration_flat_LDADD = \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/gnsrecord/libgnunetgnsrecord.la \ + libgnunetnamestore.la + test_namestore_api_zone_iteration_nick_nc_SOURCES = \ test_namestore_api_zone_iteration_nick.c test_namestore_api_zone_iteration_nick_nc_LDADD = \ @@ -391,6 +413,9 @@ check_SCRIPTS = \ EXTRA_DIST = \ test_namestore_api.conf \ + test_namestore_api_postgres.conf \ + test_namestore_api_sqlite.conf \ + test_namestore_api_flat.conf \ test_plugin_namestore_sqlite.conf \ test_plugin_namestore_postgres.conf \ test_plugin_namestore_flat.conf \ diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c index e3936a7cc..0456e5c15 100644 --- a/src/namestore/gnunet-service-namestore.c +++ b/src/namestore/gnunet-service-namestore.c @@ -1032,7 +1032,8 @@ check_record_store (void *cls, GNUNET_break (0); return GNUNET_SYSERR; } - if ((0 == name_len) || (name_len > MAX_NAME_LEN)) + if ( (0 == name_len) || + (name_len > MAX_NAME_LEN) ) { GNUNET_break (0); return GNUNET_SYSERR; @@ -1066,7 +1067,6 @@ handle_record_store (void *cls, const char *rd_ser; unsigned int rd_count; int res; - struct GNUNET_CRYPTO_EcdsaPublicKey pubkey; struct ZoneMonitor *zm; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -1093,8 +1093,6 @@ handle_record_store (void *cls, } /* Extracting and converting private key */ - GNUNET_CRYPTO_ecdsa_key_get_public (&rp_msg->private_key, - &pubkey); conv_name = GNUNET_GNSRECORD_string_to_lowercase (name_tmp); if (NULL == conv_name) { @@ -1105,11 +1103,9 @@ handle_record_store (void *cls, return; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Creating %u records for name `%s' in zone `%s'\n", + "Creating %u records for name `%s'\n", (unsigned int) rd_count, - conv_name, - GNUNET_GNSRECORD_z2s (&pubkey)); - + conv_name); if ( (0 == rd_count) && (GNUNET_NO == GSN_database->iterate_records (GSN_database->cls, @@ -1150,7 +1146,8 @@ handle_record_store (void *cls, { for (zm = monitor_head; NULL != zm; zm = zm->next) { - if ( (0 == memcmp (&rp_msg->private_key, &zm->zone, + if ( (0 == memcmp (&rp_msg->private_key, + &zm->zone, sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) || (0 == memcmp (&zm->zone, &zero, @@ -1363,7 +1360,7 @@ struct ZoneIterationProcResult /** * Process results for zone iteration from database - * + * * @param cls struct ZoneIterationProcResult * @param seq sequence number of the record * @param zone_key the zone key @@ -1446,6 +1443,10 @@ run_zone_iteration_round (struct ZoneIteration *zi, memset (&proc, 0, sizeof (proc)); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asked to return up to %llu records at position %llu\n", + (unsigned long long) limit, + (unsigned long long) zi->seq); proc.zi = zi; proc.limit = limit; start = GNUNET_TIME_absolute_get (); diff --git a/src/namestore/perf_namestore_api_zone_iteration.c b/src/namestore/perf_namestore_api_zone_iteration.c new file mode 100644 index 000000000..8b5c53576 --- /dev/null +++ b/src/namestore/perf_namestore_api_zone_iteration.c @@ -0,0 +1,389 @@ +/* + This file is part of GNUnet. + Copyright (C) 2013, 2018 GNUnet e.V. + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, + Boston, MA 02110-1301, USA. +*/ +/** + * @file namestore/perf_namestore_api_zone_iteration.c + * @brief testcase for zone iteration functionality: iterate all zones + * @author Christian Grothoff + */ +#include "platform.h" +#include "gnunet_namestore_service.h" +#include "gnunet_testing_lib.h" +#include "namestore.h" + +/** + * A #BENCHMARK_SIZE of 1000 takes less than a minute on a reasonably + * modern system, so 30 minutes should be OK even for very, very + * slow systems. + */ +#define TIMEOUT GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30) + +/** + * The runtime of the benchmark is expected to be linear + * for the iteration phase with a *good* database. The FLAT + * database uses a quadratic retrieval algorithm, + * hence it should be quadratic in the size. + */ +#define BENCHMARK_SIZE 1000 + +/** + * Maximum record size + */ +#define MAX_REC_SIZE 500 + +/** + * How big are the blocks we fetch? Note that the first block is + * always just 1 record set per current API. Smaller block + * sizes will make quadratic iteration-by-offset penalties + * more pronounced. + */ +#define BLOCK_SIZE 100 + +static struct GNUNET_NAMESTORE_Handle *nsh; + +static struct GNUNET_SCHEDULER_Task *timeout_task; + +static struct GNUNET_SCHEDULER_Task *t; + +static struct GNUNET_CRYPTO_EcdsaPrivateKey *privkey; + +static struct GNUNET_NAMESTORE_ZoneIterator *zi; + +static struct GNUNET_NAMESTORE_QueueEntry *qe; + +static int res; + +static char *directory; + +static unsigned int off; + +static unsigned int left_until_next; + +static uint8_t seen[1 + BENCHMARK_SIZE / 8]; + +static struct GNUNET_TIME_Absolute start; + + +/** + * Terminate everything + * + * @param cls NULL + */ +static void +end (void *cls) +{ + (void) cls; + if (NULL != qe) + { + GNUNET_NAMESTORE_cancel (qe); + qe = NULL; + } + if (NULL != zi) + { + GNUNET_NAMESTORE_zone_iteration_stop (zi); + zi = NULL; + } + if (NULL != nsh) + { + GNUNET_NAMESTORE_disconnect (nsh); + nsh = NULL; + } + if (NULL != t) + { + GNUNET_SCHEDULER_cancel (t); + t = NULL; + } + if (NULL != timeout_task) + { + GNUNET_SCHEDULER_cancel (timeout_task); + timeout_task = NULL; + } + if (NULL != privkey) + { + GNUNET_free (privkey); + privkey = NULL; + } + res = 1; +} + + +/** + * End with timeout. As this is a benchmark, we do not + * fail hard but return "skipped". + */ +static void +timeout (void *cls) +{ + (void) cls; + timeout_task = NULL; + GNUNET_SCHEDULER_shutdown (); + res = 77; +} + + +static struct GNUNET_GNSRECORD_Data * +create_record (unsigned int count) +{ + struct GNUNET_GNSRECORD_Data *rd; + + rd = GNUNET_malloc (count + sizeof (struct GNUNET_GNSRECORD_Data)); + rd->expiration_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS).abs_value_us; + rd->record_type = count; + rd->data_size = count; + rd->data = (void *) &rd[1]; + rd->flags = 0; + memset (&rd[1], + 'a', + count); + return rd; +} + + +static void +zone_end (void *cls) +{ + struct GNUNET_TIME_Relative delay; + + zi = NULL; + delay = GNUNET_TIME_absolute_get_duration (start); + fprintf (stdout, + "Iterating over %u records took %s\n", + off, + GNUNET_STRINGS_relative_time_to_string (delay, + GNUNET_YES)); + if (BENCHMARK_SIZE == off) + { + res = 0; + } + else + { + GNUNET_break (0); + res = 1; + } + GNUNET_SCHEDULER_shutdown (); +} + + +static void +fail_cb (void *cls) +{ + zi = NULL; + res = 2; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); +} + + +static void +zone_proc (void *cls, + const struct GNUNET_CRYPTO_EcdsaPrivateKey *zone, + const char *label, + unsigned int rd_count, + const struct GNUNET_GNSRECORD_Data *rd) +{ + struct GNUNET_GNSRECORD_Data *wrd; + unsigned int xoff; + + GNUNET_assert (NULL != zone); + if (1 != sscanf (label, + "l%u", + &xoff)) + { + res = 3; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + return; + } + if ( (xoff > BENCHMARK_SIZE) || + (0 != (seen[xoff / 8] & (1U << (xoff % 8)))) ) + { + res = 3; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + return; + } + seen[xoff / 8] |= (1U << (xoff % 8)); + wrd = create_record (xoff % MAX_REC_SIZE); + if ( (rd->record_type != wrd->record_type) || + (rd->data_size != wrd->data_size) || + (rd->flags != wrd->flags) ) + { + res = 4; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + GNUNET_free (wrd); + return; + } + if (0 != memcmp (rd->data, + wrd->data, + wrd->data_size)) + { + res = 4; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + GNUNET_free (wrd); + return; + } + GNUNET_free (wrd); + if (0 != memcmp (zone, + privkey, + sizeof (struct GNUNET_CRYPTO_EcdsaPrivateKey))) + { + res = 5; + GNUNET_break (0); + GNUNET_SCHEDULER_shutdown (); + return; + } + off++; + left_until_next--; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Obtained record %u, expecting %u more until asking for mor explicitly\n", + off, + left_until_next); + if (0 == left_until_next) + { + left_until_next = BLOCK_SIZE; + GNUNET_NAMESTORE_zone_iterator_next (zi, + left_until_next); + } +} + + +static void +publish_record (void *cls); + + +static void +put_cont (void *cls, + int32_t success, + const char *emsg) +{ + (void) cls; + qe = NULL; + GNUNET_assert (GNUNET_OK == success); + t = GNUNET_SCHEDULER_add_now (&publish_record, + NULL); +} + + +static void +publish_record (void *cls) +{ + struct GNUNET_GNSRECORD_Data *rd; + char *label; + + (void) cls; + t = NULL; + if (BENCHMARK_SIZE == off) + { + struct GNUNET_TIME_Relative delay; + + delay = GNUNET_TIME_absolute_get_duration (start); + fprintf (stdout, + "Inserting %u records took %s\n", + off, + GNUNET_STRINGS_relative_time_to_string (delay, + GNUNET_YES)); + start = GNUNET_TIME_absolute_get (); + off = 0; + left_until_next = 1; + zi = GNUNET_NAMESTORE_zone_iteration_start (nsh, + NULL, + &fail_cb, + NULL, + &zone_proc, + NULL, + &zone_end, + NULL); + GNUNET_assert (NULL != zi); + return; + } + rd = create_record ((++off) % MAX_REC_SIZE); + GNUNET_asprintf (&label, + "l%u", + off); + qe = GNUNET_NAMESTORE_records_store (nsh, + privkey, + label, + 1, rd, + &put_cont, + NULL); + GNUNET_free (label); + GNUNET_free (rd); +} + + +static void +run (void *cls, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Peer *peer) +{ + directory = NULL; + GNUNET_assert (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_string(cfg, + "PATHS", + "GNUNET_TEST_HOME", + &directory)); + GNUNET_DISK_directory_remove (directory); + GNUNET_SCHEDULER_add_shutdown (&end, + NULL); + timeout_task = GNUNET_SCHEDULER_add_delayed (TIMEOUT, + &timeout, + NULL); + nsh = GNUNET_NAMESTORE_connect (cfg); + GNUNET_assert (NULL != nsh); + privkey = GNUNET_CRYPTO_ecdsa_key_create (); + GNUNET_assert (NULL != privkey); + start = GNUNET_TIME_absolute_get (); + t = GNUNET_SCHEDULER_add_now (&publish_record, + NULL); +} + + +int +main (int argc, + char *argv[]) +{ + const char *plugin_name; + char *cfg_name; + + plugin_name = GNUNET_TESTING_get_testname_from_underscore (argv[0]); + GNUNET_asprintf (&cfg_name, + "test_namestore_api_%s.conf", + plugin_name); + res = 1; + if (0 != + GNUNET_TESTING_peer_run ("perf-namestore-api-zone-iteration", + cfg_name, + &run, + NULL)) + { + res = 1; + } + GNUNET_free (cfg_name); + if (NULL != directory) + { + GNUNET_DISK_directory_remove (directory); + GNUNET_free (directory); + } + return res; +} + + +/* end of perf_namestore_api_zone_iteration.c */ diff --git a/src/namestore/test_namestore_api_flat.conf b/src/namestore/test_namestore_api_flat.conf new file mode 100644 index 000000000..26e2f2c51 --- /dev/null +++ b/src/namestore/test_namestore_api_flat.conf @@ -0,0 +1,7 @@ +@INLINE@ test_namestore_api.conf + +[namestore] +DATABASE = flat + +[namecache] +DISABLE = YES diff --git a/src/namestore/test_namestore_api_postgres.conf b/src/namestore/test_namestore_api_postgres.conf new file mode 100644 index 000000000..259ce35e7 --- /dev/null +++ b/src/namestore/test_namestore_api_postgres.conf @@ -0,0 +1,7 @@ +@INLINE@ test_namestore_api.conf + +[namestore] +DATABASE = postgres + +[namecache] +DISABLE = YES diff --git a/src/namestore/test_namestore_api_sqlite.conf b/src/namestore/test_namestore_api_sqlite.conf new file mode 100644 index 000000000..72b609226 --- /dev/null +++ b/src/namestore/test_namestore_api_sqlite.conf @@ -0,0 +1,4 @@ +@INLINE@ test_namestore_api.conf + +[namecache] +DISABLE = YES diff --git a/src/namestore/test_plugin_namestore.c b/src/namestore/test_plugin_namestore.c index ac2a0bba9..6bccd1706 100644 --- a/src/namestore/test_plugin_namestore.c +++ b/src/namestore/test_plugin_namestore.c @@ -207,7 +207,6 @@ main (int argc, GNUNET_GETOPT_OPTION_END }; - //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite"); GNUNET_log_setup ("test-plugin-namestore", "WARNING", NULL); @@ -227,7 +226,6 @@ main (int argc, FPRINTF (stderr, "Missed some testcases: %d\n", ok); - //GNUNET_DISK_directory_remove ("/tmp/gnunet-test-plugin-namestore-sqlite"); return ok; } -- 2.25.1