From 303334e67262bb6121dfbd245c66535f259d08af Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Wed, 25 Apr 2018 16:41:22 +0200 Subject: [PATCH] enable caching private->public key mapping in memory to improve CPU consumption for large zone insertions --- src/gns/gnunet-gns-proxy.c | 14 +-- src/gnsrecord/gnsrecord_crypto.c | 108 +++++++++++++++++++++-- src/include/gnunet_gnsrecord_lib.h | 19 ++++ src/namestore/gnunet-service-namestore.c | 26 ++++-- src/namestore/namestore.conf.in | 11 ++- 5 files changed, 155 insertions(+), 23 deletions(-) diff --git a/src/gns/gnunet-gns-proxy.c b/src/gns/gnunet-gns-proxy.c index 8b9aa599e..08663a57e 100644 --- a/src/gns/gnunet-gns-proxy.c +++ b/src/gns/gnunet-gns-proxy.c @@ -855,9 +855,9 @@ mhd_content_cb (void *cls, return MHD_CONTENT_READER_END_OF_STREAM; } GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Writing %lu/%lu bytes\n", - bytes_to_copy, - s5r->io_len); + "Writing %llu/%llu bytes\n", + (unsigned long long) bytes_to_copy, + (unsigned long long) s5r->io_len); GNUNET_memcpy (buf, s5r->io_buf, bytes_to_copy); @@ -1307,12 +1307,12 @@ curl_download_cb (void *ptr, if (sizeof (s5r->io_buf) - s5r->io_len < total) { GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Pausing CURL `%s%s' download, not enough space %lu %lu %lu\n", + "Pausing CURL `%s%s' download, not enough space %llu %llu %llu\n", s5r->domain, s5r->url, - sizeof (s5r->io_buf), - s5r->io_len, - total); + (unsigned long long) sizeof (s5r->io_buf), + (unsigned long long) s5r->io_len, + (unsigned long long) total); return CURL_WRITEFUNC_PAUSE; /* not enough space */ } GNUNET_memcpy (&s5r->io_buf[s5r->io_len], diff --git a/src/gnsrecord/gnsrecord_crypto.c b/src/gnsrecord/gnsrecord_crypto.c index dd628ea76..27c83b90e 100644 --- a/src/gnsrecord/gnsrecord_crypto.c +++ b/src/gnsrecord/gnsrecord_crypto.c @@ -72,6 +72,7 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, * Sign name and records * * @param key the private key + * @param pkey associated public key * @param expire block expiration * @param label the name for the records * @param rd record data @@ -79,17 +80,17 @@ derive_block_aes_key (struct GNUNET_CRYPTO_SymmetricInitializationVector *iv, * @return NULL on error (block too large) */ struct GNUNET_GNSRECORD_Block * -GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, - struct GNUNET_TIME_Absolute expire, - const char *label, - const struct GNUNET_GNSRECORD_Data *rd, - unsigned int rd_count) +block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, + const struct GNUNET_CRYPTO_EcdsaPublicKey *pkey, + struct GNUNET_TIME_Absolute expire, + const char *label, + const struct GNUNET_GNSRECORD_Data *rd, + unsigned int rd_count) { size_t payload_len = GNUNET_GNSRECORD_records_get_size (rd_count, rd); char payload[sizeof (uint32_t) + payload_len]; struct GNUNET_GNSRECORD_Block *block; - struct GNUNET_CRYPTO_EcdsaPublicKey pkey; struct GNUNET_CRYPTO_EcdsaPrivateKey *dkey; struct GNUNET_CRYPTO_SymmetricInitializationVector iv; struct GNUNET_CRYPTO_SymmetricSessionKey skey; @@ -139,12 +140,10 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, "gns"); GNUNET_CRYPTO_ecdsa_key_get_public (dkey, &block->derived_key); - GNUNET_CRYPTO_ecdsa_key_get_public (key, - &pkey); derive_block_aes_key (&iv, &skey, label, - &pkey); + pkey); GNUNET_break (payload_len + sizeof (uint32_t) == GNUNET_CRYPTO_symmetric_encrypt (payload, payload_len + sizeof (uint32_t), @@ -166,6 +165,97 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, } +/** + * Sign name and records + * + * @param key the private key + * @param expire block expiration + * @param label the name for the records + * @param rd record data + * @param rd_count number of records + * @return NULL on error (block too large) + */ +struct GNUNET_GNSRECORD_Block * +GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, + struct GNUNET_TIME_Absolute expire, + const char *label, + const struct GNUNET_GNSRECORD_Data *rd, + unsigned int rd_count) +{ + struct GNUNET_CRYPTO_EcdsaPublicKey pkey; + + GNUNET_CRYPTO_ecdsa_key_get_public (key, + &pkey); + return block_create (key, + &pkey, + expire, + label, + rd, + rd_count); +} + + +/** + * Line in cache mapping private keys to public keys. + */ +struct KeyCacheLine +{ + /** + * A private key. + */ + struct GNUNET_CRYPTO_EcdsaPrivateKey key; + + /** + * Associated public key. + */ + struct GNUNET_CRYPTO_EcdsaPublicKey pkey; + +}; + + +/** + * Sign name and records, cache derived public key (also keeps the + * private key in static memory, so do not use this function if + * keeping the private key in the process'es RAM is a major issue). + * + * @param key the private key + * @param expire block expiration + * @param label the name for the records + * @param rd record data + * @param rd_count number of records + * @return NULL on error (block too large) + */ +struct GNUNET_GNSRECORD_Block * +GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, + struct GNUNET_TIME_Absolute expire, + const char *label, + const struct GNUNET_GNSRECORD_Data *rd, + unsigned int rd_count) +{ +#define CSIZE 64 + static struct KeyCacheLine cache[CSIZE]; + struct KeyCacheLine *line; + + line = &cache[(*(unsigned int *) key) ^ CSIZE]; + if (0 != memcmp (&line->key, + key, + sizeof (*key))) + { + /* cache miss, recompute */ + line->key = *key; + GNUNET_CRYPTO_ecdsa_key_get_public (key, + &line->pkey); + } + return block_create (key, + &line->pkey, + expire, + label, + rd, + rd_count); +} + + + /** * Check if a signature is valid. This API is used by the GNS Block * to validate signatures received from the network. diff --git a/src/include/gnunet_gnsrecord_lib.h b/src/include/gnunet_gnsrecord_lib.h index f07bd3ef3..38346ada3 100644 --- a/src/include/gnunet_gnsrecord_lib.h +++ b/src/include/gnunet_gnsrecord_lib.h @@ -557,6 +557,25 @@ GNUNET_GNSRECORD_block_create (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, unsigned int rd_count); +/** + * Sign name and records, cache derived public key (also keeps the + * private key in static memory, so do not use this function if + * keeping the private key in the process'es RAM is a major issue). + * + * @param key the private key + * @param expire block expiration + * @param label the name for the records + * @param rd record data + * @param rd_count number of records in @a rd + */ +struct GNUNET_GNSRECORD_Block * +GNUNET_GNSRECORD_block_create2 (const struct GNUNET_CRYPTO_EcdsaPrivateKey *key, + struct GNUNET_TIME_Absolute expire, + const char *label, + const struct GNUNET_GNSRECORD_Data *rd, + unsigned int rd_count); + + /** * Check if a signature is valid. This API is used by the GNS Block * to validate signatures received from the network. diff --git a/src/namestore/gnunet-service-namestore.c b/src/namestore/gnunet-service-namestore.c index f8ac6c31c..c5a37dcac 100644 --- a/src/namestore/gnunet-service-namestore.c +++ b/src/namestore/gnunet-service-namestore.c @@ -251,6 +251,12 @@ static struct ZoneMonitor *monitor_tail; */ static struct GNUNET_NotificationContext *monitor_nc; +/** + * Optimize block insertion by caching map of private keys to + * public keys in memory? + */ +static int cache_keys; + /** * Task run during shutdown. @@ -707,11 +713,18 @@ refresh_block (struct NamestoreClient *nc, ? GNUNET_TIME_UNIT_ZERO_ABS : GNUNET_GNSRECORD_record_get_expiration_time (res_count, res); - block = GNUNET_GNSRECORD_block_create (zone_key, - exp_time, - name, - res, - res_count); + if (cache_keys) + block = GNUNET_GNSRECORD_block_create2 (zone_key, + exp_time, + name, + res, + res_count); + else + block = GNUNET_GNSRECORD_block_create (zone_key, + exp_time, + name, + res, + res_count); GNUNET_assert (NULL != block); GNUNET_CRYPTO_ecdsa_key_get_public (zone_key, &pkey); @@ -1722,6 +1735,9 @@ run (void *cls, monitor_nc = GNUNET_notification_context_create (1); namecache = GNUNET_NAMECACHE_connect (cfg); /* Loading database plugin */ + cache_keys = GNUNET_CONFIGURATION_get_value_yesno (cfg, + "namestore", + "CACHE_KEYS"); if (GNUNET_OK != GNUNET_CONFIGURATION_get_value_string (cfg, "namestore", diff --git a/src/namestore/namestore.conf.in b/src/namestore/namestore.conf.in index f19ac5643..8b5e440b8 100644 --- a/src/namestore/namestore.conf.in +++ b/src/namestore/namestore.conf.in @@ -9,8 +9,17 @@ HOSTNAME = localhost BINARY = gnunet-service-namestore ACCEPT_FROM = 127.0.0.1; ACCEPT_FROM6 = ::1; + +# Which database should we use? DATABASE = sqlite +# Should we optimize publishing record by caching the mapping +# from zone private keys to zone public keys in memory? +# (Set to NO if totally paranoid about keeping private keys +# in RAM longer than necessary.) +CACHE_KEYS = YES + + [namestore-sqlite] FILENAME = $GNUNET_DATA_HOME/namestore/sqlite.db @@ -38,5 +47,3 @@ UNIXPATH = $GNUNET_RUNTIME_DIR/gnunet-service-fcfsd.sock # On what port does the FCFS daemon listen for HTTP clients? HTTPPORT = 18080 - - -- 2.25.1