From: Christian Grothoff Date: Sun, 26 Feb 2017 21:42:40 +0000 (+0100) Subject: ensure all plugins properly use BF, move shared logic to shared library X-Git-Tag: taler-0.2.1~24 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=8d7c29c4684f807d5e9a3004bbbab132b158c5aa;p=oweals%2Fgnunet.git ensure all plugins properly use BF, move shared logic to shared library --- diff --git a/src/block/bg_bf.c b/src/block/bg_bf.c index 9c4dc9060..1a17ec84e 100644 --- a/src/block/bg_bf.c +++ b/src/block/bg_bf.c @@ -232,4 +232,36 @@ GNUNET_BLOCK_GROUP_bf_test_and_set (struct GNUNET_BLOCK_Group *bg, } +/** + * How many bytes should a bloomfilter be if we have already seen + * entry_count responses? Sized so that do not have to + * re-size the filter too often (to keep it cheap). + * + * Since other peers will also add entries but not resize the filter, + * we should generally pick a slightly larger size than what the + * strict math would suggest. + * + * @param entry_count expected number of entries in the Bloom filter + * @param k number of bits set per entry + * @return must be a power of two and smaller or equal to 2^15. + */ +size_t +GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count, + unsigned int k) +{ + size_t size; + unsigned int ideal = (entry_count * k) / 4; + uint16_t max = 1 << 15; + + if (entry_count > max) + return max; + size = 8; + while ((size < max) && (size < ideal)) + size *= 2; + if (size > max) + return max; + return size; +} + + /* end of bg_bf.c */ diff --git a/src/block/plugin_block_template.c b/src/block/plugin_block_template.c index 87adae7e9..b714b6858 100644 --- a/src/block/plugin_block_template.c +++ b/src/block/plugin_block_template.c @@ -43,37 +43,6 @@ #define TEMPLATE_BF_SIZE 8 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -100,7 +69,8 @@ block_plugin_template_create_group (void *cls, guard = va_arg (va, const char *); if (0 == strcmp (guard, "seen-set-size")) - bf_size = compute_bloomfilter_size (va_arg (va, unsigned int)); + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); else if (0 == strcmp (guard, "filter-size")) bf_size = va_arg (va, unsigned int); diff --git a/src/block/plugin_block_test.c b/src/block/plugin_block_test.c index 31112e5dd..b9f71cacb 100644 --- a/src/block/plugin_block_test.c +++ b/src/block/plugin_block_test.c @@ -41,37 +41,6 @@ #define TEST_BF_SIZE 8 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -98,7 +67,8 @@ block_plugin_test_create_group (void *cls, guard = va_arg (va, const char *); if (0 == strcmp (guard, "seen-set-size")) - bf_size = compute_bloomfilter_size (va_arg (va, unsigned int)); + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); else if (0 == strcmp (guard, "filter-size")) bf_size = va_arg (va, unsigned int); diff --git a/src/dht/plugin_block_dht.c b/src/dht/plugin_block_dht.c index 29c08dd50..74498746c 100644 --- a/src/dht/plugin_block_dht.c +++ b/src/dht/plugin_block_dht.c @@ -40,37 +40,6 @@ #define BLOOMFILTER_K 16 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -97,7 +66,8 @@ block_plugin_dht_create_group (void *cls, guard = va_arg (va, const char *); if (0 == strcmp (guard, "seen-set-size")) - bf_size = compute_bloomfilter_size (va_arg (va, unsigned int)); + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); else if (0 == strcmp (guard, "filter-size")) bf_size = va_arg (va, unsigned int); diff --git a/src/dns/plugin_block_dns.c b/src/dns/plugin_block_dns.c index 65da0de63..fe416e450 100644 --- a/src/dns/plugin_block_dns.c +++ b/src/dns/plugin_block_dns.c @@ -1,6 +1,6 @@ /* This file is part of GNUnet - Copyright (C) 2013 GNUnet e.V. + Copyright (C) 2013, 2017 GNUnet e.V. GNUnet is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published @@ -31,6 +31,61 @@ #include "gnunet_block_plugin.h" #include "block_dns.h" #include "gnunet_signatures.h" +#include "gnunet_block_group_lib.h" + + +/** + * Number of bits we set per entry in the bloomfilter. + * Do not change! + */ +#define BLOOMFILTER_K 16 + + +/** + * Create a new block group. + * + * @param ctx block context in which the block group is created + * @param type type of the block for which we are creating the group + * @param nonce random value used to seed the group creation + * @param raw_data optional serialized prior state of the group, NULL if unavailable/fresh + * @param raw_data_size number of bytes in @a raw_data, 0 if unavailable/fresh + * @param va variable arguments specific to @a type + * @return block group handle, NULL if block groups are not supported + * by this @a type of block (this is not an error) + */ +static struct GNUNET_BLOCK_Group * +block_plugin_dns_create_group (void *cls, + enum GNUNET_BLOCK_Type type, + uint32_t nonce, + const void *raw_data, + size_t raw_data_size, + va_list va) +{ + unsigned int bf_size; + const char *guard; + + guard = va_arg (va, const char *); + if (0 == strcmp (guard, + "seen-set-size")) + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); + else if (0 == strcmp (guard, + "filter-size")) + bf_size = va_arg (va, unsigned int); + else + { + GNUNET_break (0); + bf_size = 8; + } + GNUNET_break (NULL == va_arg (va, const char *)); + return GNUNET_BLOCK_GROUP_bf_create (cls, + bf_size, + BLOOMFILTER_K, + type, + nonce, + raw_data, + raw_data_size); +} /** @@ -60,6 +115,7 @@ block_plugin_dns_evaluate (void *cls, size_t reply_block_size) { const struct GNUNET_DNS_Advertisement *ad; + struct GNUNET_HashCode phash; switch (type) { @@ -101,6 +157,13 @@ block_plugin_dns_evaluate (void *cls, GNUNET_break_op (0); return GNUNET_BLOCK_EVALUATION_RESULT_INVALID; } + GNUNET_CRYPTO_hash (reply_block, + reply_block_size, + &phash); + if (GNUNET_YES == + GNUNET_BLOCK_GROUP_bf_test_and_set (bg, + &phash)) + return GNUNET_BLOCK_EVALUATION_OK_DUPLICATE; return GNUNET_BLOCK_EVALUATION_OK_MORE; default: return GNUNET_BLOCK_EVALUATION_TYPE_NOT_SUPPORTED; @@ -147,6 +210,7 @@ libgnunet_plugin_block_dns_init (void *cls) api = GNUNET_new (struct GNUNET_BLOCK_PluginFunctions); api->evaluate = &block_plugin_dns_evaluate; api->get_key = &block_plugin_dns_get_key; + api->create_group = &block_plugin_dns_create_group; api->types = types; return api; } diff --git a/src/fs/plugin_block_fs.c b/src/fs/plugin_block_fs.c index 996c06dcb..b6749b418 100644 --- a/src/fs/plugin_block_fs.c +++ b/src/fs/plugin_block_fs.c @@ -38,37 +38,6 @@ #define BLOOMFILTER_K 16 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -111,7 +80,8 @@ block_plugin_fs_create_group (void *cls, } else { - size = compute_bloomfilter_size (va_arg (va, unsigned int)); + size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); } if (0 == size) size = raw_data_size; /* not for us to determine, use what we got! */ diff --git a/src/gns/plugin_block_gns.c b/src/gns/plugin_block_gns.c index 8ce2d85da..2ef48f094 100644 --- a/src/gns/plugin_block_gns.c +++ b/src/gns/plugin_block_gns.c @@ -43,37 +43,6 @@ #define GNS_BF_SIZE 8 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -100,7 +69,8 @@ block_plugin_gns_create_group (void *cls, guard = va_arg (va, const char *); if (0 == strcmp (guard, "seen-set-size")) - bf_size = compute_bloomfilter_size (va_arg (va, unsigned int)); + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); else if (0 == strcmp (guard, "filter-size")) bf_size = va_arg (va, unsigned int); diff --git a/src/include/gnunet_block_group_lib.h b/src/include/gnunet_block_group_lib.h index a1ea807f6..3a3dfb2e2 100644 --- a/src/include/gnunet_block_group_lib.h +++ b/src/include/gnunet_block_group_lib.h @@ -43,6 +43,24 @@ extern "C" #endif +/** + * How many bytes should a bloomfilter be if we have already seen + * entry_count responses? Sized so that do not have to + * re-size the filter too often (to keep it cheap). + * + * Since other peers will also add entries but not resize the filter, + * we should generally pick a slightly larger size than what the + * strict math would suggest. + * + * @param entry_count expected number of entries in the Bloom filter + * @param k number of bits set per entry + * @return must be a power of two and smaller or equal to 2^15. + */ +size_t +GNUNET_BLOCK_GROUP_compute_bloomfilter_size (unsigned int entry_count, + unsigned int k); + + /** * Create a new block group that filters duplicates using a Bloom filter. * diff --git a/src/regex/plugin_block_regex.c b/src/regex/plugin_block_regex.c index 19335e815..a345dfedb 100644 --- a/src/regex/plugin_block_regex.c +++ b/src/regex/plugin_block_regex.c @@ -31,7 +31,6 @@ #include "gnunet_signatures.h" - /** * Number of bits we set per entry in the bloomfilter. * Do not change! @@ -45,37 +44,6 @@ #define REGEX_BF_SIZE 8 -/** - * How many bytes should a bloomfilter be if we have already seen - * entry_count responses? Note that #GNUNET_CONSTANTS_BLOOMFILTER_K - * gives us the number of bits set per entry. Furthermore, we should - * not re-size the filter too often (to keep it cheap). - * - * Since other peers will also add entries but not resize the filter, - * we should generally pick a slightly larger size than what the - * strict math would suggest. - * - * @param entry_count expected number of entries in the Bloom filter - * @return must be a power of two and smaller or equal to 2^15. - */ -static size_t -compute_bloomfilter_size (unsigned int entry_count) -{ - size_t size; - unsigned int ideal = (entry_count * BLOOMFILTER_K) / 4; - uint16_t max = 1 << 15; - - if (entry_count > max) - return max; - size = 8; - while ((size < max) && (size < ideal)) - size *= 2; - if (size > max) - return max; - return size; -} - - /** * Create a new block group. * @@ -102,7 +70,8 @@ block_plugin_regex_create_group (void *cls, guard = va_arg (va, const char *); if (0 == strcmp (guard, "seen-set-size")) - bf_size = compute_bloomfilter_size (va_arg (va, unsigned int)); + bf_size = GNUNET_BLOCK_GROUP_compute_bloomfilter_size (va_arg (va, unsigned int), + BLOOMFILTER_K); else if (0 == strcmp (guard, "filter-size")) bf_size = va_arg (va, unsigned int);