From: Nathan S. Evans Date: Tue, 20 Jul 2010 11:33:59 +0000 (+0000) Subject: dht testcases and some minor other changes... X-Git-Tag: initial-import-from-subversion-38251~20911 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=cf1d9e9e9f7765a9849f6f41f33ccb7615272eb5;p=oweals%2Fgnunet.git dht testcases and some minor other changes... --- diff --git a/src/dht/Makefile.am b/src/dht/Makefile.am index fbe0c8ab3..e3f78b870 100644 --- a/src/dht/Makefile.am +++ b/src/dht/Makefile.am @@ -1,16 +1,31 @@ INCLUDES = -I$(top_srcdir)/src/include - if MINGW WINFLAGS = -Wl,--no-undefined -Wl,--export-all-symbols -lole32 -lshell32 -liconv -lstdc++ -lcomdlg32 -lgdi32 endif +if HAVE_ZLIB + ZLIB_LNK = -lz +endif + if USE_COVERAGE AM_CFLAGS = --coverage -O0 XLIB = -lgcov endif lib_LTLIBRARIES = libgnunetdht.la +# libgnunetdhtlog.la + +#libgnunetdhtlog_la_SOURCES = \ +# dhtlog_mysql.c dhtlog.h +#libgnunetdhtlog_la_LIBADD = \ +# $(top_builddir)/src/util/libgnunetutil.la \ +# $(XLIB) +#libgnunetdhtlog_la_LDFLAGS = \ +# $(GN_LIB_LDFLAGS) $(WINFLAGS) \ +# -lmysqlclient $(ZLIB_LNK) \ +# -version-info 0:0:0 + libgnunetdht_la_SOURCES = \ dht_api.c dht.h libgnunetdht_la_LIBADD = \ @@ -20,12 +35,12 @@ libgnunetdht_la_LDFLAGS = \ $(GN_LIB_LDFLAGS) $(WINFLAGS) \ -version-info 0:0:0 - bin_PROGRAMS = \ gnunet-service-dht \ gnunet-dht-get \ gnunet-dht-get-peer \ - gnunet-dht-put + gnunet-dht-put \ + gnunet-dht-driver gnunet_service_dht_SOURCES = \ gnunet-service-dht.c @@ -37,6 +52,17 @@ gnunet_service_dht_LDADD = \ $(top_builddir)/src/datacache/libgnunetdatacache.la \ $(top_builddir)/src/util/libgnunetutil.la +#gnunet_service_dht_new_SOURCES = \ +# gnunet-service-dht-new.c +#gnunet_service_dht_new_LDADD = \ +# $(top_builddir)/src/statistics/libgnunetstatistics.la \ +# $(top_builddir)/src/core/libgnunetcore.la \ +# $(top_builddir)/src/transport/libgnunettransport.la \ +# $(top_builddir)/src/hello/libgnunethello.la \ +# $(top_builddir)/src/datacache/libgnunetdatacache.la \ +# $(top_builddir)/src/util/libgnunetutil.la \ +# $(top_builddir)/src/dht/libgnunetdhtlog.la + gnunet_dht_get_SOURCES = \ gnunet-dht-get.c gnunet_dht_get_LDADD = \ @@ -58,11 +84,23 @@ gnunet_dht_put_LDADD = \ $(top_builddir)/src/dht/libgnunetdht.la \ $(top_builddir)/src/core/libgnunetcore.la \ $(top_builddir)/src/util/libgnunetutil.la + +gnunet_dht_driver_SOURCES = \ + gnunet-dht-driver.c +gnunet_dht_driver_LDADD = \ + $(top_builddir)/src/dht/libgnunetdht.la \ + $(top_builddir)/src/dht/libgnunetdhtlog.la \ + $(top_builddir)/src/core/libgnunetcore.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/dht/libgnunetdhtlog.la check_PROGRAMS = \ test_dht_api \ test_dht_twopeer \ - test_dht_twopeer_put_get + test_dht_twopeer_put_get \ + test_dht_multipeer +# test_dhtlog TESTS = test_dht_api $(check_SCRIPTS) @@ -73,6 +111,13 @@ test_dht_api_LDADD = \ $(top_builddir)/src/hello/libgnunethello.la \ $(top_builddir)/src/dht/libgnunetdht.la +test_dht_multipeer_SOURCES = \ + test_dht_multipeer.c +test_dht_multipeer_LDADD = \ + $(top_builddir)/src/util/libgnunetutil.la \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/dht/libgnunetdht.la + test_dht_twopeer_SOURCES = \ test_dht_twopeer.c test_dht_twopeer_LDADD = \ @@ -87,11 +132,20 @@ test_dht_twopeer_put_get_LDADD = \ $(top_builddir)/src/testing/libgnunettesting.la \ $(top_builddir)/src/dht/libgnunetdht.la +#test_dhtlog_SOURCES = \ +# test_dhtlog.c +#test_dhtlog_LDADD = \ +# $(top_builddir)/src/util/libgnunetutil.la \ +# $(top_builddir)/src/testing/libgnunettesting.la \ +# $(top_builddir)/src/dht/libgnunetdht.la \ +# $(top_builddir)/src/dht/libgnunetdhtlog.la + EXTRA_DIST = \ $(check_SCRIPTS) \ test_dht_api_data.conf \ test_dht_api_peer1.conf \ - test_dht_twopeer_data.conf + test_dht_twopeer_data.conf \ + test_dht_multipeer_data.conf check_SCRIPTS = \ test_dht_tools.sh diff --git a/src/dht/dht.h b/src/dht/dht.h index 319f6d26f..c6c9de1e1 100644 --- a/src/dht/dht.h +++ b/src/dht/dht.h @@ -29,9 +29,21 @@ #define DEBUG_DHT GNUNET_NO +#define DEBUG_DHT_ROUTING GNUNET_YES + +#define DHT_BLOOM_SIZE 16 + +#define DHT_BLOOM_K 8 + +#define MAX_OUTSTANDING_FORWARDS 100 + +#define DHT_FORWARD_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5) + +#define DHT_SEND_PRIORITY 4 + typedef void (*GNUNET_DHT_MessageReceivedHandler) (void *cls, const struct GNUNET_MessageHeader - * msg); + *msg); /** * Message which indicates the DHT should cancel outstanding @@ -54,17 +66,23 @@ struct GNUNET_DHT_StopMessage */ uint64_t unique_id GNUNET_PACKED; + /** + * Key of this request + */ + GNUNET_HashCode key; + }; /** * Generic DHT message, indicates that a route request - * should be issued. + * should be issued, if coming from a client. Shared + * usage for api->server and P2P message passing. */ struct GNUNET_DHT_RouteMessage { /** - * Type: GNUNET_MESSAGE_TYPE_DHT_ROUTE + * Type: GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE */ struct GNUNET_MessageHeader header; @@ -94,10 +112,13 @@ struct GNUNET_DHT_RouteMessage }; +/** + * Generic local route result message + */ struct GNUNET_DHT_RouteResultMessage { /** - * Type: GNUNET_MESSAGE_TYPE_DHT_ROUTE_RESULT + * Type: GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE_RESULT */ struct GNUNET_MessageHeader header; @@ -106,26 +127,147 @@ struct GNUNET_DHT_RouteResultMessage */ uint32_t options GNUNET_PACKED; + /** + * Unique ID identifying this request (necessary for + * client to compare to sent requests) + */ + uint64_t unique_id GNUNET_PACKED; + /** * The key that was searched for */ GNUNET_HashCode key; + /* GNUNET_MessageHeader *enc actual DHT message, copied to end of this dealy do */ +}; + +/** + * Generic P2P DHT route message + */ +struct GNUNET_DHT_P2PRouteMessage +{ + /** + * Type: GNUNET_MESSAGE_TYPE_P2P_DHT_ROUTE + */ + struct GNUNET_MessageHeader header; + + /** + * Message options + */ + uint32_t options GNUNET_PACKED; + + /** + * Hop count + */ + uint32_t hop_count GNUNET_PACKED; + + /** + * Network size estimate + */ + uint32_t network_size GNUNET_PACKED; + + /** + * Replication level for this message + */ + uint32_t desired_replication_level GNUNET_PACKED; + /** * Unique ID identifying this request */ uint64_t unique_id GNUNET_PACKED; + /* + * Bloomfilter to stop circular routes + */ + char bloomfilter[DHT_BLOOM_SIZE]; + + /** + * FIXME: add DHT logging for analysis! + */ +#if LOG_SQL + /* + * Unique query id for sql database interaction. + */ + uint64_t queryuid; + + /* + * Unique trial id for sql database interaction + */ + uint64_t trialuid; + +#endif + + /** + * The key to search for + */ + GNUNET_HashCode key; + + /* GNUNET_MessageHeader *enc actual DHT message, copied to end of this dealy do */ + +}; + +/** + * Generic P2P route result + * + * FIXME: One question is how much to include for a route result message. + * Assuming a peer receives such a message, but has no record of a + * route message, what should it do? It can either drop the message + * or try to forward it towards the original peer... However, for + * that to work we would need to include the original peer identity + * in the GET request, which adds even more data to the message. + */ +struct GNUNET_DHT_P2PRouteResultMessage +{ + /** + * Type: GNUNET_MESSAGE_TYPE_P2P_DHT_ROUTE_RESULT + */ + struct GNUNET_MessageHeader header; + + /** + * Message options + */ + uint32_t options GNUNET_PACKED; + + /** + * Hop count + */ + uint32_t hop_count GNUNET_PACKED; + + /** + * Unique ID identifying this request (may not be set) + */ + uint64_t unique_id GNUNET_PACKED; + + /* + * Bloomfilter to stop circular routes + */ + char bloomfilter[DHT_BLOOM_SIZE]; + + /** + * The key that was searched for + */ + GNUNET_HashCode key; + +#if FORWARD_UNKNOWN + /** + * Network size estimate + */ + uint32_t network_size GNUNET_PACKED; +#endif + /* GNUNET_MessageHeader *enc actual DHT message, copied to end of this dealy do */ }; + /** - * Message to insert data into the DHT + * Message to insert data into the DHT, shared + * between api->server communication and P2P communication. + * The type must be different for the two purposes. */ struct GNUNET_DHT_PutMessage { /** - * Type: GNUNET_MESSAGE_TYPE_DHT_PUT + * Type: GNUNET_MESSAGE_TYPE_DHT_PUT / GNUNET_MESSAGE_TYPE_P2P_DHT_PUT */ struct GNUNET_MessageHeader header; @@ -148,12 +290,15 @@ struct GNUNET_DHT_PutMessage /** - * Message to request data from the DHT + * Message to request data from the DHT, shared + * between P2P requests and local get requests. + * Main difference is that if the request comes in + * locally we need to remember it (for client response). */ struct GNUNET_DHT_GetMessage { /** - * Type: GNUNET_MESSAGE_TYPE_DHT_GET + * Type: GNUNET_MESSAGE_TYPE_DHT_GET / GNUNET_MESSAGE_TYPE_P2P_DHT_GET */ struct GNUNET_MessageHeader header; @@ -165,12 +310,14 @@ struct GNUNET_DHT_GetMessage }; /** - * Message to return data from the DHT + * Message to return data either to the client API + * or to respond to a request received from another + * peer. Shared format, different types. */ struct GNUNET_DHT_GetResultMessage { /** - * Type: GNUNET_MESSAGE_TYPE_DHT_GET_RESULT + * Type: GNUNET_MESSAGE_TYPE_DHT_GET_RESULT / GNUNET_MESSAGE_TYPE_DHT_P2P_GET_RESULT */ struct GNUNET_MessageHeader header; @@ -180,9 +327,9 @@ struct GNUNET_DHT_GetResultMessage uint32_t type; /** - * The key to search for + * The key that was searched for */ - GNUNET_HashCode key; + //GNUNET_HashCode key; /** * When does this entry expire? diff --git a/src/dht/dht_api.c b/src/dht/dht_api.c index 6c65ac8f4..3ff4bc523 100644 --- a/src/dht/dht_api.c +++ b/src/dht/dht_api.c @@ -585,7 +585,7 @@ service_message_handler (void *cls, switch (ntohs (msg->type)) { - case GNUNET_MESSAGE_TYPE_DHT_ROUTE_RESULT: + case GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE_RESULT: { dht_msg = (struct GNUNET_DHT_RouteResultMessage *) msg; uid = GNUNET_ntohll (dht_msg->unique_id); @@ -777,7 +777,7 @@ get_reply_iterator (void *cls, const struct GNUNET_MessageHeader *reply) result_data = (char *) &result[1]; /* Set data pointer to end of message */ get_handle->get_context.iter (get_handle->get_context.iter_cls, - result->expiration, &result->key, + result->expiration, &get_handle->route_handle->key, ntohs (result->type), data_size, result_data); } @@ -868,9 +868,9 @@ GNUNET_DHT_route_start (struct GNUNET_DHT_Handle *handle, route_handle->iter = iter; route_handle->iter_cls = iter_cls; route_handle->dht_handle = handle; + route_handle->uid = handle->uid_gen++; if (iter != NULL) { - route_handle->uid = handle->uid_gen++; hash_from_uid (route_handle->uid, &uid_key); GNUNET_CONTAINER_multihashmap_put (handle->outstanding_requests, &uid_key, route_handle, @@ -885,7 +885,7 @@ GNUNET_DHT_route_start (struct GNUNET_DHT_Handle *handle, msize = sizeof (struct GNUNET_DHT_RouteMessage) + ntohs (enc->size); message = GNUNET_malloc (msize); message->header.size = htons (msize); - message->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_ROUTE); + message->header.type = htons (GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE); memcpy (&message->key, key, sizeof (GNUNET_HashCode)); message->options = htonl (options); message->desired_replication_level = htonl (options); @@ -983,13 +983,14 @@ GNUNET_DHT_route_stop (struct GNUNET_DHT_RouteHandle *route_handle, msize = sizeof (struct GNUNET_DHT_StopMessage); message = GNUNET_malloc (msize); message->header.size = htons (msize); - message->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_STOP); + message->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_ROUTE_STOP); #if DEBUG_DHT_API GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "`%s': Remove outstanding request for uid %llu\n", "DHT API", route_handle->uid); #endif message->unique_id = GNUNET_htonll (route_handle->uid); + memcpy(&message->key, &route_handle->key, sizeof(GNUNET_HashCode)); pending = GNUNET_malloc (sizeof (struct PendingMessage)); pending->msg = (struct GNUNET_MessageHeader *) message; pending->timeout = GNUNET_TIME_relative_get_forever(); @@ -1009,6 +1010,7 @@ GNUNET_DHT_route_stop (struct GNUNET_DHT_RouteHandle *route_handle, } else { + GNUNET_free(pending); GNUNET_break(0); } diff --git a/src/dht/gnunet-dht-driver.c b/src/dht/gnunet-dht-driver.c new file mode 100644 index 000000000..2081df970 --- /dev/null +++ b/src/dht/gnunet-dht-driver.c @@ -0,0 +1,1243 @@ +/* + This file is part of GNUnet. + (C) 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file dht/gnunet-dht-driver.c + * @brief Driver for setting up a group of gnunet peers and + * then issuing GETS and PUTS on the DHT. Coarse results + * are reported, fine grained results (if requested) are + * logged to a (mysql) database. + * + * TODO: Add multiple database support; alternatively, dump + * sql readable (or easily transformed) logs to disk + * for reassembly later. This could remove the mysql + * server as a bottleneck during testing. + */ +#include "platform.h" +#include "gnunet_testing_lib.h" +#include "gnunet_core_service.h" +#include "gnunet_dht_service.h" +#include "dhtlog.h" + +/* DEFINES */ +#define VERBOSE GNUNET_YES + +/* Timeout for entire driver to run */ +#define DEFAULT_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5) + +/* Timeout for waiting for (individual) replies to get requests */ +#define DEFAULT_GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90) + +/* Timeout for waiting for gets to be sent to the service */ +#define DEFAULT_GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10) + +/* Timeout for waiting for puts to be sent to the service */ +#define DEFAULT_PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10) + +#define DEFAULT_SECONDS_PER_PEER_START GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 45) + +#define DEFAULT_TEST_DATA_SIZE 8 + +#define DEFAULT_MAX_OUTSTANDING_PUTS 10 + +#define DEFAULT_MAX_OUTSTANDING_GETS 10 + +#define DEFAULT_CONNECT_TIMEOUT 60 + +#define DEFAULT_TOPOLOGY_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 8) + +/* Structs */ + +struct TestPutContext +{ + /* This is a linked list */ + struct TestPutContext *next; + + /** + * Handle to the first peers DHT service (via the API) + */ + struct GNUNET_DHT_Handle *dht_handle; + + /** + * Handle to the PUT peer daemon + */ + struct GNUNET_TESTING_Daemon *daemon; + + /** + * Identifier for this PUT + */ + uint32_t uid; + + /** + * Task for disconnecting DHT handles + */ + GNUNET_SCHEDULER_TaskIdentifier disconnect_task; +}; + +struct TestGetContext +{ + /* This is a linked list */ + struct TestGetContext *next; + + /** + * Handle to the first peers DHT service (via the API) + */ + struct GNUNET_DHT_Handle *dht_handle; + + /** + * Handle for the DHT get request + */ + struct GNUNET_DHT_GetHandle *get_handle; + + /** + * Handle to the GET peer daemon + */ + struct GNUNET_TESTING_Daemon *daemon; + + /** + * Identifier for this GET + */ + uint32_t uid; + + /** + * Task for disconnecting DHT handles (and stopping GET) + */ + GNUNET_SCHEDULER_TaskIdentifier disconnect_task; + + /** + * Whether or not this request has been fulfilled already. + */ + int succeeded; +}; + +/** + * Simple struct to keep track of progress, and print a + * nice little percentage meter for long running tasks. + */ +struct ProgressMeter +{ + unsigned int total; + + unsigned int modnum; + + unsigned int dotnum; + + unsigned int completed; + + int print; + + char *startup_string; +}; + +/* Globals */ + +/** + * Timeout to let all get requests happen. + */ +static struct GNUNET_TIME_Relative all_get_timeout; + +/** + * Per get timeout + */ +static struct GNUNET_TIME_Relative get_timeout; + +static struct GNUNET_TIME_Relative get_delay; + +static struct GNUNET_TIME_Relative put_delay; + +static struct GNUNET_TIME_Relative seconds_per_peer_start; + +static unsigned long long test_data_size = DEFAULT_TEST_DATA_SIZE; + +static unsigned long long max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS; + +static unsigned long long max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS; + +static unsigned long long malicious_getters; + +static unsigned long long malicious_putters; + +static unsigned long long malicious_droppers; + +static unsigned long long settle_time; + +static struct GNUNET_DHTLOG_Handle *dhtlog_handle; + +static unsigned long long trialuid; + +/** + * List of GETS to perform + */ +struct TestGetContext *all_gets; + +/** + * List of PUTS to perform + */ +struct TestPutContext *all_puts; + +/** + * Directory to store temporary data in, defined in config file + */ +static char *test_directory; + +/** + * Variable used to store the number of connections we should wait for. + */ +static unsigned int expected_connections; + +/** + * Variable used to keep track of how many peers aren't yet started. + */ +static unsigned long long peers_left; + +/** + * Handle to the set of all peers run for this test. + */ +static struct GNUNET_TESTING_PeerGroup *pg; + +/** + * Global scheduler, used for all GNUNET_SCHEDULER_* functions. + */ +static struct GNUNET_SCHEDULER_Handle *sched; + +/** + * Total number of peers to run, set based on config file. + */ +static unsigned long long num_peers; + +/** + * Total number of items to insert. + */ +static unsigned long long num_puts; + +/** + * Total number of items to attempt to get. + */ +static unsigned long long num_gets; + +/** + * How many puts do we currently have in flight? + */ +static unsigned long long outstanding_puts; + +/** + * How many puts are done? + */ +static unsigned long long puts_completed; + +/** + * How many puts do we currently have in flight? + */ +static unsigned long long outstanding_gets; + +/** + * How many gets are done? + */ +static unsigned long long gets_completed; + +/** + * How many gets failed? + */ +static unsigned long long gets_failed; + +/** + * Global used to count how many connections we have currently + * been notified about (how many times has topology_callback been called + * with success?) + */ +static unsigned int total_connections; + +/** + * Global used to count how many failed connections we have + * been notified about (how many times has topology_callback + * been called with failure?) + */ +static unsigned int failed_connections; + +/* Task handle to use to schedule shutdown if something goes wrong */ +GNUNET_SCHEDULER_TaskIdentifier die_task; + +static char *blacklist_transports; + +static enum GNUNET_TESTING_Topology topology; + +static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */ + +static enum GNUNET_TESTING_Topology connect_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */ + +static enum GNUNET_TESTING_TopologyOption connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; + +static double connect_topology_option_modifier = 0.0; + +static struct ProgressMeter *hostkey_meter; + +static struct ProgressMeter *peer_start_meter; + +static struct ProgressMeter *peer_connect_meter; + +static struct ProgressMeter *put_meter; + +static struct ProgressMeter *get_meter; + +/* Global return value (0 for success, anything else for failure) */ +static int ok; + +/** + * Create a meter to keep track of the progress of some task. + * + * @param total the total number of items to complete + * @param start_string a string to prefix the meter with (if printing) + * @param print GNUNET_YES to print the meter, GNUNET_NO to count + * internally only + * + * @return the progress meter + */ +static struct ProgressMeter * +create_meter(unsigned int total, char * start_string, int print) +{ + struct ProgressMeter *ret; + ret = GNUNET_malloc(sizeof(struct ProgressMeter)); + ret->print = print; + ret->total = total; + ret->modnum = total / 4; + ret->dotnum = (total / 50) + 1; + if (start_string != NULL) + ret->startup_string = GNUNET_strdup(start_string); + else + ret->startup_string = GNUNET_strdup(""); + + return ret; +} + +/** + * Update progress meter (increment by one). + * + * @param meter the meter to update and print info for + * + * @return GNUNET_YES if called the total requested, + * GNUNET_NO if more items expected + */ +static int +update_meter(struct ProgressMeter *meter) +{ + if (meter->print == GNUNET_YES) + { + if (meter->completed % meter->modnum == 0) + { + if (meter->completed == 0) + { + fprintf(stdout, "%sProgress: [0%%", meter->startup_string); + } + else + fprintf(stdout, "%d%%", (int)(((float)meter->completed / meter->total) * 100)); + } + else if (meter->completed % meter->dotnum == 0) + fprintf(stdout, "."); + + if (meter->completed + 1 == meter->total) + fprintf(stdout, "%d%%]\n", 100); + fflush(stdout); + } + meter->completed++; + + if (meter->completed == meter->total) + return GNUNET_YES; + return GNUNET_NO; +} + +/** + * Release resources for meter + * + * @param meter the meter to free + */ +static void +free_meter(struct ProgressMeter *meter) +{ + GNUNET_free_non_null(meter->startup_string); + GNUNET_free_non_null(meter); +} + +/** + * Check whether peers successfully shut down. + */ +void shutdown_callback (void *cls, + const char *emsg) +{ + if (emsg != NULL) + { + if (ok == 0) + ok = 2; + } +} + +/** + * Task to release DHT handles for PUT + */ +static void +put_disconnect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put->dht_handle = NULL; +} + +/** + * Function scheduled to be run on the successful completion of this + * testcase. + */ +static void +finish_testing (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + GNUNET_assert (pg != NULL); + struct TestPutContext *test_put = all_puts; + struct TestGetContext *test_get = all_gets; + + while (test_put != NULL) + { + if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + if (test_put->dht_handle != NULL) + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put = test_put->next; + } + + while (test_get != NULL) + { + if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + if (test_get->get_handle != NULL) + GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL); + if (test_get->dht_handle != NULL) + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get = test_get->next; + } + + GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL); + + /* FIXME: optionally get stats for dropped messages, etc. */ + if (dhtlog_handle != NULL) + dhtlog_handle->update_trial (trialuid, 0, 0, 0); + + if (hostkey_meter != NULL) + free_meter(hostkey_meter); + if (hostkey_meter != NULL) + free_meter(peer_start_meter); + if (hostkey_meter != NULL) + free_meter(peer_connect_meter); + if (hostkey_meter != NULL) + free_meter(put_meter); + if (hostkey_meter != NULL) + free_meter(get_meter); + + ok = 0; +} + + +/** + * Check if the get_handle is being used, if so stop the request. Either + * way, schedule the end_badly_cont function which actually shuts down the + * test. + */ +static void +end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failing test with error: `%s'!\n", (char *)cls); + + struct TestPutContext *test_put = all_puts; + struct TestGetContext *test_get = all_gets; + + while (test_put != NULL) + { + if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + if (test_put->dht_handle != NULL) + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put = test_put->next; + } + + while (test_get != NULL) + { + if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + if (test_get->get_handle != NULL) + GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL); + if (test_get->dht_handle != NULL) + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get = test_get->next; + } + + GNUNET_TESTING_daemons_stop (pg, DEFAULT_TIMEOUT, &shutdown_callback, NULL); + + if (hostkey_meter != NULL) + free_meter(hostkey_meter); + if (hostkey_meter != NULL) + free_meter(peer_start_meter); + if (hostkey_meter != NULL) + free_meter(peer_connect_meter); + if (hostkey_meter != NULL) + free_meter(put_meter); + if (hostkey_meter != NULL) + free_meter(get_meter); + + ok = 1; +} + +/** + * Task to release DHT handle associated with GET request. + */ +static void +get_stop_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + outstanding_gets--; /* GET is really finished */ + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get->dht_handle = NULL; + +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed); +#endif + update_meter(get_meter); + if ((gets_completed == num_gets) && (outstanding_gets == 0))/* All gets successful */ + { + GNUNET_SCHEDULER_cancel(sched, die_task); + GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL); + } + else if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0)) /* Had some failures */ + { + GNUNET_SCHEDULER_cancel(sched, die_task); + GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL); + } +} + +/** + * Task to release get handle. + */ +static void +get_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + + if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT) + gets_failed++; + GNUNET_assert(test_get->get_handle != NULL); + GNUNET_DHT_get_stop(test_get->get_handle, &get_stop_finished, test_get); + test_get->get_handle = NULL; + test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK; +} + +/** + * Iterator called if the GET request initiated returns a response. + * + * @param cls closure + * @param exp when will this value expire + * @param key key of the result + * @param type type of the result + * @param size number of bytes in data + * @param data pointer to the result data + */ +void get_result_iterator (void *cls, + struct GNUNET_TIME_Absolute exp, + const GNUNET_HashCode * key, + uint32_t type, + uint32_t size, + const void *data) +{ + struct TestGetContext *test_get = cls; + GNUNET_HashCode search_key; /* Key stored under */ + char original_data[test_data_size]; /* Made up data to store */ + + memset(original_data, test_get->uid, sizeof(original_data)); + GNUNET_CRYPTO_hash(original_data, test_data_size, &search_key); + + if (test_get->succeeded == GNUNET_YES) + return; /* Get has already been successful, probably ending now */ + + if ((0 != memcmp(&search_key, key, sizeof (GNUNET_HashCode))) || (0 != memcmp(original_data, data, sizeof(original_data)))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Key or data is not the same as was inserted!\n"); + } + else + { + gets_completed++; + test_get->succeeded = GNUNET_YES; + } +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n"); +#endif + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + GNUNET_SCHEDULER_add_continuation(sched, &get_stop_task, test_get, GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + +/** + * Continuation telling us GET request was sent. + */ +static void +get_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + // Is there something to be done here? + if (tc->reason != GNUNET_SCHEDULER_REASON_PREREQ_DONE) + return; +} + +/** + * Set up some data, and call API PUT function + */ +static void +do_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + GNUNET_HashCode key; /* Made up key to store data under */ + char data[test_data_size]; /* Made up data to store */ + + if (num_gets == 0) + { + GNUNET_SCHEDULER_cancel(sched, die_task); + GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL); + } + if (test_get == NULL) + return; /* End of the list */ + memset(data, test_get->uid, sizeof(data)); + GNUNET_CRYPTO_hash(data, test_data_size, &key); + + if (outstanding_gets > max_outstanding_gets) + { + GNUNET_SCHEDULER_add_delayed (sched, get_delay, &do_get, test_get); + return; + } + + test_get->dht_handle = GNUNET_DHT_connect(sched, test_get->daemon->cfg, 10); + /* Insert the data at the first peer */ + GNUNET_assert(test_get->dht_handle != NULL); + outstanding_gets++; + test_get->get_handle = GNUNET_DHT_get_start(test_get->dht_handle, + GNUNET_TIME_relative_get_forever(), + 1, + &key, + &get_result_iterator, + test_get, + &get_continuation, + test_get); +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n", + test_get->uid, + test_get->daemon->shortname); +#endif + test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, get_timeout, &get_stop_task, test_get); + GNUNET_SCHEDULER_add_now (sched, &do_get, test_get->next); +} + +/** + * Called when the PUT request has been transmitted to the DHT service. + * Schedule the GET request for some time in the future. + */ +static void +put_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + outstanding_puts--; + puts_completed++; + + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + test_put->disconnect_task = GNUNET_SCHEDULER_add_now(sched, &put_disconnect_task, test_put); + if (GNUNET_YES == update_meter(put_meter)) + { + GNUNET_assert(outstanding_puts == 0); + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, all_get_timeout, + &end_badly, "from do gets"); + GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 100), &do_get, all_gets); + return; + } +} + +/** + * Set up some data, and call API PUT function + */ +static void +do_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + GNUNET_HashCode key; /* Made up key to store data under */ + char data[test_data_size]; /* Made up data to store */ + uint32_t rand; + + if (test_put == NULL) + return; /* End of list */ + + memset(data, test_put->uid, sizeof(data)); + GNUNET_CRYPTO_hash(data, test_data_size, &key); + + if (outstanding_puts > max_outstanding_puts) + { + GNUNET_SCHEDULER_add_delayed (sched, put_delay, &do_put, test_put); + return; + } + +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n", + test_put->uid, + test_put->daemon->shortname); +#endif + test_put->dht_handle = GNUNET_DHT_connect(sched, test_put->daemon->cfg, 10); + + GNUNET_assert(test_put->dht_handle != NULL); + outstanding_puts++; + GNUNET_DHT_put(test_put->dht_handle, + &key, + 1, + sizeof(data), data, + GNUNET_TIME_absolute_get_forever(), + GNUNET_TIME_relative_get_forever(), + &put_finished, test_put); + test_put->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_get_forever(), &put_disconnect_task, test_put); + rand = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, 2); + GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, rand), &do_put, test_put->next); +} + + +/** + * Set up some all of the put and get operations we want + * to do. Allocate data structure for each, add to list, + * then call actual insert functions. + */ +static void +setup_puts_and_gets (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + int i; + uint32_t temp_daemon; + struct TestPutContext *test_put; + struct TestGetContext *test_get; + int remember[num_puts][num_peers]; + + for (i = 0; i < num_puts; i++) + { + test_put = GNUNET_malloc(sizeof(struct TestPutContext)); + test_put->uid = i; + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + test_put->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon); + test_put->next = all_puts; + all_puts = test_put; + } + + for (i = 0; i < num_gets; i++) + { + test_get = GNUNET_malloc(sizeof(struct TestGetContext)); + test_get->uid = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_puts); + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + while (remember[test_get->uid][temp_daemon] == 1) + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + test_get->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon); + remember[test_get->uid][temp_daemon] = 1; + test_get->next = all_gets; + all_gets = test_get; + } + + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, num_puts * 2), + &end_badly, "from do puts"); + GNUNET_SCHEDULER_add_now (sched, &do_put, all_puts); +} +/** + * This function is called whenever a connection attempt is finished between two of + * the started peers (started with GNUNET_TESTING_daemons_start). The total + * number of times this function is called should equal the number returned + * from the GNUNET_TESTING_connect_topology call. + * + * The emsg variable is NULL on success (peers connected), and non-NULL on + * failure (peers failed to connect). + */ +void +topology_callback (void *cls, + const struct GNUNET_PeerIdentity *first, + const struct GNUNET_PeerIdentity *second, + uint32_t distance, + const struct GNUNET_CONFIGURATION_Handle *first_cfg, + const struct GNUNET_CONFIGURATION_Handle *second_cfg, + struct GNUNET_TESTING_Daemon *first_daemon, + struct GNUNET_TESTING_Daemon *second_daemon, + const char *emsg) +{ + if (emsg == NULL) + { + total_connections++; +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n", + first_daemon->shortname, + second_daemon->shortname, + distance); +#endif + } +#if VERBOSE + else + { + failed_connections++; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n", + first_daemon->shortname, + second_daemon->shortname, emsg); + } +#endif + GNUNET_assert(peer_connect_meter != NULL); + if (GNUNET_YES == update_meter(peer_connect_meter)) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Created %d total connections, which is our target number! Starting next phase of testing.\n", + total_connections); +#endif + if (dhtlog_handle != NULL) + dhtlog_handle->update_connections (trialuid, total_connections); + + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, DEFAULT_TIMEOUT, + &end_badly, "from setup puts/gets"); + + GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, settle_time), &setup_puts_and_gets, NULL); + } + else if (total_connections + failed_connections == expected_connections) + { + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from topology_callback (too many failed connections)"); + } +} + +static void +peers_started_callback (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Daemon *d, const char *emsg) +{ + if (emsg != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to start daemon with error: `%s'\n", + emsg); + return; + } + GNUNET_assert (id != NULL); + +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n", + (num_peers - peers_left) + 1, num_peers); +#endif + + peers_left--; + + if (GNUNET_YES == update_meter(peer_start_meter)) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "All %d daemons started, now connecting peers!\n", + num_peers); +#endif + GNUNET_SCHEDULER_cancel (sched, die_task); + + expected_connections = -1; + if ((pg != NULL) && (peers_left == 0)) + { + expected_connections = GNUNET_TESTING_connect_topology (pg, connect_topology, connect_topology_option, connect_topology_option_modifier); + peer_connect_meter = create_meter(expected_connections, "Peer connection ", GNUNET_YES); +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Have %d expected connections\n", expected_connections); +#endif + } + + if (expected_connections == GNUNET_SYSERR) + { + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from connect topology (bad return)"); + } + + die_task = GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, DEFAULT_CONNECT_TIMEOUT * expected_connections), + &end_badly, "from connect topology (timeout)"); + + ok = 0; + } +} + +static void +create_topology () +{ + peers_left = num_peers; /* Reset counter */ + if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology, blacklist_transports) != GNUNET_SYSERR) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Topology set up, now starting peers!\n"); +#endif + GNUNET_TESTING_daemons_continue_startup(pg); + } + else + { + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from create topology (bad return)"); + } + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_relative_multiply(seconds_per_peer_start, num_peers), + &end_badly, "from continue startup (timeout)"); +} + +/** + * Callback indicating that the hostkey was created for a peer. + * + * @param cls NULL + * @param id the peer identity + * @param d the daemon handle (pretty useless at this point, remove?) + * @param emsg non-null on failure + */ +void hostkey_callback (void *cls, + const struct GNUNET_PeerIdentity *id, + struct GNUNET_TESTING_Daemon *d, + const char *emsg) +{ + if (emsg != NULL) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg); + } + +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Hostkey (%d/%d) created for peer `%s'\n", + num_peers - peers_left, num_peers, GNUNET_i2s(id)); +#endif + + peers_left--; + if (GNUNET_YES == update_meter(hostkey_meter)) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "All %d hostkeys created, now creating topology!\n", + num_peers); +#endif + GNUNET_SCHEDULER_cancel (sched, die_task); + /* Set up task in case topology creation doesn't finish + * within a reasonable amount of time */ + die_task = GNUNET_SCHEDULER_add_delayed (sched, + DEFAULT_TOPOLOGY_TIMEOUT, + &end_badly, "from create_topology"); + GNUNET_SCHEDULER_add_now(sched, &create_topology, NULL); + ok = 0; + } +} + + +static void +run (void *cls, + struct GNUNET_SCHEDULER_Handle *s, + char *const *args, + const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + char * topology_str; + char * connect_topology_str; + char * blacklist_topology_str; + char * connect_topology_option_str; + char * connect_topology_option_modifier_string; + char *trialmessage; + char * topology_percentage_str; + float topology_percentage; + char * topology_probability_str; + float topology_probability; + unsigned long long temp_config_number; + + sched = s; + + /* Get path from configuration file */ + if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string(cfg, "paths", "servicehome", &test_directory)) + { + ok = 404; + return; + } + + /** + * Get DHT specific testing options. + */ + if ((GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(cfg, "dht_testing", "mysql_logging"))|| + (GNUNET_YES == GNUNET_CONFIGURATION_get_value_yesno(cfg, "dht_testing", "mysql_logging_extended"))) + { + dhtlog_handle = GNUNET_DHTLOG_connect(cfg); + if (dhtlog_handle == NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Could not connect to mysql server for logging, will NOT log dht operations!"); + ok = 3306; + return; + } + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "dht_testing", "comment", + &trialmessage)) + trialmessage = NULL; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "malicious_getters", + &malicious_getters)) + malicious_getters = 0; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "malicious_putters", + &malicious_putters)) + malicious_putters = 0; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "malicious_droppers", + &malicious_droppers)) + malicious_droppers = 0; + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "settle_time", + &settle_time)) + settle_time = 0; + + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_puts", + &num_puts)) + num_puts = num_peers; + + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_gets", + &num_gets)) + num_gets = num_peers; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "get_timeout", + &temp_config_number)) + get_timeout = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_config_number); + else + get_timeout = DEFAULT_GET_TIMEOUT; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "concurrent_puts", + &temp_config_number)) + max_outstanding_puts = temp_config_number; + else + max_outstanding_puts = DEFAULT_MAX_OUTSTANDING_PUTS; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "concurrent_gets", + &temp_config_number)) + max_outstanding_gets = temp_config_number; + else + max_outstanding_gets = DEFAULT_MAX_OUTSTANDING_GETS; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "timeout", + &temp_config_number)) + all_get_timeout = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_config_number); + else + all_get_timeout.value = get_timeout.value * ((num_gets / max_outstanding_gets) + 1); + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "get_delay", + &temp_config_number)) + get_delay = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_config_number); + else + get_delay = DEFAULT_GET_DELAY; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "put_delay", + &temp_config_number)) + put_delay = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_config_number); + else + put_delay = DEFAULT_PUT_DELAY; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "peer_start_timeout", + &temp_config_number)) + seconds_per_peer_start = GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, temp_config_number); + else + seconds_per_peer_start = DEFAULT_SECONDS_PER_PEER_START; + + if (GNUNET_OK == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "data_size", + &temp_config_number)) + test_data_size = temp_config_number; + else + test_data_size = DEFAULT_TEST_DATA_SIZE; + + /** + * Get testing related options. + */ + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "topology", + &topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&topology, topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "TOPOLOGY"); + topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */ + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "percentage", + &topology_percentage_str)) + topology_percentage = 0.5; + else + { + topology_percentage = atof (topology_percentage_str); + } + + if (GNUNET_OK != + GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "probability", + &topology_probability_str)) + topology_probability = 0.5; + else + { + topology_probability = atof (topology_probability_str); + } + + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology", + &connect_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&connect_topology, connect_topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid connect topology `%s' given for section %s option %s\n", connect_topology_str, "TESTING", "CONNECT_TOPOLOGY"); + } + GNUNET_free_non_null(connect_topology_str); + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology_option", + &connect_topology_option_str)) && (GNUNET_NO == GNUNET_TESTING_topology_option_get(&connect_topology_option, connect_topology_option_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid connect topology option `%s' given for section %s option %s\n", connect_topology_option_str, "TESTING", "CONNECT_TOPOLOGY_OPTION"); + connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */ + } + GNUNET_free_non_null(connect_topology_option_str); + if (GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "connect_topology_option_modifier", + &connect_topology_option_modifier_string)) + { + if (sscanf(connect_topology_option_modifier_string, "%lf", &connect_topology_option_modifier) != 1) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Invalid value `%s' for option `%s' in section `%s': expected float\n"), + connect_topology_option_modifier_string, + "connect_topology_option_modifier", + "TESTING"); + } + GNUNET_free (connect_topology_option_modifier_string); + } + + if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "blacklist_transports", + &blacklist_transports)) + blacklist_transports = NULL; + + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "blacklist_topology", + &blacklist_topology_str)) && + (GNUNET_NO == GNUNET_TESTING_topology_get(&blacklist_topology, blacklist_topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "BLACKLIST_TOPOLOGY"); + } + GNUNET_free_non_null(topology_str); + GNUNET_free_non_null(blacklist_topology_str); + + /* Get number of peers to start from configuration */ + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers", + &num_peers)) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Number of peers must be specified in section %s option %s\n", topology_str, "TESTING", "NUM_PEERS"); + } + + /* Set peers_left so we know when all peers started */ + peers_left = num_peers; + + /* Set up a task to end testing if peer start fails */ + die_task = GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_relative_multiply(seconds_per_peer_start, num_peers), + &end_badly, "didn't generate all hostkeys within allowed startup time!"); + + if (dhtlog_handle == NULL) + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "dhtlog_handle is NULL!"); + + if ((trialmessage != NULL) && (dhtlog_handle != NULL)) + { + dhtlog_handle->insert_trial (&trialuid, peers_left, topology, + blacklist_topology, connect_topology, + connect_topology_option, + connect_topology_option_modifier, topology_percentage, + topology_probability, num_puts, num_gets, + max_outstanding_gets, settle_time, 1, + malicious_getters, malicious_putters, + malicious_droppers, trialmessage); + } + else if (dhtlog_handle != NULL) + { + dhtlog_handle->insert_trial (&trialuid, peers_left, topology, + blacklist_topology, connect_topology, + connect_topology_option, + connect_topology_option_modifier, topology_percentage, + topology_probability, num_puts, num_gets, + max_outstanding_gets, settle_time, 1, + malicious_getters, malicious_putters, + malicious_droppers, ""); + } + + hostkey_meter = create_meter(peers_left, "Hostkeys created ", GNUNET_YES); + peer_start_meter = create_meter(peers_left, "Peers started ", GNUNET_YES); + + put_meter = create_meter(num_gets, "Puts completed ", GNUNET_YES); + get_meter = create_meter(num_gets, "Gets completed ", GNUNET_YES); + pg = GNUNET_TESTING_daemons_start (sched, cfg, + peers_left, + GNUNET_TIME_relative_multiply(seconds_per_peer_start, num_peers), + &hostkey_callback, NULL, + &peers_started_callback, NULL, + &topology_callback, NULL, + NULL); + +} + + +int +main (int argc, char *argv[]) +{ + int ret; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + + ret = GNUNET_PROGRAM_run (argc, + argv, "gnunet-dht-driver", "nohelp", + options, &run, &ok); + + if (ret != GNUNET_OK) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "`gnunet-dht-driver': Failed with error code %d\n", ret); + } + + /** + * Need to remove base directory, subdirectories taken care + * of by the testing framework. + */ + if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to remove testing directory %s\n", test_directory); + } + return ret; +} + +/* end of test_dht_twopeer_put_get.c */ diff --git a/src/dht/gnunet-service-dht.c b/src/dht/gnunet-service-dht.c index 32c01e384..eca3131b0 100644 --- a/src/dht/gnunet-service-dht.c +++ b/src/dht/gnunet-service-dht.c @@ -313,7 +313,7 @@ send_reply_to_client (struct ClientList *client, pending_message = GNUNET_malloc (sizeof (struct PendingMessage) + tsize); pending_message->msg = (struct GNUNET_MessageHeader *)&pending_message[1]; reply = (struct GNUNET_DHT_RouteResultMessage *)&pending_message[1]; - reply->header.type = htons (GNUNET_MESSAGE_TYPE_DHT_ROUTE_RESULT); + reply->header.type = htons (GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE_RESULT); reply->header.size = htons (tsize); reply->unique_id = GNUNET_htonll (uid); memcpy (&reply[1], message, msize); @@ -353,7 +353,6 @@ datacache_get_iterator (void *cls, get_result->header.size = htons (sizeof (struct GNUNET_DHT_GetResultMessage) + size); get_result->expiration = exp; - memcpy (&get_result->key, key, sizeof (GNUNET_HashCode)); get_result->type = htons (type); memcpy (&get_result[1], data, size); send_reply_to_client (datacache_get_ctx->client, &get_result->header, @@ -573,7 +572,7 @@ handle_dht_start_message (void *cls, struct GNUNET_SERVER_Client *client, * be processed everywhere _AND_ we want it processed everywhere, then * handle it locally. */ - handle_locally = GNUNET_YES; + handle_locally = GNUNET_NO; if (handle_locally == GNUNET_YES) { switch (enc_type) @@ -596,6 +595,9 @@ handle_dht_start_message (void *cls, struct GNUNET_SERVER_Client *client, "`%s': Message type (%d) not handled\n", "DHT", enc_type); } } + else + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "`%s': Message type (%d) not handled\n", "DHT", enc_type); GNUNET_free (message_context); GNUNET_SERVER_receive_done (client, GNUNET_OK); @@ -759,15 +761,15 @@ core_init (void *cls, static struct GNUNET_SERVER_MessageHandler plugin_handlers[] = { - {&handle_dht_start_message, NULL, GNUNET_MESSAGE_TYPE_DHT_ROUTE, 0}, - {&handle_dht_stop_message, NULL, GNUNET_MESSAGE_TYPE_DHT_STOP, 0}, + {&handle_dht_start_message, NULL, GNUNET_MESSAGE_TYPE_LOCAL_DHT_ROUTE, 0}, + {&handle_dht_stop_message, NULL, GNUNET_MESSAGE_TYPE_DHT_ROUTE_STOP, 0}, {NULL, NULL, 0, 0} }; static struct GNUNET_CORE_MessageHandler core_handlers[] = { - {&handle_dht_p2p_route_request, GNUNET_MESSAGE_TYPE_DHT_ROUTE, 0}, - {&handle_dht_p2p_route_result, GNUNET_MESSAGE_TYPE_DHT_ROUTE_RESULT, 0}, + {&handle_dht_p2p_route_request, GNUNET_MESSAGE_TYPE_P2P_DHT_ROUTE, 0}, + {&handle_dht_p2p_route_result, GNUNET_MESSAGE_TYPE_P2P_DHT_ROUTE_RESULT, 0}, {NULL, 0, 0} }; diff --git a/src/dht/test_dht_api_data.conf b/src/dht/test_dht_api_data.conf index 047b2ea3d..0dcd95691 100644 --- a/src/dht/test_dht_api_data.conf +++ b/src/dht/test_dht_api_data.conf @@ -58,10 +58,10 @@ HOSTNAME = localhost PORT = 2092 [dht] -DEBUG = NO +DEBUG = YES ACCEPT_FROM6 = ::1; ACCEPT_FROM = 127.0.0.1; -BINARY = gnunet-service-dht +BINARY = gnunet-service-dht-new CONFIG = $DEFAULTCONFIG HOME = $SERVICEHOME HOSTNAME = localhost diff --git a/src/dht/test_dht_multipeer.c b/src/dht/test_dht_multipeer.c new file mode 100644 index 000000000..3762bdaf9 --- /dev/null +++ b/src/dht/test_dht_multipeer.c @@ -0,0 +1,915 @@ +/* + This file is part of GNUnet. + (C) 2009 Christian Grothoff (and other contributing authors) + + GNUnet is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published + by the Free Software Foundation; either version 3, or (at your + option) any later version. + + GNUnet is distributed in the hope that it will be useful, but + WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + General Public License for more details. + + You should have received a copy of the GNU General Public License + along with GNUnet; see the file COPYING. If not, write to the + Free Software Foundation, Inc., 59 Temple Place - Suite 330, + Boston, MA 02111-1307, USA. +*/ +/** + * @file dht/test_dht_multipeer.c + * @brief testcase for testing DHT service with + * multiple peers. + */ +#include "platform.h" +#include "gnunet_testing_lib.h" +#include "gnunet_core_service.h" +#include "gnunet_dht_service.h" + +/* DEFINES */ +#define VERBOSE GNUNET_YES + +/* Timeout for entire testcase */ +#define TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_MINUTES, 5) + +/* Timeout for waiting for replies to get requests */ +#define GET_TIMEOUT GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 90) + +/* Timeout for waiting for gets to complete */ +#define GET_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1) + +/* Timeout for waiting for puts to complete */ +#define PUT_DELAY GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 1) + +#define SECONDS_PER_PEER_START 45 + +/* If number of peers not in config file, use this number */ +#define DEFAULT_NUM_PEERS 5 + +#define TEST_DATA_SIZE 8 + +#define MAX_OUTSTANDING_PUTS 10 + +#define MAX_OUTSTANDING_GETS 10 + +/* Structs */ + +struct TestPutContext +{ + /* This is a linked list */ + struct TestPutContext *next; + + /** + * Handle to the first peers DHT service (via the API) + */ + struct GNUNET_DHT_Handle *dht_handle; + + /** + * Handle to the PUT peer daemon + */ + struct GNUNET_TESTING_Daemon *daemon; + + /** + * Identifier for this PUT + */ + uint32_t uid; + + /** + * Task for disconnecting DHT handles + */ + GNUNET_SCHEDULER_TaskIdentifier disconnect_task; +}; + +struct TestGetContext +{ + /* This is a linked list */ + struct TestGetContext *next; + + /** + * Handle to the first peers DHT service (via the API) + */ + struct GNUNET_DHT_Handle *dht_handle; + + /** + * Handle for the DHT get request + */ + struct GNUNET_DHT_GetHandle *get_handle; + + /** + * Handle to the GET peer daemon + */ + struct GNUNET_TESTING_Daemon *daemon; + + /** + * Identifier for this GET + */ + uint32_t uid; + + /** + * Task for disconnecting DHT handles (and stopping GET) + */ + GNUNET_SCHEDULER_TaskIdentifier disconnect_task; + + /** + * Whether or not this request has been fulfilled already. + */ + int succeeded; +}; + +/* Globals */ + +/** + * List of GETS to perform + */ +struct TestGetContext *all_gets; + +/** + * List of PUTS to perform + */ +struct TestPutContext *all_puts; + +/** + * Directory to store temp data in, defined in config file + */ +static char *test_directory; + +/** + * Variable used to store the number of connections we should wait for. + */ +static unsigned int expected_connections; + +/** + * Variable used to keep track of how many peers aren't yet started. + */ +static unsigned long long peers_left; + +/** + * Handle to the set of all peers run for this test. + */ +static struct GNUNET_TESTING_PeerGroup *pg; + + +/** + * Global scheduler, used for all GNUNET_SCHEDULER_* functions. + */ +static struct GNUNET_SCHEDULER_Handle *sched; + +/** + * Total number of peers to run, set based on config file. + */ +static unsigned long long num_peers; + +/** + * Total number of items to insert. + */ +static unsigned long long num_puts; + +/** + * Total number of items to attempt to get. + */ +static unsigned long long num_gets; + +/** + * How many puts do we currently have in flight? + */ +static unsigned long long outstanding_puts; + +/** + * How many puts are done? + */ +static unsigned long long puts_completed; + +/** + * How many puts do we currently have in flight? + */ +static unsigned long long outstanding_gets; + +/** + * How many gets are done? + */ +static unsigned long long gets_completed; + +/** + * How many gets failed? + */ +static unsigned long long gets_failed; + +/** + * Global used to count how many connections we have currently + * been notified about (how many times has topology_callback been called + * with success?) + */ +static unsigned int total_connections; + +/** + * Global used to count how many failed connections we have + * been notified about (how many times has topology_callback + * been called with failure?) + */ +static unsigned int failed_connections; + +/* Task handle to use to schedule test failure */ +GNUNET_SCHEDULER_TaskIdentifier die_task; + +static char *blacklist_transports; + +static enum GNUNET_TESTING_Topology topology; + +static enum GNUNET_TESTING_Topology blacklist_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* Don't do any blacklisting */ + +static enum GNUNET_TESTING_Topology connection_topology = GNUNET_TESTING_TOPOLOGY_NONE; /* NONE actually means connect all allowed peers */ + +static enum GNUNET_TESTING_TopologyOption connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; + +static double connect_topology_option_modifier = 0.0; + +/* Global return value (0 for success, anything else for failure) */ +static int ok; + + +/** + * Check whether peers successfully shut down. + */ +void shutdown_callback (void *cls, + const char *emsg) +{ + if (emsg != NULL) + { + if (ok == 0) + ok = 2; + } +} + +/** + * Task to release DHT handles for PUT + */ +static void +put_disconnect_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + test_put->disconnect_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put->dht_handle = NULL; +} + +/** + * Function scheduled to be run on the successful completion of this + * testcase. + */ +static void +finish_testing (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + GNUNET_assert (pg != NULL); + struct TestPutContext *test_put = all_puts; + struct TestGetContext *test_get = all_gets; + + while (test_put != NULL) + { + if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + if (test_put->dht_handle != NULL) + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put = test_put->next; + } + + while (test_get != NULL) + { + if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + if (test_get->get_handle != NULL) + GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL); + if (test_get->dht_handle != NULL) + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get = test_get->next; + } + + GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL); + ok = 0; +} + + +/** + * Check if the get_handle is being used, if so stop the request. Either + * way, schedule the end_badly_cont function which actually shuts down the + * test. + */ +static void +end_badly (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failing test with error: `%s'!\n", (char *)cls); + + struct TestPutContext *test_put = all_puts; + struct TestGetContext *test_get = all_gets; + + while (test_put != NULL) + { + if (test_put->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + if (test_put->dht_handle != NULL) + GNUNET_DHT_disconnect(test_put->dht_handle); + test_put = test_put->next; + } + + while (test_get != NULL) + { + if (test_get->disconnect_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + if (test_get->get_handle != NULL) + GNUNET_DHT_get_stop(test_get->get_handle, NULL, NULL); + if (test_get->dht_handle != NULL) + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get = test_get->next; + } + + GNUNET_TESTING_daemons_stop (pg, TIMEOUT, &shutdown_callback, NULL); + ok = 1; +} + +/** + * Task to release DHT handle associated with GET request. + */ +static void +get_stop_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + outstanding_gets--; /* GET is really finished */ + GNUNET_DHT_disconnect(test_get->dht_handle); + test_get->dht_handle = NULL; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "%d gets succeeded, %d gets failed!\n", gets_completed, gets_failed); + if ((gets_completed == num_gets) && (outstanding_gets == 0))/* All gets successful */ + { + GNUNET_SCHEDULER_cancel(sched, die_task); + GNUNET_SCHEDULER_add_now(sched, &finish_testing, NULL); + } + else if ((gets_completed + gets_failed == num_gets) && (outstanding_gets == 0)) /* Had some failures */ + { + GNUNET_SCHEDULER_cancel(sched, die_task); + GNUNET_SCHEDULER_add_now(sched, &end_badly, "not all gets succeeded!\n"); + } +} + +/** + * Task to release get handle. + */ +static void +get_stop_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + GNUNET_HashCode search_key; /* Key stored under */ + char original_data[TEST_DATA_SIZE]; /* Made up data to store */ + + memset(original_data, test_get->uid, sizeof(original_data)); + GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key); + + if (tc->reason == GNUNET_SCHEDULER_REASON_TIMEOUT) + { + gets_failed++; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Get from peer %s for key %s failed!\n", test_get->daemon->shortname, GNUNET_h2s(&search_key)); + } + GNUNET_assert(test_get->get_handle != NULL); + GNUNET_DHT_get_stop(test_get->get_handle, &get_stop_finished, test_get); + test_get->get_handle = NULL; + test_get->disconnect_task = GNUNET_SCHEDULER_NO_TASK; +} + +/** + * Iterator called if the GET request initiated returns a response. + * + * @param cls closure + * @param exp when will this value expire + * @param key key of the result + * @param type type of the result + * @param size number of bytes in data + * @param data pointer to the result data + */ +void get_result_iterator (void *cls, + struct GNUNET_TIME_Absolute exp, + const GNUNET_HashCode * key, + uint32_t type, + uint32_t size, + const void *data) +{ + struct TestGetContext *test_get = cls; + GNUNET_HashCode search_key; /* Key stored under */ + char original_data[TEST_DATA_SIZE]; /* Made up data to store */ + + memset(original_data, test_get->uid, sizeof(original_data)); + GNUNET_CRYPTO_hash(original_data, TEST_DATA_SIZE, &search_key); + + if (test_get->succeeded == GNUNET_YES) + return; /* Get has already been successful, probably ending now */ + + if ((0 != memcmp(&search_key, key, sizeof (GNUNET_HashCode))) || (0 != memcmp(original_data, data, sizeof(original_data)))) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Key or data is not the same as was inserted!\n"); + } + else + { + gets_completed++; + test_get->succeeded = GNUNET_YES; + } + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Received correct GET response!\n"); + GNUNET_SCHEDULER_cancel(sched, test_get->disconnect_task); + GNUNET_SCHEDULER_add_continuation(sched, &get_stop_task, test_get, GNUNET_SCHEDULER_REASON_PREREQ_DONE); +} + +/** + * Continuation telling us GET request was sent. + */ +static void +get_continuation (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + // Is there something to be done here? + if (tc->reason != GNUNET_SCHEDULER_REASON_PREREQ_DONE) + return; +} + +/** + * Set up some data, and call API PUT function + */ +static void +do_get (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestGetContext *test_get = cls; + GNUNET_HashCode key; /* Made up key to store data under */ + char data[TEST_DATA_SIZE]; /* Made up data to store */ + + if (test_get == NULL) + return; /* End of the list */ + memset(data, test_get->uid, sizeof(data)); + GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key); + + if (outstanding_gets > MAX_OUTSTANDING_GETS) + { + GNUNET_SCHEDULER_add_delayed (sched, GET_DELAY, &do_get, test_get); + return; + } + + test_get->dht_handle = GNUNET_DHT_connect(sched, test_get->daemon->cfg, 10); + /* Insert the data at the first peer */ + GNUNET_assert(test_get->dht_handle != NULL); + outstanding_gets++; + test_get->get_handle = GNUNET_DHT_get_start(test_get->dht_handle, + GNUNET_TIME_relative_get_forever(), + 1, + &key, + &get_result_iterator, + test_get, + &get_continuation, + test_get); +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting get for uid %u from peer %s\n", + test_get->uid, + test_get->daemon->shortname); +#endif + test_get->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GET_TIMEOUT, &get_stop_task, test_get); + GNUNET_SCHEDULER_add_now (sched, &do_get, test_get->next); +} + +/** + * Called when the PUT request has been transmitted to the DHT service. + * Schedule the GET request for some time in the future. + */ +static void +put_finished (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + outstanding_puts--; + puts_completed++; + + GNUNET_SCHEDULER_cancel(sched, test_put->disconnect_task); + test_put->disconnect_task = GNUNET_SCHEDULER_add_now(sched, &put_disconnect_task, test_put); + if (puts_completed == num_puts) + { + GNUNET_assert(outstanding_puts == 0); + GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 10), &do_get, all_gets); + return; + } +} + +/** + * Set up some data, and call API PUT function + */ +static void +do_put (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + struct TestPutContext *test_put = cls; + GNUNET_HashCode key; /* Made up key to store data under */ + char data[TEST_DATA_SIZE]; /* Made up data to store */ + + if (test_put == NULL) + return; /* End of list */ + + memset(data, test_put->uid, sizeof(data)); + GNUNET_CRYPTO_hash(data, TEST_DATA_SIZE, &key); + + if (outstanding_puts > MAX_OUTSTANDING_PUTS) + { + GNUNET_SCHEDULER_add_delayed (sched, PUT_DELAY, &do_put, test_put); + return; + } + +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Starting put for uid %u from peer %s\n", + test_put->uid, + test_put->daemon->shortname); +#endif + test_put->dht_handle = GNUNET_DHT_connect(sched, test_put->daemon->cfg, 10); + + GNUNET_assert(test_put->dht_handle != NULL); + outstanding_puts++; + GNUNET_DHT_put(test_put->dht_handle, + &key, + 1, + sizeof(data), data, + GNUNET_TIME_absolute_get_forever(), + GNUNET_TIME_relative_get_forever(), + &put_finished, test_put); + test_put->disconnect_task = GNUNET_SCHEDULER_add_delayed(sched, GNUNET_TIME_relative_get_forever(), &put_disconnect_task, test_put); + GNUNET_SCHEDULER_add_now(sched, &do_put, test_put->next); +} + + +/** + * Set up some all of the put and get operations we want + * to do. Allocate data structure for each, add to list, + * then call actual insert functions. + */ +static void +setup_puts_and_gets (void *cls, const struct GNUNET_SCHEDULER_TaskContext * tc) +{ + int i; + uint32_t temp_daemon; + struct TestPutContext *test_put; + struct TestGetContext *test_get; + int remember[num_puts][num_peers]; + + for (i = 0; i < num_puts; i++) + { + test_put = GNUNET_malloc(sizeof(struct TestPutContext)); + test_put->uid = i; + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + test_put->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon); + test_put->next = all_puts; + all_puts = test_put; + } + + for (i = 0; i < num_gets; i++) + { + test_get = GNUNET_malloc(sizeof(struct TestGetContext)); + test_get->uid = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_puts); + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + while (remember[test_get->uid][temp_daemon] == 1) + temp_daemon = GNUNET_CRYPTO_random_u32(GNUNET_CRYPTO_QUALITY_WEAK, num_peers); + test_get->daemon = GNUNET_TESTING_daemon_get(pg, temp_daemon); + remember[test_get->uid][temp_daemon] = 1; + test_get->next = all_gets; + all_gets = test_get; + } + + GNUNET_SCHEDULER_add_now (sched, &do_put, all_puts); +} +/** + * This function is called whenever a connection attempt is finished between two of + * the started peers (started with GNUNET_TESTING_daemons_start). The total + * number of times this function is called should equal the number returned + * from the GNUNET_TESTING_connect_topology call. + * + * The emsg variable is NULL on success (peers connected), and non-NULL on + * failure (peers failed to connect). + */ +void +topology_callback (void *cls, + const struct GNUNET_PeerIdentity *first, + const struct GNUNET_PeerIdentity *second, + uint32_t distance, + const struct GNUNET_CONFIGURATION_Handle *first_cfg, + const struct GNUNET_CONFIGURATION_Handle *second_cfg, + struct GNUNET_TESTING_Daemon *first_daemon, + struct GNUNET_TESTING_Daemon *second_daemon, + const char *emsg) +{ + if (emsg == NULL) + { + total_connections++; +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "connected peer %s to peer %s, distance %u\n", + first_daemon->shortname, + second_daemon->shortname, + distance); +#endif + } +#if VERBOSE + else + { + failed_connections++; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to connect peer %s to peer %s with error :\n%s\n", + first_daemon->shortname, + second_daemon->shortname, emsg); + } +#endif + + if (total_connections == expected_connections) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Created %d total connections, which is our target number! Starting next phase of testing.\n", + total_connections); +#endif + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, TIMEOUT, + &end_badly, "from setup puts/gets"); + + GNUNET_SCHEDULER_add_delayed (sched, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, 2), &setup_puts_and_gets, NULL); + } + else if (total_connections + failed_connections == expected_connections) + { + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from topology_callback (too many failed connections)"); + } +} + +static void +peers_started_callback (void *cls, + const struct GNUNET_PeerIdentity *id, + const struct GNUNET_CONFIGURATION_Handle *cfg, + struct GNUNET_TESTING_Daemon *d, const char *emsg) +{ + if (emsg != NULL) + { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Failed to start daemon with error: `%s'\n", + emsg); + return; + } + GNUNET_assert (id != NULL); + +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Started daemon %llu out of %llu\n", + (num_peers - peers_left) + 1, num_peers); +#endif + + peers_left--; + if (peers_left == 0) + { + +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "All %d daemons started, now connecting peers!\n", + num_peers); +#endif + GNUNET_SCHEDULER_cancel (sched, die_task); + + expected_connections = -1; + if ((pg != NULL) && (peers_left == 0)) + { + expected_connections = GNUNET_TESTING_connect_topology (pg, connection_topology, connect_topology_option, connect_topology_option_modifier); +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Have %d expected connections\n", expected_connections); +#endif + } + + if (expected_connections == GNUNET_SYSERR) + { + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from connect topology (bad return)"); + } + + die_task = GNUNET_SCHEDULER_add_delayed (sched, + TIMEOUT, + &end_badly, "from connect topology (timeout)"); + + ok = 0; + } +} + +static void +create_topology () +{ + peers_left = num_peers; /* Reset counter */ + if (GNUNET_TESTING_create_topology (pg, topology, blacklist_topology, blacklist_transports) != GNUNET_SYSERR) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Topology set up, now starting peers!\n"); +#endif + GNUNET_TESTING_daemons_continue_startup(pg); + } + else + { + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_now (sched, + &end_badly, "from create topology (bad return)"); + } + GNUNET_SCHEDULER_cancel (sched, die_task); + die_task = GNUNET_SCHEDULER_add_delayed (sched, + TIMEOUT, + &end_badly, "from continue startup (timeout)"); +} + +/** + * Callback indicating that the hostkey was created for a peer. + * + * @param cls NULL + * @param id the peer identity + * @param d the daemon handle (pretty useless at this point, remove?) + * @param emsg non-null on failure + */ +void hostkey_callback (void *cls, + const struct GNUNET_PeerIdentity *id, + struct GNUNET_TESTING_Daemon *d, + const char *emsg) +{ + if (emsg != NULL) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Hostkey callback received error: %s\n", emsg); + } + +#if VERBOSE > 1 + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Hostkey (%d/%d) created for peer `%s'\n", + num_peers - peers_left, num_peers, GNUNET_i2s(id)); +#endif + + + peers_left--; + if (peers_left == 0) + { +#if VERBOSE + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "All %d hostkeys created, now creating topology!\n", + num_peers); +#endif + GNUNET_SCHEDULER_cancel (sched, die_task); + /* Set up task in case topology creation doesn't finish + * within a reasonable amount of time */ + die_task = GNUNET_SCHEDULER_add_delayed (sched, + TIMEOUT, + &end_badly, "from create_topology"); + GNUNET_SCHEDULER_add_now(sched, &create_topology, NULL); + ok = 0; + } +} + + +static void +run (void *cls, + struct GNUNET_SCHEDULER_Handle *s, + char *const *args, + const char *cfgfile, const struct GNUNET_CONFIGURATION_Handle *cfg) +{ + char * topology_str; + char * connect_topology_str; + char * blacklist_topology_str; + char * connect_topology_option_str; + char * connect_topology_option_modifier_string; + sched = s; + + /* Get path from configuration file */ + if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string(cfg, "paths", "servicehome", &test_directory)) + { + ok = 404; + return; + } + + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "topology", + &topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&topology, topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "TOPOLOGY"); + topology = GNUNET_TESTING_TOPOLOGY_CLIQUE; /* Defaults to NONE, so set better default here */ + } + + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology", + &connect_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&connection_topology, connect_topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid connect topology `%s' given for section %s option %s\n", connect_topology_str, "TESTING", "CONNECT_TOPOLOGY"); + } + GNUNET_free_non_null(connect_topology_str); + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "connect_topology_option", + &connect_topology_option_str)) && (GNUNET_NO == GNUNET_TESTING_topology_option_get(&connect_topology_option, connect_topology_option_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid connect topology option `%s' given for section %s option %s\n", connect_topology_option_str, "TESTING", "CONNECT_TOPOLOGY_OPTION"); + connect_topology_option = GNUNET_TESTING_TOPOLOGY_OPTION_ALL; /* Defaults to NONE, set to ALL */ + } + GNUNET_free_non_null(connect_topology_option_str); + if (GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "connect_topology_option_modifier", + &connect_topology_option_modifier_string)) + { + if (sscanf(connect_topology_option_modifier_string, "%lf", &connect_topology_option_modifier) != 1) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + _("Invalid value `%s' for option `%s' in section `%s': expected float\n"), + connect_topology_option_modifier_string, + "connect_topology_option_modifier", + "TESTING"); + } + GNUNET_free (connect_topology_option_modifier_string); + } + + if (GNUNET_YES != GNUNET_CONFIGURATION_get_value_string (cfg, "testing", "blacklist_transports", + &blacklist_transports)) + blacklist_transports = NULL; + + if ((GNUNET_YES == + GNUNET_CONFIGURATION_get_value_string(cfg, "testing", "blacklist_topology", + &blacklist_topology_str)) && (GNUNET_NO == GNUNET_TESTING_topology_get(&blacklist_topology, blacklist_topology_str))) + { + GNUNET_log (GNUNET_ERROR_TYPE_WARNING, + "Invalid topology `%s' given for section %s option %s\n", topology_str, "TESTING", "BLACKLIST_TOPOLOGY"); + } + GNUNET_free_non_null(topology_str); + GNUNET_free_non_null(blacklist_topology_str); + + /* Get number of peers to start from configuration */ + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "testing", "num_peers", + &num_peers)) + num_peers = DEFAULT_NUM_PEERS; + + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_puts", + &num_puts)) + num_puts = DEFAULT_NUM_PEERS; + + if (GNUNET_SYSERR == + GNUNET_CONFIGURATION_get_value_number (cfg, "dht_testing", "num_gets", + &num_gets)) + num_gets = DEFAULT_NUM_PEERS; + + /* Set peers_left so we know when all peers started */ + peers_left = num_peers; + + /* Set up a task to end testing if peer start fails */ + die_task = GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), + &end_badly, "didn't generate all hostkeys within a reasonable amount of time!!!"); + + pg = GNUNET_TESTING_daemons_start (sched, cfg, + peers_left, GNUNET_TIME_relative_multiply(GNUNET_TIME_UNIT_SECONDS, SECONDS_PER_PEER_START * num_peers), &hostkey_callback, NULL, &peers_started_callback, NULL, + &topology_callback, NULL, NULL); + +} + +static int +check () +{ + int ret; + /* Arguments for GNUNET_PROGRAM_run */ + char *const argv[] = {"test-dht-multipeer", /* Name to give running binary */ + "-c", + "test_dht_multipeer_data.conf", /* Config file to use */ +#if VERBOSE + "-L", "DEBUG", +#endif + NULL + }; + struct GNUNET_GETOPT_CommandLineOption options[] = { + GNUNET_GETOPT_OPTION_END + }; + /* Run the run function as a new program */ + ret = GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, + argv, "test-dht-multipeer", "nohelp", + options, &run, &ok); + if (ret != GNUNET_OK) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "`test-dht-multipeer': Failed with error code %d\n", ret); + } + return ok; +} + +int +main (int argc, char *argv[]) +{ + int ret; + + GNUNET_log_setup ("test-dht-multipeer", +#if VERBOSE + "DEBUG", +#else + "WARNING", +#endif + NULL); + ret = check (); + /** + * Need to remove base directory, subdirectories taken care + * of by the testing framework. + */ + if (GNUNET_DISK_directory_remove (test_directory) != GNUNET_OK) + { + GNUNET_log(GNUNET_ERROR_TYPE_WARNING, "Failed to remove testing directory %s\n", test_directory); + } + return ret; +} + +/* end of test_dht_twopeer_put_get.c */ diff --git a/src/dht/test_dht_multipeer_data.conf b/src/dht/test_dht_multipeer_data.conf new file mode 100644 index 000000000..303d2f60d --- /dev/null +++ b/src/dht/test_dht_multipeer_data.conf @@ -0,0 +1,87 @@ +[fs] +ACCEPT_FROM6 = ::1; +ACCEPT_FROM = 127.0.0.1; +BINARY = gnunet-service-fs +CONFIG = $DEFAULTCONFIG +HOME = $SERVICEHOME +HOSTNAME = localhost +PORT = 2094 +INDEXDB = $SERVICEHOME/idxinfo.lst + +[dht] +DEBUG = YES +STOP_ON_CLOSEST = YES +AUTOSTART = YES +ACCEPT_FROM6 = ::1; +ACCEPT_FROM = 127.0.0.1; +BINARY = gnunet-service-dht-new +#PREFIX = xterm -T dht -e gdb --args +#PREFIX = valgrind --log-file=dht_%p +CONFIG = $DEFAULTCONFIG +HOME = $SERVICEHOME +HOSTNAME = localhost +PORT = 2100 + + +[dhtcache] +QUOTA = 1000000 +DATABASE = sqlite + +[transport] +PLUGINS = tcp +DEBUG = NO +ACCEPT_FROM6 = ::1; +ACCEPT_FROM = 127.0.0.1; +NEIGHBOUR_LIMIT = 50 +BINARY = gnunet-service-transport +CONFIG = $DEFAULTCONFIG +HOME = $SERVICEHOME +HOSTNAME = localhost +PORT = 12365 + +[core] +TOTAL_QUOTA_OUT = 3932160 +TOTAL_QUOTA_IN = 3932160 +ACCEPT_FROM6 = ::1; +ACCEPT_FROM = 127.0.0.1; +BINARY = gnunet-service-core +CONFIG = $DEFAULTCONFIG +HOME = $SERVICEHOME +HOSTNAME = localhost +PORT = 12092 + +[arm] +DEFAULTSERVICES = core dht +ACCEPT_FROM6 = ::1; +ACCEPT_FROM = 127.0.0.1; +BINARY = gnunet-service-arm +CONFIG = $DEFAULTCONFIG +HOME = $SERVICEHOME +HOSTNAME = localhost +PORT = 12366 +DEBUG = YES + +[transport-tcp] +TIMEOUT = 300000 +PORT = 12368 + +[DHT_TESTING] +MYSQL_LOGGING_EXTENDED = NO +NUM_GETS = 10 +NUM_PUTS = 10 + +[TESTING] +CONNECT_TOPOLOGY = CLIQUE +CONNECT_TOPOLOGY_OPTION = CONNECT_RANDOM_SUBSET +CONNECT_TOPOLOGY_OPTION_MODIFIER = 2 +#LOGNMODIFIER = .65 +#PERCENTAGE = .75 +WEAKRANDOM = YES +NUM_PEERS = 5 + +[gnunetd] +HOSTKEY = $SERVICEHOME/.hostkey + +[PATHS] +DEFAULTCONFIG = test_dht_multipeer_data.conf +SERVICEHOME = /tmp/test-dht-multipeer/ diff --git a/src/dht/test_dht_twopeer_data.conf b/src/dht/test_dht_twopeer_data.conf index e551cb611..a4e8be65e 100644 --- a/src/dht/test_dht_twopeer_data.conf +++ b/src/dht/test_dht_twopeer_data.conf @@ -9,15 +9,19 @@ PORT = 2094 INDEXDB = $SERVICEHOME/idxinfo.lst [dht] -DEBUG = NO +DEBUG = YES AUTOSTART = YES ACCEPT_FROM6 = ::1; ACCEPT_FROM = 127.0.0.1; BINARY = gnunet-service-dht +#BINARY = gnunet-service-dht-new +#BINARY = /home/mrwiggles/documents/research/gnunet/gnunet-ng/src/dht/.libs/gnunet-service-dht-new +#PREFIX = xterm -T dht -e gdb --args CONFIG = $DEFAULTCONFIG HOME = $SERVICEHOME HOSTNAME = localhost PORT = 2100 +DISABLE_SOCKET_FORWARDING = YES [dhtcache] QUOTA = 1000000