From: Christian Grothoff Date: Thu, 21 Oct 2010 12:00:05 +0000 (+0000) Subject: more testcases, allow location uris in fs_download X-Git-Tag: initial-import-from-subversion-38251~19999 X-Git-Url: https://git.librecmc.org/?a=commitdiff_plain;h=7e655d5542a9912e14e779915acef2791cfa55c7;p=oweals%2Fgnunet.git more testcases, allow location uris in fs_download --- diff --git a/src/fs/Makefile.am b/src/fs/Makefile.am index bbac950a0..c798e9947 100644 --- a/src/fs/Makefile.am +++ b/src/fs/Makefile.am @@ -135,7 +135,9 @@ check_PROGRAMS = \ test_fs_uri \ test_gnunet_service_fs_migration \ test_gnunet_service_fs_p2p \ - perf_gnunet_service_fs_p2p + perf_gnunet_service_fs_p2p \ + perf_gnunet_service_fs_p2p_dht \ + perf_gnunet_service_fs_p2p_index if HAVE_PYTHON_PEXPECT @@ -303,6 +305,22 @@ perf_gnunet_service_fs_p2p_LDADD = \ $(top_builddir)/src/fs/libgnunetfs.la \ $(top_builddir)/src/util/libgnunetutil.la +perf_gnunet_service_fs_p2p_index_SOURCES = \ + perf_gnunet_service_fs_p2p.c +perf_gnunet_service_fs_p2p_index_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + +perf_gnunet_service_fs_p2p_dht_SOURCES = \ + perf_gnunet_service_fs_p2p.c +perf_gnunet_service_fs_p2p_dht_LDADD = \ + $(top_builddir)/src/fs/libgnunetfstest.a \ + $(top_builddir)/src/testing/libgnunettesting.la \ + $(top_builddir)/src/fs/libgnunetfs.la \ + $(top_builddir)/src/util/libgnunetutil.la + do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g' diff --git a/src/fs/fs.h b/src/fs/fs.h index 9d662b267..68ed2184e 100644 --- a/src/fs/fs.h +++ b/src/fs/fs.h @@ -37,6 +37,18 @@ /** * Maximum number of outgoing messages we queue per peer. + * + * Performance measurements for 2 peer setup for 50 MB file + * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1): + * + * 2: 1700 kb/s, 1372 kb/s + * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s + * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s + * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s + * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s + * + * Conclusion: 16 seems to be a pretty good value (stable + * and high performance, no excessive memory use). */ #define MAX_QUEUE_PER_PEER 16 diff --git a/src/fs/fs_download.c b/src/fs/fs_download.c index c7596de14..2be58aa1a 100644 --- a/src/fs/fs_download.c +++ b/src/fs/fs_download.c @@ -32,7 +32,7 @@ #include "fs.h" #include "fs_tree.h" -#define DEBUG_DOWNLOAD GNUNET_YES +#define DEBUG_DOWNLOAD GNUNET_NO /** * Determine if the given download (options and meta data) should cause @@ -1719,9 +1719,11 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, struct GNUNET_FS_ProgressInfo pi; struct GNUNET_FS_DownloadContext *dc; - GNUNET_assert (GNUNET_FS_uri_test_chk (uri)); + GNUNET_assert (GNUNET_FS_uri_test_chk (uri) || + GNUNET_FS_uri_test_loc (uri) ); + if ( (offset + length < offset) || - (offset + length > uri->data.chk.file_length) ) + (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) ) { GNUNET_break (0); return NULL; @@ -1762,7 +1764,7 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, dc->anonymity = anonymity; dc->options = options; dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE)); - dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length)); + dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri)); if ( (filename == NULL) && (is_recursive_download (dc) ) ) { @@ -1787,7 +1789,9 @@ GNUNET_FS_download_start (struct GNUNET_FS_Handle *h, pi.value.download.specifics.start.meta = meta; GNUNET_FS_download_make_status_ (&pi, dc); schedule_block_download (dc, - &dc->uri->data.chk.chk, + (dc->uri->type == chk) + ? &dc->uri->data.chk.chk + : &dc->uri->data.loc.fi.chk, 0, 1 /* 0 == CHK, 1 == top */); GNUNET_FS_download_sync_ (dc); diff --git a/src/fs/fs_test_lib.c b/src/fs/fs_test_lib.c index dd646eafe..f91092c4e 100644 --- a/src/fs/fs_test_lib.c +++ b/src/fs/fs_test_lib.c @@ -27,6 +27,7 @@ * @author Christian Grothoff */ #include "platform.h" +#include "fs.h" #include "fs_test_lib.h" #include "gnunet_testing_lib.h" @@ -105,6 +106,11 @@ struct GNUNET_FS_TestDaemon */ struct GNUNET_FS_Uri *publish_uri; + /** + * Name of the temporary file used, or NULL for none. + */ + char *publish_tmp_file; + /** * Scheduler to use (for download_cont). */ @@ -562,6 +568,11 @@ GNUNET_FS_TEST_daemons_stop (struct GNUNET_SCHEDULER_Handle *sched, GNUNET_FS_stop (daemons[i]->fs); if (daemons[i]->cfg != NULL) GNUNET_CONFIGURATION_destroy (daemons[i]->cfg); + if (NULL != daemons[i]->publish_tmp_file) + { + GNUNET_DISK_directory_remove (daemons[i]->publish_tmp_file); + GNUNET_free (daemons[i]->publish_tmp_file); + } GNUNET_free (daemons[i]); daemons[i] = NULL; } @@ -647,25 +658,71 @@ GNUNET_FS_TEST_publish (struct GNUNET_SCHEDULER_Handle *sched, GNUNET_FS_TEST_UriContinuation cont, void *cont_cls) { - GNUNET_assert (daemon->publish_cont == NULL); struct GNUNET_FS_FileInformation *fi; + struct GNUNET_DISK_FileHandle *fh; + char *emsg; + uint64_t off; + char buf[DBLOCK_SIZE]; + size_t bsize; + GNUNET_assert (daemon->publish_cont == NULL); daemon->publish_cont = cont; daemon->publish_cont_cls = cont_cls; daemon->publish_seed = seed; daemon->verbose = verbose; daemon->publish_sched = sched; - fi = GNUNET_FS_file_information_create_from_reader (daemon->fs, - daemon, - size, - &file_generator, - daemon, - NULL, - NULL, - do_index, - anonymity, - 42 /* priority */, - GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS)); + if (GNUNET_YES == do_index) + { + GNUNET_assert (daemon->publish_tmp_file == NULL); + daemon->publish_tmp_file = GNUNET_DISK_mktemp ("fs-test-publish-index"); + GNUNET_assert (daemon->publish_tmp_file != NULL); + fh = GNUNET_DISK_file_open (daemon->publish_tmp_file, + GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_CREATE, + GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE); + GNUNET_assert (NULL != fh); + off = 0; + while (off < size) + { + bsize = GNUNET_MIN (sizeof (buf), + size - off); + GNUNET_assert (bsize == + file_generator (daemon, + off, + bsize, + buf, + &emsg)); + GNUNET_assert (emsg == NULL); + GNUNET_assert (bsize == + GNUNET_DISK_file_write (fh, + buf, + bsize)); + off += bsize; + } + GNUNET_assert (GNUNET_OK == + GNUNET_DISK_file_close (fh)); + fi = GNUNET_FS_file_information_create_from_file (daemon->fs, + daemon, + daemon->publish_tmp_file, + NULL, NULL, + do_index, + anonymity, + 42 /* priority */, + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS)); + } + else + { + fi = GNUNET_FS_file_information_create_from_reader (daemon->fs, + daemon, + size, + &file_generator, + daemon, + NULL, + NULL, + do_index, + anonymity, + 42 /* priority */, + GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS)); + } daemon->publish_context = GNUNET_FS_publish_start (daemon->fs, fi, NULL, NULL, NULL, diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf index 4e7dabd9d..25f9979bb 100644 --- a/src/fs/fs_test_lib_data.conf +++ b/src/fs/fs_test_lib_data.conf @@ -20,7 +20,7 @@ HOSTNAME = localhost DEFAULTSERVICES = fs [datastore] -DEBUG = YES +# DEBUG = YES #PREFIX = valgrind --tool=memcheck --leak-check=yes QUOTA = 2000000000 @@ -52,7 +52,7 @@ PORT = 43471 HOSTNAME = localhost #OPTIONS = -L DEBUG ACTIVEMIGRATION = NO -DEBUG = YES +# DEBUG = YES #PREFIX = valgrind --tool=memcheck --leak-check=yes #BINARY = /home/grothoff/gn9/bin/gnunet-service-fs #PREFIX = xterm -e gdb -x cmd --args diff --git a/src/fs/gnunet-service-fs.c b/src/fs/gnunet-service-fs.c index 9ba722c3c..f59f18546 100644 --- a/src/fs/gnunet-service-fs.c +++ b/src/fs/gnunet-service-fs.c @@ -54,14 +54,14 @@ * might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for * a rather wasteful mode of operation (that might still get the highest * throughput overall). + * + * Performance measurements (for 50 MB file, 2 peers): + * + * - Without delays: 3300 kb/s + * - With delays: 101 kb/s */ #define SUPPORT_DELAYS GNUNET_NO -/** - * Currently experimental code... - */ -#define ENABLE_LOAD_MGMT GNUNET_YES - /** * Size for the hash map for DHT requests from the FS * service. Should be about the number of concurrent @@ -101,6 +101,24 @@ * give us 5 MB/s. OTOH, obviously re-trying the same peer can be * rather inefficient in larger networks, hence picking 1 is in * general not the best choice. + * + * Performance measurements (for 50 MB file, 2 peers, no delays): + * + * - 1: 3300 kb/s (consistently) + * - 3: 2046 kb/s, 754 kb/s, 3490 kb/s + * - 5: 759 kb/s, 968 kb/s, 1160 kb/s + * + * Note that this does NOT mean that the value should be 1 since + * a 2-peer network is far from representative here (and this fails + * to take into consideration bandwidth wasted by repeatedly + * sending queries to peers that don't have the content). Also, + * it is expected that higher values lead to more inconsistent + * measurements since this only affects lost messages towards the + * end of the download. + * + * Finally, we should probably consider changing this and making + * it dependent on the number of connected peers or a related + * metric (bad magic constants...). */ #define RETRY_PROBABILITY_INV 1 @@ -1565,7 +1583,7 @@ peer_connect_handler (void *cls, uint32_t trust; cp = GNUNET_malloc (sizeof (struct ConnectedPeer)); - cp->transmission_delay = GNUNET_LOAD_value_init (GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT); + cp->transmission_delay = GNUNET_LOAD_value_init (latency); cp->pid = GNUNET_PEER_intern (peer); fn = get_trust_filename (peer); @@ -1589,6 +1607,39 @@ peer_connect_handler (void *cls, } +/** + * Method called whenever a given peer has a status change. + * + * @param cls closure + * @param peer peer identity this notification is about + * @param latency reported latency of the connection with 'other' + * @param distance reported distance (DV) to 'other' + * @param bandwidth_in available amount of inbound bandwidth + * @param bandwidth_out available amount of outbound bandwidth + * @param timeout absolute time when this peer will time out + * unless we see some further activity from it + */ +static void +peer_status_handler (void *cls, + const struct + GNUNET_PeerIdentity * peer, + struct GNUNET_TIME_Relative latency, + uint32_t distance, + struct GNUNET_BANDWIDTH_Value32NBO bandwidth_in, + struct GNUNET_BANDWIDTH_Value32NBO bandwidth_out, + struct GNUNET_TIME_Absolute timeout) +{ + struct ConnectedPeer *cp; + + cp = GNUNET_CONTAINER_multihashmap_get (connected_peers, + &peer->hashPubKey); + GNUNET_assert (cp != NULL); + GNUNET_LOAD_value_set_decline (cp->transmission_delay, + latency); +} + + + /** * Increase the host credit by a value. * @@ -3804,12 +3855,8 @@ process_local_reply (void *cls, gettext_noop ("# processing result set cut short due to load"), 1, GNUNET_NO); - /* FIXME: if this is activated, we might stall large downloads - indefinitely since (presumably) the load can never go down again! */ -#if ENABLE_LOAD_MGMT GNUNET_DATASTORE_get_next (dsh, GNUNET_NO); return; -#endif } GNUNET_DATASTORE_get_next (dsh, GNUNET_YES); } @@ -4021,14 +4068,7 @@ handle_p2p_get (void *cls, "Dropping query from `%s', this peer is too busy.\n", GNUNET_i2s (other)); #endif - GNUNET_STATISTICS_update (stats, - gettext_noop ("# requests dropped due to high load"), - 1, - GNUNET_NO); -#if ENABLE_LOAD_MGMT - /* FIXME: this causes problems... */ return GNUNET_OK; -#endif } #if DEBUG_FS GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, @@ -4053,12 +4093,7 @@ handle_p2p_get (void *cls, /* don't have BW to send to peer, or would likely take longer than we have for it, so at best indirect the query */ priority = 0; -#if ENABLE_LOAD_MGMT - /* FIXME: if this line is enabled, the 'perf' test for larger files simply "hangs"; - the cause seems to be that the load goes up (to the point where we do this) - and then never goes down again... (outch) */ pr->forward_only = GNUNET_YES; -#endif } pr->type = type; pr->mingle = ntohl (gm->filter_mutator); @@ -4457,7 +4492,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s, } connected_peers = GNUNET_CONTAINER_multihashmap_create (enc); query_request_map = GNUNET_CONTAINER_multihashmap_create (max_pending_requests); - rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_SECONDS); + rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL); peer_request_map = GNUNET_CONTAINER_multihashmap_create (enc); requests_by_expiration_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN); core = GNUNET_CORE_connect (sched, @@ -4467,7 +4502,7 @@ main_init (struct GNUNET_SCHEDULER_Handle *s, NULL, &peer_connect_handler, &peer_disconnect_handler, - NULL, + &peer_status_handler, NULL, GNUNET_NO, NULL, GNUNET_NO, p2p_handlers); diff --git a/src/fs/perf_gnunet_service_fs_p2p.c b/src/fs/perf_gnunet_service_fs_p2p.c index ac4458994..fe57c1666 100644 --- a/src/fs/perf_gnunet_service_fs_p2p.c +++ b/src/fs/perf_gnunet_service_fs_p2p.c @@ -27,12 +27,12 @@ #include "fs_test_lib.h" #include "gnunet_testing_lib.h" -#define VERBOSE GNUNET_YES +#define VERBOSE GNUNET_NO /** * File-size we use for testing. */ -#define FILESIZE (1024 * 1024 * 1) +#define FILESIZE (1024 * 1024 * 50) /** * How long until we give up on transmitting the message? @@ -51,6 +51,8 @@ static int ok; static struct GNUNET_TIME_Absolute start_time; +static const char *progname; + static void do_stop (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) @@ -87,21 +89,19 @@ static struct StatValues stats[] = { "fs", "# results found locally"}, { "fs", "# requests forwarded due to high load"}, { "fs", "# requests done for free (low load)"}, + { "fs", "# requests dropped, priority insufficient"}, + { "fs", "# requests done for a price (normal load)"}, + { "fs", "# requests dropped by datastore (queue length limit)"}, { "fs", "# P2P searches received"}, - { "fs", "# replies received for local clients"}, { "fs", "# P2P searches discarded (queue length bound)"}, - { "fs", "# requests dropped due to high load"}, - { "fs", "# requests dropped by datastore (queue length limit)"}, + { "fs", "# replies received for local clients"}, { "fs", "# queries retransmitted to same target"}, { "fs", "cummulative artificial delay introduced (ms)"}, { "core", "# bytes decrypted"}, { "core", "# bytes encrypted"}, - { "core", "# transmissions delayed due to corking"}, { "transport", "# bytes received via TCP"}, { "transport", "# bytes transmitted via TCP"}, { "datacache", "# bytes stored"}, - { "dht", "# DHT ROUTE Requests Seen"}, - { "dht", "# DHT ROUTE Requests Forwarded"}, { NULL, NULL} }; @@ -236,6 +236,8 @@ static void do_download (void *cls, const struct GNUNET_FS_Uri *uri) { + int anonymity; + if (NULL == uri) { GNUNET_FS_TEST_daemons_stop (sched, @@ -250,10 +252,14 @@ do_download (void *cls, "Downloading %llu bytes\n", (unsigned long long) FILESIZE); start_time = GNUNET_TIME_absolute_get (); + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; GNUNET_FS_TEST_download (sched, daemons[0], TIMEOUT, - 1, SEED, uri, + anonymity, SEED, uri, VERBOSE, &do_report, NULL); } @@ -263,6 +269,9 @@ static void do_publish (void *cls, const char *emsg) { + int do_index; + int anonymity; + if (NULL != emsg) { GNUNET_FS_TEST_daemons_stop (sched, @@ -277,10 +286,20 @@ do_publish (void *cls, GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Publishing %llu bytes\n", (unsigned long long) FILESIZE); + if (NULL != strstr (progname, "index")) + do_index = GNUNET_YES; + else + do_index = GNUNET_NO; + if (NULL != strstr (progname, "dht")) + anonymity = 0; + else + anonymity = 1; + GNUNET_FS_TEST_publish (sched, daemons[NUM_DAEMONS-1], TIMEOUT, - 1, GNUNET_NO, FILESIZE, SEED, + anonymity, + do_index, FILESIZE, SEED, VERBOSE, &do_download, NULL); } @@ -343,9 +362,9 @@ main (int argc, char *argv[]) struct GNUNET_GETOPT_CommandLineOption options[] = { GNUNET_GETOPT_OPTION_END }; - + progname = argv[0]; GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); - GNUNET_log_setup ("perf_gnunet_service_fs_p2p", + GNUNET_log_setup ("perf_gnunet_service_fs_p2p_index", #if VERBOSE "DEBUG", #else @@ -353,10 +372,10 @@ main (int argc, char *argv[]) #endif NULL); GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1, - argvx, "perf-gnunet-service-fs-p2p", + argvx, "perf-gnunet-service-fs-p2p-index", "nohelp", options, &run, NULL); GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/"); return ok; } -/* end of perf_gnunet_service_fs_p2p.c */ +/* end of perf_gnunet_service_fs_p2p_index.c */