test_fs_uri \
test_gnunet_service_fs_migration \
test_gnunet_service_fs_p2p \
- perf_gnunet_service_fs_p2p
+ perf_gnunet_service_fs_p2p \
+ perf_gnunet_service_fs_p2p_dht \
+ perf_gnunet_service_fs_p2p_index
if HAVE_PYTHON_PEXPECT
$(top_builddir)/src/fs/libgnunetfs.la \
$(top_builddir)/src/util/libgnunetutil.la
+perf_gnunet_service_fs_p2p_index_SOURCES = \
+ perf_gnunet_service_fs_p2p.c
+perf_gnunet_service_fs_p2p_index_LDADD = \
+ $(top_builddir)/src/fs/libgnunetfstest.a \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/fs/libgnunetfs.la \
+ $(top_builddir)/src/util/libgnunetutil.la
+
+perf_gnunet_service_fs_p2p_dht_SOURCES = \
+ perf_gnunet_service_fs_p2p.c
+perf_gnunet_service_fs_p2p_dht_LDADD = \
+ $(top_builddir)/src/fs/libgnunetfstest.a \
+ $(top_builddir)/src/testing/libgnunettesting.la \
+ $(top_builddir)/src/fs/libgnunetfs.la \
+ $(top_builddir)/src/util/libgnunetutil.la
+
do_subst = $(SED) -e 's,[@]PYTHON[@],$(PYTHON),g'
/**
* Maximum number of outgoing messages we queue per peer.
+ *
+ * Performance measurements for 2 peer setup for 50 MB file
+ * (with MAX_DATASTORE_QUEUE = 1 and RETRY_PROBABILITY_INV = 1):
+ *
+ * 2: 1700 kb/s, 1372 kb/s
+ * 8: 2117 kb/s, 1284 kb/s, 1112 kb/s
+ * 16: 3500 kb/s, 3200 kb/s, 3388 kb/s
+ * 32: 3441 kb/s, 3163 kb/s, 3277 kb/s
+ * 128: 1700 kb/s; 2010 kb/s, 3383 kb/s, 1156 kb/s
+ *
+ * Conclusion: 16 seems to be a pretty good value (stable
+ * and high performance, no excessive memory use).
*/
#define MAX_QUEUE_PER_PEER 16
#include "fs.h"
#include "fs_tree.h"
-#define DEBUG_DOWNLOAD GNUNET_YES
+#define DEBUG_DOWNLOAD GNUNET_NO
/**
* Determine if the given download (options and meta data) should cause
struct GNUNET_FS_ProgressInfo pi;
struct GNUNET_FS_DownloadContext *dc;
- GNUNET_assert (GNUNET_FS_uri_test_chk (uri));
+ GNUNET_assert (GNUNET_FS_uri_test_chk (uri) ||
+ GNUNET_FS_uri_test_loc (uri) );
+
if ( (offset + length < offset) ||
- (offset + length > uri->data.chk.file_length) )
+ (offset + length > GNUNET_FS_uri_chk_get_file_size (uri)) )
{
GNUNET_break (0);
return NULL;
dc->anonymity = anonymity;
dc->options = options;
dc->active = GNUNET_CONTAINER_multihashmap_create (1 + 2 * (length / DBLOCK_SIZE));
- dc->treedepth = GNUNET_FS_compute_depth (GNUNET_ntohll(dc->uri->data.chk.file_length));
+ dc->treedepth = GNUNET_FS_compute_depth (GNUNET_FS_uri_chk_get_file_size(dc->uri));
if ( (filename == NULL) &&
(is_recursive_download (dc) ) )
{
pi.value.download.specifics.start.meta = meta;
GNUNET_FS_download_make_status_ (&pi, dc);
schedule_block_download (dc,
- &dc->uri->data.chk.chk,
+ (dc->uri->type == chk)
+ ? &dc->uri->data.chk.chk
+ : &dc->uri->data.loc.fi.chk,
0,
1 /* 0 == CHK, 1 == top */);
GNUNET_FS_download_sync_ (dc);
* @author Christian Grothoff
*/
#include "platform.h"
+#include "fs.h"
#include "fs_test_lib.h"
#include "gnunet_testing_lib.h"
*/
struct GNUNET_FS_Uri *publish_uri;
+ /**
+ * Name of the temporary file used, or NULL for none.
+ */
+ char *publish_tmp_file;
+
/**
* Scheduler to use (for download_cont).
*/
GNUNET_FS_stop (daemons[i]->fs);
if (daemons[i]->cfg != NULL)
GNUNET_CONFIGURATION_destroy (daemons[i]->cfg);
+ if (NULL != daemons[i]->publish_tmp_file)
+ {
+ GNUNET_DISK_directory_remove (daemons[i]->publish_tmp_file);
+ GNUNET_free (daemons[i]->publish_tmp_file);
+ }
GNUNET_free (daemons[i]);
daemons[i] = NULL;
}
GNUNET_FS_TEST_UriContinuation cont,
void *cont_cls)
{
- GNUNET_assert (daemon->publish_cont == NULL);
struct GNUNET_FS_FileInformation *fi;
+ struct GNUNET_DISK_FileHandle *fh;
+ char *emsg;
+ uint64_t off;
+ char buf[DBLOCK_SIZE];
+ size_t bsize;
+ GNUNET_assert (daemon->publish_cont == NULL);
daemon->publish_cont = cont;
daemon->publish_cont_cls = cont_cls;
daemon->publish_seed = seed;
daemon->verbose = verbose;
daemon->publish_sched = sched;
- fi = GNUNET_FS_file_information_create_from_reader (daemon->fs,
- daemon,
- size,
- &file_generator,
- daemon,
- NULL,
- NULL,
- do_index,
- anonymity,
- 42 /* priority */,
- GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+ if (GNUNET_YES == do_index)
+ {
+ GNUNET_assert (daemon->publish_tmp_file == NULL);
+ daemon->publish_tmp_file = GNUNET_DISK_mktemp ("fs-test-publish-index");
+ GNUNET_assert (daemon->publish_tmp_file != NULL);
+ fh = GNUNET_DISK_file_open (daemon->publish_tmp_file,
+ GNUNET_DISK_OPEN_WRITE | GNUNET_DISK_OPEN_CREATE,
+ GNUNET_DISK_PERM_USER_READ | GNUNET_DISK_PERM_USER_WRITE);
+ GNUNET_assert (NULL != fh);
+ off = 0;
+ while (off < size)
+ {
+ bsize = GNUNET_MIN (sizeof (buf),
+ size - off);
+ GNUNET_assert (bsize ==
+ file_generator (daemon,
+ off,
+ bsize,
+ buf,
+ &emsg));
+ GNUNET_assert (emsg == NULL);
+ GNUNET_assert (bsize ==
+ GNUNET_DISK_file_write (fh,
+ buf,
+ bsize));
+ off += bsize;
+ }
+ GNUNET_assert (GNUNET_OK ==
+ GNUNET_DISK_file_close (fh));
+ fi = GNUNET_FS_file_information_create_from_file (daemon->fs,
+ daemon,
+ daemon->publish_tmp_file,
+ NULL, NULL,
+ do_index,
+ anonymity,
+ 42 /* priority */,
+ GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+ }
+ else
+ {
+ fi = GNUNET_FS_file_information_create_from_reader (daemon->fs,
+ daemon,
+ size,
+ &file_generator,
+ daemon,
+ NULL,
+ NULL,
+ do_index,
+ anonymity,
+ 42 /* priority */,
+ GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_HOURS));
+ }
daemon->publish_context = GNUNET_FS_publish_start (daemon->fs,
fi,
NULL, NULL, NULL,
DEFAULTSERVICES = fs
[datastore]
-DEBUG = YES
+# DEBUG = YES
#PREFIX = valgrind --tool=memcheck --leak-check=yes
QUOTA = 2000000000
HOSTNAME = localhost
#OPTIONS = -L DEBUG
ACTIVEMIGRATION = NO
-DEBUG = YES
+# DEBUG = YES
#PREFIX = valgrind --tool=memcheck --leak-check=yes
#BINARY = /home/grothoff/gn9/bin/gnunet-service-fs
#PREFIX = xterm -e gdb -x cmd --args
* might want to consider changing 'RETRY_PROBABILITY_INV' to 1 for
* a rather wasteful mode of operation (that might still get the highest
* throughput overall).
+ *
+ * Performance measurements (for 50 MB file, 2 peers):
+ *
+ * - Without delays: 3300 kb/s
+ * - With delays: 101 kb/s
*/
#define SUPPORT_DELAYS GNUNET_NO
-/**
- * Currently experimental code...
- */
-#define ENABLE_LOAD_MGMT GNUNET_YES
-
/**
* Size for the hash map for DHT requests from the FS
* service. Should be about the number of concurrent
* give us 5 MB/s. OTOH, obviously re-trying the same peer can be
* rather inefficient in larger networks, hence picking 1 is in
* general not the best choice.
+ *
+ * Performance measurements (for 50 MB file, 2 peers, no delays):
+ *
+ * - 1: 3300 kb/s (consistently)
+ * - 3: 2046 kb/s, 754 kb/s, 3490 kb/s
+ * - 5: 759 kb/s, 968 kb/s, 1160 kb/s
+ *
+ * Note that this does NOT mean that the value should be 1 since
+ * a 2-peer network is far from representative here (and this fails
+ * to take into consideration bandwidth wasted by repeatedly
+ * sending queries to peers that don't have the content). Also,
+ * it is expected that higher values lead to more inconsistent
+ * measurements since this only affects lost messages towards the
+ * end of the download.
+ *
+ * Finally, we should probably consider changing this and making
+ * it dependent on the number of connected peers or a related
+ * metric (bad magic constants...).
*/
#define RETRY_PROBABILITY_INV 1
uint32_t trust;
cp = GNUNET_malloc (sizeof (struct ConnectedPeer));
- cp->transmission_delay = GNUNET_LOAD_value_init (GNUNET_CONSTANTS_IDLE_CONNECTION_TIMEOUT);
+ cp->transmission_delay = GNUNET_LOAD_value_init (latency);
cp->pid = GNUNET_PEER_intern (peer);
fn = get_trust_filename (peer);
}
+/**
+ * Method called whenever a given peer has a status change.
+ *
+ * @param cls closure
+ * @param peer peer identity this notification is about
+ * @param latency reported latency of the connection with 'other'
+ * @param distance reported distance (DV) to 'other'
+ * @param bandwidth_in available amount of inbound bandwidth
+ * @param bandwidth_out available amount of outbound bandwidth
+ * @param timeout absolute time when this peer will time out
+ * unless we see some further activity from it
+ */
+static void
+peer_status_handler (void *cls,
+ const struct
+ GNUNET_PeerIdentity * peer,
+ struct GNUNET_TIME_Relative latency,
+ uint32_t distance,
+ struct GNUNET_BANDWIDTH_Value32NBO bandwidth_in,
+ struct GNUNET_BANDWIDTH_Value32NBO bandwidth_out,
+ struct GNUNET_TIME_Absolute timeout)
+{
+ struct ConnectedPeer *cp;
+
+ cp = GNUNET_CONTAINER_multihashmap_get (connected_peers,
+ &peer->hashPubKey);
+ GNUNET_assert (cp != NULL);
+ GNUNET_LOAD_value_set_decline (cp->transmission_delay,
+ latency);
+}
+
+
+
/**
* Increase the host credit by a value.
*
gettext_noop ("# processing result set cut short due to load"),
1,
GNUNET_NO);
- /* FIXME: if this is activated, we might stall large downloads
- indefinitely since (presumably) the load can never go down again! */
-#if ENABLE_LOAD_MGMT
GNUNET_DATASTORE_get_next (dsh, GNUNET_NO);
return;
-#endif
}
GNUNET_DATASTORE_get_next (dsh, GNUNET_YES);
}
"Dropping query from `%s', this peer is too busy.\n",
GNUNET_i2s (other));
#endif
- GNUNET_STATISTICS_update (stats,
- gettext_noop ("# requests dropped due to high load"),
- 1,
- GNUNET_NO);
-#if ENABLE_LOAD_MGMT
- /* FIXME: this causes problems... */
return GNUNET_OK;
-#endif
}
#if DEBUG_FS
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
/* don't have BW to send to peer, or would likely take longer than we have for it,
so at best indirect the query */
priority = 0;
-#if ENABLE_LOAD_MGMT
- /* FIXME: if this line is enabled, the 'perf' test for larger files simply "hangs";
- the cause seems to be that the load goes up (to the point where we do this)
- and then never goes down again... (outch) */
pr->forward_only = GNUNET_YES;
-#endif
}
pr->type = type;
pr->mingle = ntohl (gm->filter_mutator);
}
connected_peers = GNUNET_CONTAINER_multihashmap_create (enc);
query_request_map = GNUNET_CONTAINER_multihashmap_create (max_pending_requests);
- rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_SECONDS);
+ rt_entry_lifetime = GNUNET_LOAD_value_init (GNUNET_TIME_UNIT_FOREVER_REL);
peer_request_map = GNUNET_CONTAINER_multihashmap_create (enc);
requests_by_expiration_heap = GNUNET_CONTAINER_heap_create (GNUNET_CONTAINER_HEAP_ORDER_MIN);
core = GNUNET_CORE_connect (sched,
NULL,
&peer_connect_handler,
&peer_disconnect_handler,
- NULL,
+ &peer_status_handler,
NULL, GNUNET_NO,
NULL, GNUNET_NO,
p2p_handlers);
#include "fs_test_lib.h"
#include "gnunet_testing_lib.h"
-#define VERBOSE GNUNET_YES
+#define VERBOSE GNUNET_NO
/**
* File-size we use for testing.
*/
-#define FILESIZE (1024 * 1024 * 1)
+#define FILESIZE (1024 * 1024 * 50)
/**
* How long until we give up on transmitting the message?
static struct GNUNET_TIME_Absolute start_time;
+static const char *progname;
+
static void
do_stop (void *cls,
const struct GNUNET_SCHEDULER_TaskContext *tc)
{ "fs", "# results found locally"},
{ "fs", "# requests forwarded due to high load"},
{ "fs", "# requests done for free (low load)"},
+ { "fs", "# requests dropped, priority insufficient"},
+ { "fs", "# requests done for a price (normal load)"},
+ { "fs", "# requests dropped by datastore (queue length limit)"},
{ "fs", "# P2P searches received"},
- { "fs", "# replies received for local clients"},
{ "fs", "# P2P searches discarded (queue length bound)"},
- { "fs", "# requests dropped due to high load"},
- { "fs", "# requests dropped by datastore (queue length limit)"},
+ { "fs", "# replies received for local clients"},
{ "fs", "# queries retransmitted to same target"},
{ "fs", "cummulative artificial delay introduced (ms)"},
{ "core", "# bytes decrypted"},
{ "core", "# bytes encrypted"},
- { "core", "# transmissions delayed due to corking"},
{ "transport", "# bytes received via TCP"},
{ "transport", "# bytes transmitted via TCP"},
{ "datacache", "# bytes stored"},
- { "dht", "# DHT ROUTE Requests Seen"},
- { "dht", "# DHT ROUTE Requests Forwarded"},
{ NULL, NULL}
};
do_download (void *cls,
const struct GNUNET_FS_Uri *uri)
{
+ int anonymity;
+
if (NULL == uri)
{
GNUNET_FS_TEST_daemons_stop (sched,
"Downloading %llu bytes\n",
(unsigned long long) FILESIZE);
start_time = GNUNET_TIME_absolute_get ();
+ if (NULL != strstr (progname, "dht"))
+ anonymity = 0;
+ else
+ anonymity = 1;
GNUNET_FS_TEST_download (sched,
daemons[0],
TIMEOUT,
- 1, SEED, uri,
+ anonymity, SEED, uri,
VERBOSE,
&do_report, NULL);
}
do_publish (void *cls,
const char *emsg)
{
+ int do_index;
+ int anonymity;
+
if (NULL != emsg)
{
GNUNET_FS_TEST_daemons_stop (sched,
GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
"Publishing %llu bytes\n",
(unsigned long long) FILESIZE);
+ if (NULL != strstr (progname, "index"))
+ do_index = GNUNET_YES;
+ else
+ do_index = GNUNET_NO;
+ if (NULL != strstr (progname, "dht"))
+ anonymity = 0;
+ else
+ anonymity = 1;
+
GNUNET_FS_TEST_publish (sched,
daemons[NUM_DAEMONS-1],
TIMEOUT,
- 1, GNUNET_NO, FILESIZE, SEED,
+ anonymity,
+ do_index, FILESIZE, SEED,
VERBOSE,
&do_download, NULL);
}
struct GNUNET_GETOPT_CommandLineOption options[] = {
GNUNET_GETOPT_OPTION_END
};
-
+ progname = argv[0];
GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/");
- GNUNET_log_setup ("perf_gnunet_service_fs_p2p",
+ GNUNET_log_setup ("perf_gnunet_service_fs_p2p_index",
#if VERBOSE
"DEBUG",
#else
#endif
NULL);
GNUNET_PROGRAM_run ((sizeof (argvx) / sizeof (char *)) - 1,
- argvx, "perf-gnunet-service-fs-p2p",
+ argvx, "perf-gnunet-service-fs-p2p-index",
"nohelp", options, &run, NULL);
GNUNET_DISK_directory_remove ("/tmp/gnunet-test-fs-lib/");
return ok;
}
-/* end of perf_gnunet_service_fs_p2p.c */
+/* end of perf_gnunet_service_fs_p2p_index.c */
projects / oweals / gnunet.git / commitdiff