From 24c7345ea01a015a00a711e91eb893c5c1018abc Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Thu, 18 Mar 2010 11:43:23 +0000 Subject: [PATCH] move --- BUGS | 94 -------------- TODO | 117 ++++++++++++++---- contrib/defaults.conf | 2 +- src/core/core.h | 2 +- src/datastore/plugin_datastore_sqlite.c | 2 +- src/fs/fs_test_lib_data.conf | 2 +- src/fs/gnunet-service-fs_indexing.c | 2 +- src/fs/test_fs_download_data.conf | 2 +- src/hostlist/hostlist-client.c | 2 +- src/hostlist/hostlist-server.c | 2 +- src/peerinfo/peerinfo.h | 2 +- .../test_testing_data_topology_clique.conf | 2 +- src/topology/gnunet-daemon-topology.c | 2 +- .../test_gnunet_service_topology_data.conf | 6 +- src/transport/Makefile.am | 2 +- src/transport/plugin_transport_udp.c | 3 +- src/transport/plugin_transport_udp_nat.c | 2 +- src/transport/test_plugin_transport.c | 2 +- .../test_transport_api_udp_peer2.conf | 1 + src/transport/transport.h | 6 +- 20 files changed, 117 insertions(+), 138 deletions(-) delete mode 100644 BUGS diff --git a/BUGS b/BUGS deleted file mode 100644 index 67c7b407f..000000000 --- a/BUGS +++ /dev/null @@ -1,94 +0,0 @@ -This file lists minor work items (also possibly called "known bugs"). -We are not tracking them in Mantis yet since there are too many and no -sane end-user should care about this codebase yet anyway. - - -* UTIL: - - only connect() sockets that are ready (select()) [Nils] - [On W32, we need to select after calling socket before - doing connect etc.] -* TRANSPORT: - - implement transport API to pretty-print transport address - + transport_api extension (API extension!) - + service-transport extension (protocol extension) - - add calls to statistics in various places - - implement gnunet-transport (transport configurator / tester) - - UPnP-based IP detection - (Note: build library always, build service when libxml2/etc. are available) - - instantly filter addresses from *other* peers that - are *equal* to our own address + port (i.e., localhost:2086). We - no longer filter those for outgoing (helps with loopback testing - and keeps the code clean), but we should filter strictly *impossible* - incoming addresses! This is for efficiency, not correctness. - - We currently are happy to take any address told to us in a WELCOME - to our set of addresses; we should have some minimal threshold-based - scheme, limiting both the total number of addresses that we accept - this way as well as requiring multiple confirmations; also, we - should possibly try to confirm that the given address works for - us ourselves (loopback-style) before adding it to the list - [SECURITY issue] - + we may be able to simplify WELCOME messages (no need to add - addresses there anymore, but may help to learn them there anyway...). - + we probably want some kind of voting/counting for learning IP addresses - (maybe including IP addresses in ads proportional to how often others - report them? we at least need some protection against >64k HELLOs!), - + provide a way to give the user a list of "learned" IP addresses and - a way to easily "veto" addresses off the list! - => If MiM attacker uses vetoed address, blacklist the specific IP for - the presumed neighbour! - - [./transport/gnunet-service-transport.c:173]: (style) struct or union member 'TransportPlugin::rebuild' is never used - - [./transport/plugin_transport_tcp.c:391]: (style) struct or union member 'Plugin::address_update_task' is never used -* FS: - - [./fs/gnunet-service-fs.c:208]: (style) struct or union member 'LocalGetContext::results_bf_size' is never used - - [./fs/gnunet-service-fs.c:501]: (style) struct or union member 'PendingRequest::used_pids_size' is never used - - [./fs/gnunet-service-fs.c:654]: (style) struct or union member 'ConnectedPeer::last_client_replies' is never used - - [./fs/gnunet-service-fs.c:669]: (style) struct or union member 'ConnectedPeer::avg_delay' is never used - - [./fs/gnunet-service-fs.c:675]: (style) struct or union member 'ConnectedPeer::avg_priority' is never used - - [./fs/gnunet-service-fs.c:688]: (style) struct or union member 'ConnectedPeer::pending_requests' is never used - - [./fs/gnunet-service-fs.c:694]: (style) struct or union member 'ConnectedPeer::last_p2p_replies_woff' is never used - - [./fs/gnunet-service-fs.c:700]: (style) struct or union member 'ConnectedPeer::last_client_replies_woff' is never used -* TOPOLOGY: - - If the topology daemon crashes, peers that were put on the - blacklist with transport will never be removed from it (until - transport service dies); we should use the blacklist notification - API to learn about the exact set of blacklisted peers at all times - (FIXME: the transport_api implementation of blacklisting - also does not work nicely for this since it won't let us know about - disconnect-reconnect events and the implicit whitelisting - that might happen here; that's not so bad since we will - re-blacklist on pre-connect attempts anyway, so this is - a minor issue). - - the code uses the term 'blacklist' for both peers that are forbidden - to connect (i.e. F2F mode) as well as peers that we currently - won't try to actively connect to ourselves (since we just tried); - This is confusing. We need two distinct terms. - - move code to use hash table instead of linked list - - instead of periodically discarding blacklisted entries, - simply add task that is triggered at the right time (earlier free, - more balanced load) - - check if new HELLO learned is different from old HELLO - before resetting entire state! -* SETUP: - - auto-generate "defaults.conf" using gnunet-setup from "config.scm" - - integrate all options into "config.scm" - - change config-file writing to exclude options set to default values -* ARM: - - better tracking of which config changes actually need to cause process restarts by ARM. - - listen for requests to discover dependencies between services (and avoid - having to explicitly program start requests) - - better crash management (attach debugging support, capture and analyze - debug output, detect random vs. deterministic crashes) - - shutdown sequence? -* HTTPS transport - - Better SSL-support for MHD - - https integration -* GAP improvements: - - active reply route caching design & implementation of service, - gap extension! -* TESTING: - - consider changing API for peer-group termination to - call continuation when done - -* HOSTLIST: - - 'server' uses 'GNUNET_PEERINFO_iterate', should probably switch to notification API - (for more instant / up-to-date hostlists at lower cost) [OPTIMIZATION] diff --git a/TODO b/TODO index bde3d97b4..0d2725793 100644 --- a/TODO +++ b/TODO @@ -14,11 +14,61 @@ away), in order in which they will likely be done: * FS [Christian] 0.9.0pre0: -* HOSTLIST/PEERINFO/CORE/TRANSPORT/TOPOLOGY: - - hostlist server learns about 2nd peer, but soon forgets (not - clear if this is still an issue, or if validation failed or - if this was an issue with the client's parsing of the reply, which - was broken...) +* TRANSPORT: + - TCP not used bi-directionally (especially important also for PONG!) + Need a way for plugin to tell to service that it can *reliably* transmit + PONGs. Need *good* way to ensure only one (plugin) session is active at + any given point in time; need systematic way to periodically probe latency/ + transport cost changes + - disconnect notifications from TCP not always as nice as we might want + them to be (but how can we do this cleanly given that UDP is connectionless?) + - hostlist testcase shows that (randomly) we jump to an address that + seems to be a plugin function *after* the plugin has been unloaded + (directly from scheduler); check that *all* plugin tasks are cancelled + after plugin is unloaded! (need better testcases for that...) + - implement transport API to pretty-print transport address + + transport_api extension (API extension!) + + service-transport extension (protocol extension) + - implement gnunet-transport (transport configurator / tester) + - instantly filter addresses from *other* peers that + are *equal* to our own address + port (i.e., localhost:2086). We + no longer filter those for outgoing (helps with loopback testing + and keeps the code clean), but we should filter strictly *impossible* + incoming addresses! This is for efficiency, not correctness. + - UPnP-based IP detection + (Note: build library always, build service when libxml2/etc. are available) + - We currently are happy to take any address told to us in a WELCOME + to our set of addresses; we should have some minimal threshold-based + scheme, limiting both the total number of addresses that we accept + this way as well as requiring multiple confirmations; also, we + should possibly try to confirm that the given address works for + us ourselves (loopback-style) before adding it to the list + [SECURITY issue] + + we may be able to simplify WELCOME messages (no need to add + addresses there anymore, but may help to learn them there anyway...). + + we probably want some kind of voting/counting for learning IP addresses + (maybe including IP addresses in ads proportional to how often others + report them? we at least need some protection against >64k HELLOs!), + + provide a way to give the user a list of "learned" IP addresses and + a way to easily "veto" addresses off the list! + => If MiM attacker uses vetoed address, blacklist the specific IP for + the presumed neighbour! + - [./transport/gnunet-service-transport.c:173]: (style) struct or union member 'TransportPlugin::rebuild' is never used + - [./transport/plugin_transport_tcp.c:391]: (style) struct or union member 'Plugin::address_update_task' is never used +* FS: + - [./fs/gnunet-service-fs.c:208]: (style) struct or union member 'LocalGetContext::results_bf_size' is never used + - [./fs/gnunet-service-fs.c:501]: (style) struct or union member 'PendingRequest::used_pids_size' is never used + - [./fs/gnunet-service-fs.c:654]: (style) struct or union member 'ConnectedPeer::last_client_replies' is never used + - [./fs/gnunet-service-fs.c:669]: (style) struct or union member 'ConnectedPeer::avg_delay' is never used + - [./fs/gnunet-service-fs.c:675]: (style) struct or union member 'ConnectedPeer::avg_priority' is never used + - [./fs/gnunet-service-fs.c:688]: (style) struct or union member 'ConnectedPeer::pending_requests' is never used + - [./fs/gnunet-service-fs.c:694]: (style) struct or union member 'ConnectedPeer::last_p2p_replies_woff' is never used + - [./fs/gnunet-service-fs.c:700]: (style) struct or union member 'ConnectedPeer::last_client_replies_woff' is never used +* CORE: + - SET_KEY is triggered every 2.5 minutes (after session is up); should + just do PING (not SET_KEY + PING) + - "Forcing disconnect of XXX due to inactivity" -- can happen every few MS!? + (disconnect does not really succeed, or what?) * TOPOLOGY: - needs more testing (especially F2F topology) - needs to re-try connecting after disconnect (currently, it @@ -27,28 +77,34 @@ away), in order in which they will likely be done: cause seems to be the 'blacklist_after_attempt' being set to 1h, which is rather long -- and should probably be adjusted based on the number of connections / known peers) -* CORE: - - peers do not seem to connect (setkey, ping are there; PONG is sent, - not sure what else happens; eventually, the "neighbour" entry times - out and is removed) + - If the topology daemon crashes, peers that were put on the + blacklist with transport will never be removed from it (until + transport service dies); we should use the blacklist notification + API to learn about the exact set of blacklisted peers at all times + (FIXME: the transport_api implementation of blacklisting + also does not work nicely for this since it won't let us know about + disconnect-reconnect events and the implicit whitelisting + that might happen here; that's not so bad since we will + re-blacklist on pre-connect attempts anyway, so this is + a minor issue). + - the code uses the term 'blacklist' for both peers that are forbidden + to connect (i.e. F2F mode) as well as peers that we currently + won't try to actively connect to ourselves (since we just tried); + This is confusing. We need two distinct terms. + - move code to use hash table instead of linked list + - instead of periodically discarding blacklisted entries, + simply add task that is triggered at the right time (earlier free, + more balanced load) + - check if new HELLO learned is different from old HELLO + before resetting entire state! * UTIL: - - load: need way to determine network load (up/down) -- and quickly so! - trust: need *fast* way to check/update trust in peers (async peerinfo would not be right) - scheduler should change OS process priority based on task priority; should make better use of task priorities in general -* TRANSPORT: - - disconnect notifications from TCP not always as nice as we might want - them to be (but how can we do this cleanly given that UDP is connectionless?) - - TCP not used bi-directionally (especially important also for PONG!) - Need a way for plugin to tell to service that it can *reliably* transmit - PONGs. Need *good* way to ensure only one (plugin) session is active at - any given point in time; need systematic way to periodically probe latency/ - transport cost changes - - hostlist testcase shows that (randomly) we jump to an address that - seems to be a plugin function *after* the plugin has been unloaded - (directly from scheduler); check that *all* plugin tasks are cancelled - after plugin is unloaded! (need better testcases for that...) + - only connect() sockets that are ready (select()) [Nils] + [On W32, we need to select after calling socket before + doing connect etc.] * STATISTICS: - synchronous/asynchronous API (& implementation) is not nice; => provide notification-based API @@ -58,9 +114,18 @@ away), in order in which they will likely be done: * ARM: - need to get rid of synchronous API for service starts (cause all kinds of problems) [=> eliminate for need to tell ARM about service starts most of the time!] [Safey] + - better tracking of which config changes actually need to cause process restarts by ARM. + - listen for requests to discover dependencies between services (and avoid + having to explicitly program start requests) + - better crash management (attach debugging support, capture and analyze + debug output, detect random vs. deterministic crashes) + - shutdown sequence? * HELLO: - need function to test "equivalency" of HELLOs; use in topology! * FS: + - GAP improvements: + + active reply route caching design & implementation of service, + gap extension! - gnunet-publish cannot be aborted using CTRL-C - gnunet-publish segfaults if file does not exist - on some systems, keyword search does not find locally published content @@ -112,6 +177,9 @@ away), in order in which they will likely be done: + test basic peer re-configure [Nate] + test topology creation [Nate] + test churn generation [Nate] +* TESTING: + - consider changing API for peer-group termination to + call continuation when done * WWW: - Get IPv6 hooked up - change DNS @@ -127,9 +195,14 @@ away), in order in which they will likely be done: + implement DV transport plugin [Nate & CG] + implement testcases [Nate] + implement performance tests [Nate] +* SETUP: + - design & implement new setup tool * GUIs to implement: - gtk (RC-pre1) + how to integrate scheduler with GTK event loop! +* HOSTLIST: + - 'server' uses 'GNUNET_PEERINFO_iterate', should probably switch to notification API + (for more instant / up-to-date hostlists at lower cost) [OPTIMIZATION] 0.9.0pre2: * Module features to implement: diff --git a/contrib/defaults.conf b/contrib/defaults.conf index 7e194ad08..f7750a0a8 100644 --- a/contrib/defaults.conf +++ b/contrib/defaults.conf @@ -136,7 +136,7 @@ ALLOW_SHUTDOWN = YES # quotas are in bytes per second now! TOTAL_QUOTA_IN = 65536 TOTAL_QUOTA_OUT = 65536 -DEBUG = YES +# DEBUG = YES # USERNAME = # MAXBUF = # TIMEOUT = diff --git a/src/core/core.h b/src/core/core.h index 45d138140..ea2925797 100644 --- a/src/core/core.h +++ b/src/core/core.h @@ -29,7 +29,7 @@ /** * General core debugging. */ -#define DEBUG_CORE GNUNET_YES +#define DEBUG_CORE GNUNET_NO /** * Debugging interaction core-clients. diff --git a/src/datastore/plugin_datastore_sqlite.c b/src/datastore/plugin_datastore_sqlite.c index aa90859d6..1d38f8326 100644 --- a/src/datastore/plugin_datastore_sqlite.c +++ b/src/datastore/plugin_datastore_sqlite.c @@ -29,7 +29,7 @@ #include "plugin_datastore.h" #include -#define DEBUG_SQLITE GNUNET_YES +#define DEBUG_SQLITE GNUNET_NO /** * After how many payload-changing operations diff --git a/src/fs/fs_test_lib_data.conf b/src/fs/fs_test_lib_data.conf index 20004ab16..fb1bdc533 100644 --- a/src/fs/fs_test_lib_data.conf +++ b/src/fs/fs_test_lib_data.conf @@ -42,7 +42,7 @@ PORT = 43470 HOSTNAME = localhost #TOTAL_QUOTA_IN = 3932160 #TOTAL_QUOTA_OUT = 3932160 -DEBUG = YES +#DEBUG = YES #PREFIX = valgrind --tool=memcheck --leak-check=yes #BINARY = /home/grothoff/bin/gnunet-service-core diff --git a/src/fs/gnunet-service-fs_indexing.c b/src/fs/gnunet-service-fs_indexing.c index 37ef08ba5..ca9a226a2 100644 --- a/src/fs/gnunet-service-fs_indexing.c +++ b/src/fs/gnunet-service-fs_indexing.c @@ -38,7 +38,7 @@ #include "gnunet-service-fs_indexing.h" #include "fs.h" -#define DEBUG_FS GNUNET_YES +#define DEBUG_FS GNUNET_NO /** * In-memory information about indexed files (also available diff --git a/src/fs/test_fs_download_data.conf b/src/fs/test_fs_download_data.conf index e4aa966d5..366a30844 100644 --- a/src/fs/test_fs_download_data.conf +++ b/src/fs/test_fs_download_data.conf @@ -19,7 +19,7 @@ HOSTNAME = localhost DEFAULTSERVICES = resolver datastore transport core fs [datastore] -DEBUG = YES +# DEBUG = YES [statistics] PORT = 42467 diff --git a/src/hostlist/hostlist-client.c b/src/hostlist/hostlist-client.c index 90d67ee75..1c0deb511 100644 --- a/src/hostlist/hostlist-client.c +++ b/src/hostlist/hostlist-client.c @@ -32,7 +32,7 @@ #include "gnunet_transport_service.h" #include -#define DEBUG_HOSTLIST_CLIENT GNUNET_YES +#define DEBUG_HOSTLIST_CLIENT GNUNET_NO /** * Number of connections that we must have to NOT download diff --git a/src/hostlist/hostlist-server.c b/src/hostlist/hostlist-server.c index 5c5571ff6..bd4203475 100644 --- a/src/hostlist/hostlist-server.c +++ b/src/hostlist/hostlist-server.c @@ -30,7 +30,7 @@ #include "gnunet_hello_lib.h" #include "gnunet_peerinfo_service.h" -#define DEBUG_HOSTLIST_SERVER GNUNET_YES +#define DEBUG_HOSTLIST_SERVER GNUNET_NO /** * How often should we recalculate our response to hostlist requests? diff --git a/src/peerinfo/peerinfo.h b/src/peerinfo/peerinfo.h index 4cb52fa7c..607aad4db 100644 --- a/src/peerinfo/peerinfo.h +++ b/src/peerinfo/peerinfo.h @@ -27,7 +27,7 @@ #include "gnunet_time_lib.h" #include "gnunet_peerinfo_service.h" -#define DEBUG_PEERINFO GNUNET_YES +#define DEBUG_PEERINFO GNUNET_NO /** * Add the given peer to the list. This message diff --git a/src/testing/test_testing_data_topology_clique.conf b/src/testing/test_testing_data_topology_clique.conf index 1a0dd9d7d..78e4a94e1 100644 --- a/src/testing/test_testing_data_topology_clique.conf +++ b/src/testing/test_testing_data_topology_clique.conf @@ -31,7 +31,7 @@ PORT = 2569 PORT = 2570 #PREFIX = xterm -e xterm -T CORE -e gdb --args #PREFIX = valgrind --tool=memcheck --log-file=logs%p -DEBUG = YES +#DEBUG = YES [testing] NUM_PEERS = 4 diff --git a/src/topology/gnunet-daemon-topology.c b/src/topology/gnunet-daemon-topology.c index 789efcd6e..b466dfbbe 100644 --- a/src/topology/gnunet-daemon-topology.c +++ b/src/topology/gnunet-daemon-topology.c @@ -34,7 +34,7 @@ #include "gnunet_util_lib.h" -#define DEBUG_TOPOLOGY GNUNET_YES +#define DEBUG_TOPOLOGY GNUNET_NO /** * For how long do we blacklist a peer after a failed connection diff --git a/src/topology/test_gnunet_service_topology_data.conf b/src/topology/test_gnunet_service_topology_data.conf index d429e6c2c..d920e9e88 100644 --- a/src/topology/test_gnunet_service_topology_data.conf +++ b/src/topology/test_gnunet_service_topology_data.conf @@ -8,7 +8,7 @@ PORT = 2664 [transport] PORT = 2665 PLUGINS = tcp -DEBUG = YES +# DEBUG = YES #PREFIX = xterm -e xterm -T transport -e gdb -x cmd --args #PREFIX = valgrind --tool=memcheck --log-file=logs%p @@ -27,10 +27,10 @@ PORT = 2669 [core] PORT = 2670 -DEBUG = YES +# DEBUG = YES [topology] -DEBUG = YES +# DEBUG = YES #PREFIX = valgrind --tool=memcheck [testing] diff --git a/src/transport/Makefile.am b/src/transport/Makefile.am index 0094c6d0e..7d9974c6b 100644 --- a/src/transport/Makefile.am +++ b/src/transport/Makefile.am @@ -123,7 +123,7 @@ test_transport_api_udp_SOURCES = \ test_transport_api_udp_LDADD = \ $(top_builddir)/src/transport/libgnunettransport.la \ $(top_builddir)/src/util/libgnunetutil.la - + test_transport_api_udp_nat_SOURCES = \ test_transport_api.c test_transport_api_udp_nat_LDADD = \ diff --git a/src/transport/plugin_transport_udp.c b/src/transport/plugin_transport_udp.c index 5bf555c90..094b207f4 100644 --- a/src/transport/plugin_transport_udp.c +++ b/src/transport/plugin_transport_udp.c @@ -690,14 +690,13 @@ udp_plugin_address_pretty_printer (void *cls, void * libgnunet_plugin_transport_udp_init (void *cls) { - unsigned long long mtu; - struct GNUNET_TRANSPORT_PluginEnvironment *env = cls; struct GNUNET_TRANSPORT_PluginFunctions *api; struct Plugin *plugin; struct GNUNET_SERVICE_Context *service; unsigned long long aport; unsigned long long bport; + unsigned long long mtu; service = GNUNET_SERVICE_start ("transport-udp", env->sched, env->cfg); if (service == NULL) diff --git a/src/transport/plugin_transport_udp_nat.c b/src/transport/plugin_transport_udp_nat.c index cbda5c6bc..3733130e2 100644 --- a/src/transport/plugin_transport_udp_nat.c +++ b/src/transport/plugin_transport_udp_nat.c @@ -54,7 +54,7 @@ #include "plugin_transport.h" #include "transport.h" -#define DEBUG_UDP_NAT GNUNET_YES +#define DEBUG_UDP_NAT GNUNET_NO /* * Transport cost to peer, always 1 for UDP (direct connection) diff --git a/src/transport/test_plugin_transport.c b/src/transport/test_plugin_transport.c index f9539430e..2248f007e 100644 --- a/src/transport/test_plugin_transport.c +++ b/src/transport/test_plugin_transport.c @@ -37,7 +37,7 @@ #include "plugin_transport.h" #include "transport.h" -#define VERBOSE GNUNET_YES +#define VERBOSE GNUNET_NO /** * How long until we give up on transmitting the message? diff --git a/src/transport/test_transport_api_udp_peer2.conf b/src/transport/test_transport_api_udp_peer2.conf index 14ae37f6b..e65633e1a 100644 --- a/src/transport/test_transport_api_udp_peer2.conf +++ b/src/transport/test_transport_api_udp_peer2.conf @@ -72,6 +72,7 @@ CONFIG = $DEFAULTCONFIG HOME = $SERVICEHOME HOSTNAME = localhost PORT = 22365 +# PREFIX = valgrind --track-origins=yes --leak-check=full --log-file=valgrind_udp_peer2.log [peerinfo] TRUST = $SERVICEHOME/data/credit/ diff --git a/src/transport/transport.h b/src/transport/transport.h index d66b87b35..acaa20a52 100644 --- a/src/transport/transport.h +++ b/src/transport/transport.h @@ -30,9 +30,9 @@ #include "gnunet_time_lib.h" #include "gnunet_transport_service.h" -#define DEBUG_TRANSPORT GNUNET_YES -#define DEBUG_TRANSPORT_TIMEOUT GNUNET_YES -#define DEBUG_TRANSPORT_DISCONNECT GNUNET_YES +#define DEBUG_TRANSPORT GNUNET_NO +#define DEBUG_TRANSPORT_TIMEOUT GNUNET_NO +#define DEBUG_TRANSPORT_DISCONNECT GNUNET_NO /** * For how long do we allow unused bandwidth -- 2.25.1