From 69710b875141c1c2cc2eacd4221356d6e457b8c6 Mon Sep 17 00:00:00 2001 From: Christian Grothoff Date: Sun, 10 Jan 2010 23:40:03 +0000 Subject: [PATCH] working on hostlist --- TODO | 9 + src/hostlist/Makefile.am | 4 +- src/hostlist/gnunet-daemon-hostlist.c | 10 +- src/hostlist/hostlist-client.c | 26 ++- src/hostlist/hostlist-server.c | 8 + src/hostlist/test_gnunet_daemon_hostlist.c | 161 ++++++++++++++---- .../test_gnunet_daemon_hostlist_peer1.conf | 6 + .../test_gnunet_daemon_hostlist_peer2.conf | 6 + 8 files changed, 189 insertions(+), 41 deletions(-) diff --git a/TODO b/TODO index 82bea4f77..4bb13870c 100644 --- a/TODO +++ b/TODO @@ -15,6 +15,8 @@ Urgent items (before announcing ng.gnunet.org): * TEST: - topology - hostlist + * is 'update_response' / 'finish_response' etc ever called? (hostlist-server) + * we still have serious shutdown problems in the testcase (on failure) * CORE: - soliciting traffic for clients that registered for it is not implemented (in the service, client API supports GNUNET_MESSAGE_TYPE_CORE_SOLICIT_TRAFFIC @@ -38,6 +40,13 @@ Urgent items (before announcing ng.gnunet.org): - load: need way to determine network load (up/down) - trust: need *fast* way to check/update trust in peers (async peerinfo would not be right) + - OS: existing waitpid call is not nice (not integratable with scheduler! fix this!) +* STATISTICS: + - synchronous/asynchronous API (& implementation) is not nice + - does not seem to work with timeouts (especially if service is not running) +* ARM: + - need to get rid of synchronous API for service starts (cause all kinds of problems) + [=> eliminate for need to tell ARM about service starts most of the time!] * Module features to implement: - advanced FS API parts + namespaces: fundamental namespace API diff --git a/src/hostlist/Makefile.am b/src/hostlist/Makefile.am index ac1409961..6aca797d3 100644 --- a/src/hostlist/Makefile.am +++ b/src/hostlist/Makefile.am @@ -27,12 +27,10 @@ gnunet_daemon_hostlist_LDADD = \ gnunet_daemon_hostlist_CPPFLAGS = \ @LIBCURL_CPPFLAGS@ - - check_PROGRAMS = \ test_gnunet_daemon_hostlist -# TESTS = $(check_PROGRAMS) $(check_SCRIPTS) +# TESTS = $(check_PROGRAMS) test_gnunet_daemon_hostlist_SOURCES = \ test_gnunet_daemon_hostlist.c diff --git a/src/hostlist/gnunet-daemon-hostlist.c b/src/hostlist/gnunet-daemon-hostlist.c index 2c4aca6d0..e8c9bc8c7 100644 --- a/src/hostlist/gnunet-daemon-hostlist.c +++ b/src/hostlist/gnunet-daemon-hostlist.c @@ -118,13 +118,21 @@ core_init (void *cls, static void cleaning_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { + if (bootstrapping) + { + GNUNET_HOSTLIST_client_stop (); + } + if (provide_hostlist) + { + GNUNET_HOSTLIST_server_stop (); + } + GNUNET_CORE_disconnect (core); if (stats != NULL) { GNUNET_STATISTICS_destroy (stats, GNUNET_NO); stats = NULL; } - GNUNET_CORE_disconnect (core); } diff --git a/src/hostlist/hostlist-client.c b/src/hostlist/hostlist-client.c index 9a3a31dfe..865ed2674 100644 --- a/src/hostlist/hostlist-client.c +++ b/src/hostlist/hostlist-client.c @@ -132,7 +132,9 @@ download_hostlist_processor (void *ptr, total = size * nmemb; if ( (total == 0) || (bogus_url) ) - return total; /* ok, no data or bogus data */ + { + return total; /* ok, no data or bogus data */ + } left = total; while (left > 0) { @@ -161,6 +163,9 @@ download_hostlist_processor (void *ptr, break; if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message*)msg) == msize) { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Received valid `%s' message from hostlist server.\n", + "HELLO"); GNUNET_TRANSPORT_offer_hello (transport, msg); } else @@ -318,6 +323,11 @@ multi_ready (void *cls, struct CURLMsg *msg; CURLMcode mret; + if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN)) + { + clean_up (); + return; + } if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_TIMEOUT)) { GNUNET_log (GNUNET_ERROR_TYPE_WARNING, @@ -365,6 +375,9 @@ multi_ready (void *cls, clean_up (); return; } + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Download of hostlist `%s' completed.\n"), + current_url); clean_up (); } @@ -521,6 +534,7 @@ static void check_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) { + if (connection_count < MIN_CONNECTIONS) download_hostlist (); else @@ -597,6 +611,8 @@ disconnect_handler (void *cls, static void primary_task (void *cls, int success) { + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Statistics request done, scheduling hostlist download\n"); schedule_hostlist_task (); } @@ -608,6 +624,9 @@ process_stat (void *cls, uint64_t value, int is_persistent) { + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Initial time between hostlist downloads is %llums\n"), + (unsigned long long) value); hostlist_delay.value = value; return GNUNET_OK; } @@ -656,8 +675,9 @@ GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c, void GNUNET_HOSTLIST_client_stop () { - GNUNET_SCHEDULER_cancel (sched, - current_task); + if (current_task != GNUNET_SCHEDULER_NO_TASK) + GNUNET_SCHEDULER_cancel (sched, + current_task); GNUNET_free_non_null (proxy); proxy = NULL; if (sched != NULL) diff --git a/src/hostlist/hostlist-server.c b/src/hostlist/hostlist-server.c index 38e144a9d..b15fdc408 100644 --- a/src/hostlist/hostlist-server.c +++ b/src/hostlist/hostlist-server.c @@ -144,6 +144,9 @@ update_response (void *cls, GNUNET_TIME_UNIT_MINUTES, &host_processor, results); + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Created new hostlist response with %u bytes\n"), + results->size); } @@ -174,6 +177,8 @@ access_handler_callback (void *cls, if (0 != strcmp (method, MHD_HTTP_METHOD_GET)) return MHD_NO; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Received request for our hostlist\n")); if (NULL == *con_cls) { (*con_cls) = &dummy; @@ -282,6 +287,9 @@ GNUNET_HOSTLIST_server_start (const struct GNUNET_CONFIGURATION_Handle *c, "HTTPPORT", &port)) return GNUNET_SYSERR; + GNUNET_log (GNUNET_ERROR_TYPE_INFO, + _("Hostlist service starts on port %llu\n"), + port); daemon_handle = MHD_start_daemon (MHD_USE_IPv6, (unsigned short) port, &accept_policy_callback, diff --git a/src/hostlist/test_gnunet_daemon_hostlist.c b/src/hostlist/test_gnunet_daemon_hostlist.c index ca7a2369a..66fbf35aa 100644 --- a/src/hostlist/test_gnunet_daemon_hostlist.c +++ b/src/hostlist/test_gnunet_daemon_hostlist.c @@ -40,12 +40,15 @@ static int ok; static struct GNUNET_SCHEDULER_Handle *sched; + +static GNUNET_SCHEDULER_TaskIdentifier timeout_task; struct PeerContext { struct GNUNET_CONFIGURATION_Handle *cfg; struct GNUNET_TRANSPORT_Handle *th; struct GNUNET_MessageHeader *hello; + struct GNUNET_ARM_Handle *arm; #if START_ARM pid_t arm_pid; #endif @@ -57,6 +60,52 @@ static struct PeerContext p2; +/** + * Timeout, give up. + */ +static void +timeout_error (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + timeout_task = GNUNET_SCHEDULER_NO_TASK; + GNUNET_log (GNUNET_ERROR_TYPE_ERROR, + "Timeout trying to connect peers, test failed.\n"); + GNUNET_TRANSPORT_disconnect (p1.th); + p1.th = NULL; + GNUNET_TRANSPORT_disconnect (p2.th); + p2.th = NULL; + GNUNET_SCHEDULER_shutdown (sched); +} + + +/** + * Function called to notify transport users that another + * peer connected to us. + * + * @param cls closure + * @param peer the peer that connected + * @param latency current latency of the connection + */ +static void +notify_connect (void *cls, + const struct GNUNET_PeerIdentity * peer, + struct GNUNET_TIME_Relative latency) +{ + if (peer == NULL) + return; + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Peers connected, shutting down.\n"); + GNUNET_assert (ok == 4); + ok = 0; + + GNUNET_SCHEDULER_cancel (sched, + timeout_task); + GNUNET_TRANSPORT_disconnect (p1.th); + p1.th = NULL; + GNUNET_TRANSPORT_disconnect (p2.th); + p2.th = NULL; +} + + static void process_hello (void *cls, struct GNUNET_TIME_Relative latency, @@ -65,22 +114,13 @@ process_hello (void *cls, { struct PeerContext *p = cls; - GNUNET_assert (peer != NULL); + if (message == NULL) + return; GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "Received (my) `%s' from transport service of `%4s'\n", - "HELLO", GNUNET_i2s (peer)); - GNUNET_assert (message != NULL); - p->hello = GNUNET_malloc (ntohs (message->size)); - memcpy (p->hello, message, ntohs (message->size)); - if ((p == &p1) && (p2.th != NULL)) - GNUNET_TRANSPORT_offer_hello (p2.th, message); - if ((p == &p2) && (p1.th != NULL)) - GNUNET_TRANSPORT_offer_hello (p1.th, message); - - if ((p == &p1) && (p2.hello != NULL)) - GNUNET_TRANSPORT_offer_hello (p1.th, p2.hello); - if ((p == &p2) && (p1.hello != NULL)) - GNUNET_TRANSPORT_offer_hello (p2.th, p1.hello); + "Received HELLO, starting hostlist service.\n"); + GNUNET_assert ( (ok >= 2) && (ok <= 3) ); + ok++; + GNUNET_ARM_start_services (p->cfg, sched, "hostlist", NULL); } @@ -98,12 +138,74 @@ setup_peer (struct PeerContext *p, const char *cfgname) #endif GNUNET_assert (GNUNET_OK == GNUNET_CONFIGURATION_load (p->cfg, cfgname)); GNUNET_ARM_start_services (p->cfg, sched, "core", NULL); - p->th = GNUNET_TRANSPORT_connect (sched, p->cfg, p, NULL, NULL, NULL); + p->th = GNUNET_TRANSPORT_connect (sched, p->cfg, p, NULL, + ¬ify_connect, NULL); GNUNET_assert (p->th != NULL); GNUNET_TRANSPORT_get_hello (p->th, TIMEOUT, &process_hello, p); } +static void +waitpid_task (void *cls, + const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + struct PeerContext *p = cls; + +#if START_ARM + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Killing ARM process.\n"); + if (0 != PLIBC_KILL (p->arm_pid, SIGTERM)) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); + if (GNUNET_OS_process_wait(p->arm_pid) != GNUNET_OK) + GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "ARM process %u stopped\n", p->arm_pid); +#endif + GNUNET_CONFIGURATION_destroy (p->cfg); +} + + +static void +stop_cb (void *cls, + int success) +{ + struct PeerContext *p = cls; + + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + success + ? "ARM stopped core service\n" + : "ARM failed to stop core service\n"); + GNUNET_ARM_disconnect (p->arm); + p->arm = NULL; + /* make sure this runs after all other tasks are done */ + GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_UNIT_SECONDS, + &waitpid_task, p); +} + + +static void +stop_arm (struct PeerContext *p) +{ + GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, + "Asking ARM to stop core service\n"); + p->arm = GNUNET_ARM_connect (p->cfg, sched, NULL); + GNUNET_ARM_stop_service (p->arm, "core", GNUNET_TIME_UNIT_SECONDS, + &stop_cb, p); +} + + +/** + * Try again to connect to transport service. + */ +static void +shutdown_task (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc) +{ + stop_arm (&p1); + stop_arm (&p2); +} + + static void run (void *cls, struct GNUNET_SCHEDULER_Handle *s, @@ -114,27 +216,20 @@ run (void *cls, GNUNET_assert (ok == 1); ok++; sched = s; + timeout_task = GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, + 15), + &timeout_error, + NULL); + GNUNET_SCHEDULER_add_delayed (sched, + GNUNET_TIME_UNIT_FOREVER_REL, + &shutdown_task, + NULL); setup_peer (&p1, "test_gnunet_daemon_hostlist_peer1.conf"); setup_peer (&p2, "test_gnunet_daemon_hostlist_peer2.conf"); } -static void -stop_arm (struct PeerContext *p) -{ - GNUNET_ARM_stop_services (p->cfg, sched, "core", NULL); -#if START_ARM - if (0 != PLIBC_KILL (p->arm_pid, SIGTERM)) - GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "kill"); - if (GNUNET_OS_process_wait(p->arm_pid) != GNUNET_OK) - GNUNET_log_strerror (GNUNET_ERROR_TYPE_WARNING, "waitpid"); - GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, - "ARM process %u stopped\n", p->arm_pid); -#endif - GNUNET_CONFIGURATION_destroy (p->cfg); -} - - static int check () { @@ -152,8 +247,6 @@ check () GNUNET_PROGRAM_run ((sizeof (argv) / sizeof (char *)) - 1, argv, "test-gnunet-daemon-hostlist", "nohelp", options, &run, &ok); - stop_arm (&p1); - stop_arm (&p2); return ok; } diff --git a/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf b/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf index 1228a98e8..6d15d4394 100644 --- a/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf +++ b/src/hostlist/test_gnunet_daemon_hostlist_peer1.conf @@ -12,6 +12,7 @@ PLUGINS = tcp [arm] PORT = 12966 +DEFAULTSERVICES = resolver transport core statistics [statistics] PORT = 12967 @@ -27,3 +28,8 @@ PORT = 12970 [testing] WEAKRANDOM = YES + +[hostlist] +HTTPPORT = 12980 +SERVERS = http://localhost:12981/ +OPTIONS = -b -p -L DEBUG \ No newline at end of file diff --git a/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf b/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf index f8c9d4ef1..3cf1ff202 100644 --- a/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf +++ b/src/hostlist/test_gnunet_daemon_hostlist_peer2.conf @@ -12,6 +12,7 @@ PLUGINS = tcp [arm] PORT = 22966 +DEFAULTSERVICES = resolver transport core statistics [statistics] PORT = 22967 @@ -27,3 +28,8 @@ PORT = 22970 [testing] WEAKRANDOM = YES + +[hostlist] +HTTPPORT = 12981 +SERVERS = http://localhost:12980/ +OPTIONS = -b -p -L DEBUG -- 2.25.1