2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009, 2010 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file hostlist/hostlist-client.c
23 * @brief hostlist support. Downloads HELLOs via HTTP.
24 * @author Christian Grothoff
25 * @author Matthias Wachs
29 #include "hostlist-client.h"
30 #include "gnunet_core_service.h"
31 #include "gnunet_hello_lib.h"
32 #include "gnunet_statistics_service.h"
33 #include "gnunet_transport_service.h"
34 #include "gnunet-daemon-hostlist.h"
35 #include <curl/curl.h>
36 #include "gnunet_common.h"
37 #include "gnunet_bio_lib.h"
41 * Number of connections that we must have to NOT download
44 #define MIN_CONNECTIONS 4
47 * Interval between two advertised hostlist tests
49 #define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
52 * A single hostlist obtained by hostlist advertisements
57 * previous entry, used to manage entries in a double linked list
59 struct Hostlist *prev;
62 * next entry, used to manage entries in a double linked list
64 struct Hostlist *next;
67 * URI where hostlist can be obtained
69 const char *hostlist_uri;
72 * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
73 * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
74 * intial value = HOSTLIST_INITIAL
75 * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
76 * increased every successful download by number of obtained HELLO messages
77 * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
82 * Time the hostlist advertisement was recieved and the entry was created
84 struct GNUNET_TIME_Absolute time_creation;
87 * Last time the hostlist was obtained
89 struct GNUNET_TIME_Absolute time_last_usage;
92 * Number of HELLO messages obtained during last download
97 * Number of times the hostlist was successfully obtained
107 static const struct GNUNET_CONFIGURATION_Handle *cfg;
112 static struct GNUNET_STATISTICS_Handle *stats;
117 static struct GNUNET_TRANSPORT_Handle *transport;
120 * Proxy that we are using (can be NULL).
125 * Number of bytes valid in 'download_buffer'.
127 static size_t download_pos;
130 * Current URL that we are using.
132 static char *current_url;
135 * Current CURL handle.
140 * Current multi-CURL handle.
145 * How many bytes did we download from the current hostlist URL?
147 static uint32_t stat_bytes_downloaded;
150 * Amount of time we wait between hostlist downloads.
152 static struct GNUNET_TIME_Relative hostlist_delay;
155 * ID of the task, checking if hostlist download should take plate
157 static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;
160 * ID of the task downloading the hostlist
162 static GNUNET_SCHEDULER_TaskIdentifier ti_download;
165 * ID of the task saving the hostlsit in a regular intervall
167 static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;
170 * ID of the task called to initiate a download
172 static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;
175 * ID of the task controlling the locking between two hostlist tests
177 static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;
180 * At what time MUST the current hostlist request be done?
182 static struct GNUNET_TIME_Absolute end_time;
185 * Head of the linked list used to store hostlists
187 static struct Hostlist *linked_list_head;
190 * Tail of the linked list used to store hostlists
192 static struct Hostlist *linked_list_tail;
195 * Current hostlist used for downloading
197 static struct Hostlist *current_hostlist;
200 * Size of the linke list used to store hostlists
202 static unsigned int linked_list_size;
205 * Head of the linked list used to store hostlists
207 static struct Hostlist *hostlist_to_test;
210 * Handle for our statistics GET operation.
212 static struct GNUNET_STATISTICS_GetHandle *sget;
215 * Set to GNUNET_YES if the current URL had some problems.
217 static int stat_bogus_url;
220 * Value controlling if a hostlist is tested at the moment
222 static int stat_testing_hostlist;
225 * Value controlling if a hostlist testing is allowed at the moment
227 static int stat_testing_allowed;
230 * Value controlling if a hostlist download is running at the moment
232 static int stat_download_in_progress;
235 * Value saying if a preconfigured bootstrap server is used
237 static unsigned int stat_use_bootstrap;
240 * Set if we are allowed to learn new hostlists and use them
242 static int stat_learning;
245 * Value saying if hostlist download was successful
247 static unsigned int stat_download_successful;
250 * Value saying how many valid HELLO messages were obtained during download
252 static unsigned int stat_hellos_obtained;
255 * Number of active connections (according to core service).
257 static unsigned int stat_connection_count;
261 * Process downloaded bits by calling callback on each HELLO.
263 * @param ptr buffer with downloaded data
264 * @param size size of a record
265 * @param nmemb number of records downloaded
267 * @return number of bytes that were processed (always size*nmemb)
270 callback_download (void *ptr, size_t size, size_t nmemb, void *ctx)
272 static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
273 const char *cbuf = ptr;
274 const struct GNUNET_MessageHeader *msg;
280 total = size * nmemb;
281 stat_bytes_downloaded += total;
282 if ((total == 0) || (stat_bogus_url))
284 return total; /* ok, no data or bogus data */
287 GNUNET_STATISTICS_update (stats,
289 ("# bytes downloaded from hostlist servers"),
290 (int64_t) total, GNUNET_NO);
292 while ((left > 0) || (download_pos > 0))
294 cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
295 memcpy (&download_buffer[download_pos], cbuf, cpy);
299 if (download_pos < sizeof (struct GNUNET_MessageHeader))
301 GNUNET_assert (0 == left);
304 msg = (const struct GNUNET_MessageHeader *) download_buffer;
305 msize = ntohs (msg->size);
306 if (msize < sizeof (struct GNUNET_MessageHeader))
308 GNUNET_STATISTICS_update (stats,
310 ("# invalid HELLOs downloaded from hostlist servers"),
312 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
313 _("Invalid `%s' message received from hostlist at `%s'\n"),
314 "HELLO", current_url);
315 stat_hellos_obtained++;
319 if (download_pos < msize)
321 GNUNET_assert (left == 0);
324 if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
326 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
327 "Received valid `%s' message from hostlist server.\n",
329 GNUNET_STATISTICS_update (stats,
331 ("# valid HELLOs downloaded from hostlist servers"),
333 stat_hellos_obtained++;
334 GNUNET_TRANSPORT_offer_hello (transport, msg, NULL, NULL);
338 GNUNET_STATISTICS_update (stats,
340 ("# invalid HELLOs downloaded from hostlist servers"),
342 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
343 _("Invalid `%s' message received from hostlist at `%s'\n"),
344 "HELLO", current_url);
345 stat_bogus_url = GNUNET_YES;
346 stat_hellos_obtained++;
349 memmove (download_buffer, &download_buffer[msize], download_pos - msize);
350 download_pos -= msize;
357 * Obtain a hostlist URL that we should use.
359 * @return NULL if there is no URL available
362 get_bootstrap_server ()
370 GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
373 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
374 "hostlist", "SERVERS");
379 if (strlen (servers) > 0)
382 pos = strlen (servers) - 1;
385 if (servers[pos] == ' ')
392 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
393 "hostlist", "SERVERS");
394 GNUNET_free (servers);
398 urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
399 pos = strlen (servers) - 1;
402 if (servers[pos] == ' ')
414 ret = GNUNET_strdup (&servers[pos]);
415 GNUNET_free (servers);
420 * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
421 * @return uri to use, NULL if there is no URL available
427 unsigned int counter;
428 struct Hostlist *pos;
430 if (GNUNET_NO == stat_learning)
432 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
433 "Using preconfigured bootstrap server\n");
434 current_hostlist = NULL;
435 return get_bootstrap_server ();
438 if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
440 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
441 "Testing new advertised hostlist if it is obtainable\n");
442 current_hostlist = hostlist_to_test;
443 return GNUNET_strdup (hostlist_to_test->hostlist_uri);
446 if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
448 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
449 "Using preconfigured bootstrap server\n");
450 current_hostlist = NULL;
451 return get_bootstrap_server ();
454 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
456 pos = linked_list_head;
457 while (counter < index)
462 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
464 current_hostlist = pos;
465 return GNUNET_strdup (pos->hostlist_uri);
469 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)
473 * Method to save hostlist to a file during hostlist client shutdown
474 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
477 save_hostlist_file (int shutdown);
481 * add val2 to val1 with overflow check
482 * @param val1 value 1
483 * @param val2 value 2
487 checked_add (uint64_t val1, uint64_t val2)
489 static uint64_t temp;
490 static uint64_t maxv;
503 * Subtract val2 from val1 with underflow check
504 * @param val1 value 1
505 * @param val2 value 2
509 checked_sub (uint64_t val1, uint64_t val2)
513 return (val1 - val2);
518 * Method to check if a URI is in hostlist linked list
519 * @param uri uri to check
520 * @return GNUNET_YES if existing in linked list, GNUNET_NO if not
523 linked_list_contains (const char *uri)
525 struct Hostlist *pos;
527 pos = linked_list_head;
530 if (0 == strcmp (pos->hostlist_uri, uri))
539 * Method returning the hostlist element with the lowest quality in the datastore
540 * @return hostlist with lowest quality
542 static struct Hostlist *
543 linked_list_get_lowest_quality ()
545 struct Hostlist *pos;
546 struct Hostlist *lowest;
548 if (linked_list_size == 0)
550 lowest = linked_list_head;
551 pos = linked_list_head->next;
554 if (pos->quality < lowest->quality)
563 * Method to insert a hostlist into the datastore. If datastore
564 * contains maximum number of elements, the elements with lowest
565 * quality is dismissed
570 struct Hostlist *lowest_quality;
572 if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
574 /* No free entries available, replace existing entry */
575 lowest_quality = linked_list_get_lowest_quality ();
576 GNUNET_assert (lowest_quality != NULL);
577 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
578 "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
579 lowest_quality->hostlist_uri,
580 (unsigned long long) lowest_quality->quality);
581 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
584 GNUNET_free (lowest_quality);
586 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
589 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
590 linked_list_size, GNUNET_NO);
591 stat_testing_hostlist = GNUNET_NO;
596 * Method updating hostlist statistics
603 if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
604 ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
606 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
607 "Updating hostlist statics for URI `%s'\n",
608 current_hostlist->hostlist_uri);
609 current_hostlist->hello_count = stat_hellos_obtained;
610 current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
611 current_hostlist->quality =
612 checked_add (current_hostlist->quality,
613 (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
614 if (GNUNET_YES == stat_download_successful)
616 current_hostlist->times_used++;
617 current_hostlist->quality =
618 checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
619 GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
620 current_hostlist->hostlist_uri);
622 GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
626 current_hostlist->quality =
627 checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
629 current_hostlist = NULL;
630 /* Alternating the usage of preconfigured and learned hostlists */
632 if (stat_testing_hostlist == GNUNET_YES)
635 if (GNUNET_YES == stat_learning)
637 if (stat_use_bootstrap == GNUNET_YES)
638 stat_use_bootstrap = GNUNET_NO;
640 stat_use_bootstrap = GNUNET_YES;
643 stat_use_bootstrap = GNUNET_YES;
648 * Clean up the state from the task that downloaded the
649 * hostlist and schedule the next task.
656 if ((stat_testing_hostlist == GNUNET_YES) &&
657 (GNUNET_NO == stat_download_successful) && (NULL != hostlist_to_test))
659 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
661 ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
662 hostlist_to_test->hostlist_uri);
665 if (stat_testing_hostlist == GNUNET_YES)
667 stat_testing_hostlist = GNUNET_NO;
669 if (NULL != hostlist_to_test)
671 GNUNET_free (hostlist_to_test);
672 hostlist_to_test = NULL;
677 mret = curl_multi_remove_handle (multi, curl);
678 if (mret != CURLM_OK)
680 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
681 "curl_multi_remove_handle", __FILE__, __LINE__,
682 curl_multi_strerror (mret));
684 mret = curl_multi_cleanup (multi);
685 if (mret != CURLM_OK)
686 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
687 "curl_multi_cleanup", __FILE__, __LINE__,
688 curl_multi_strerror (mret));
693 curl_easy_cleanup (curl);
696 GNUNET_free_non_null (current_url);
698 stat_bytes_downloaded = 0;
699 stat_download_in_progress = GNUNET_NO;
704 * Task that is run when we are ready to receive more data from the hostlist
707 * @param cls closure, unused
708 * @param tc task context, unused
711 task_download (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
715 * Ask CURL for the select set and then schedule the
716 * receiving task with the scheduler.
726 struct GNUNET_NETWORK_FDSet *grs;
727 struct GNUNET_NETWORK_FDSet *gws;
729 struct GNUNET_TIME_Relative rtime;
735 mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
736 if (mret != CURLM_OK)
738 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
739 "curl_multi_fdset", __FILE__, __LINE__,
740 curl_multi_strerror (mret));
744 mret = curl_multi_timeout (multi, &timeout);
745 if (mret != CURLM_OK)
747 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
748 "curl_multi_timeout", __FILE__, __LINE__,
749 curl_multi_strerror (mret));
754 GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
755 GNUNET_TIME_relative_multiply
756 (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
757 grs = GNUNET_NETWORK_fdset_create ();
758 gws = GNUNET_NETWORK_fdset_create ();
759 GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
760 GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
761 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
762 "Scheduling task for hostlist download using cURL\n");
764 GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
766 &task_download, multi);
767 GNUNET_NETWORK_fdset_destroy (gws);
768 GNUNET_NETWORK_fdset_destroy (grs);
773 * Task that is run when we are ready to receive more data from the hostlist
776 * @param cls closure, unused
777 * @param tc task context, unused
780 task_download (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
786 ti_download = GNUNET_SCHEDULER_NO_TASK;
787 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
789 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
790 "Shutdown requested while trying to download hostlist from `%s'\n",
796 if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
798 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
799 _("Timeout trying to download hostlist from `%s'\n"),
805 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
806 "Ready for processing hostlist client request\n");
810 if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
812 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
813 _("Download limit of %u bytes exceeded, stopping download\n"),
814 MAX_BYTES_PER_HOSTLISTS);
818 mret = curl_multi_perform (multi, &running);
823 msg = curl_multi_info_read (multi, &running);
824 GNUNET_break (msg != NULL);
830 if ((msg->data.result != CURLE_OK) &&
831 (msg->data.result != CURLE_GOT_NOTHING))
832 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
833 _("Download of hostlist from `%s' failed: `%s'\n"),
835 curl_easy_strerror (msg->data.result));
838 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
839 _("Download of hostlist `%s' completed.\n"),
841 stat_download_successful = GNUNET_YES;
843 if (GNUNET_YES == stat_testing_hostlist)
845 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
847 ("Adding successfully tested hostlist `%s' datastore.\n"),
850 hostlist_to_test = NULL;
851 stat_testing_hostlist = GNUNET_NO;
861 while ((running > 0));
864 while (mret == CURLM_CALL_MULTI_PERFORM);
866 if (mret != CURLM_OK)
868 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
869 "curl_multi_perform", __FILE__, __LINE__,
870 curl_multi_strerror (mret));
878 * Main function that will download a hostlist and process its
888 current_url = download_get_url ();
889 if (current_url == NULL)
891 curl = curl_easy_init ();
899 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
900 _("Bootstrapping using hostlist at `%s'.\n"), current_url);
902 stat_download_in_progress = GNUNET_YES;
903 stat_download_successful = GNUNET_NO;
904 stat_hellos_obtained = 0;
905 stat_bytes_downloaded = 0;
907 GNUNET_STATISTICS_update (stats,
908 gettext_noop ("# hostlist downloads initiated"), 1,
911 CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
914 CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
920 CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
926 CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
927 CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
928 CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
929 CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
930 /* no need to abort if the above failed */
931 CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
937 CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
939 CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
941 CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
942 if (0 == strncmp (current_url, "http", 4))
943 CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
944 CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
945 CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
947 /* this should no longer be needed; we're now single-threaded! */
948 CURL_EASY_SETOPT (curl, CURLOPT_NOSIGNAL, 1);
950 multi = curl_multi_init ();
957 mret = curl_multi_add_handle (multi, curl);
958 if (mret != CURLM_OK)
960 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
961 "curl_multi_add_handle", __FILE__, __LINE__,
962 curl_multi_strerror (mret));
963 mret = curl_multi_cleanup (multi);
964 if (mret != CURLM_OK)
965 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
966 "curl_multi_cleanup", __FILE__, __LINE__,
967 curl_multi_strerror (mret));
972 end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
978 task_download_dispatcher (void *cls,
979 const struct GNUNET_SCHEDULER_TaskContext *tc)
981 ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
982 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
984 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
985 if (GNUNET_NO == stat_download_in_progress)
987 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
988 download_hostlist ();
992 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
993 "Download in progess, have to wait...\n");
994 ti_download_dispatcher_task =
995 GNUNET_SCHEDULER_add_delayed (WAITING_INTERVALL,
996 &task_download_dispatcher, NULL);
1002 * Task that checks if we should try to download a hostlist.
1003 * If so, we initiate the download, otherwise we schedule
1004 * this task again for a later time.
1007 task_check (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
1010 struct GNUNET_TIME_Relative delay;
1012 ti_check_download = GNUNET_SCHEDULER_NO_TASK;
1013 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1017 curl_global_cleanup ();
1018 return; /* in shutdown */
1020 if ( (stat_connection_count < MIN_CONNECTIONS) &&
1021 (GNUNET_SCHEDULER_NO_TASK == ti_download_dispatcher_task) )
1022 ti_download_dispatcher_task =
1023 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1025 delay = hostlist_delay;
1026 if (0 == hostlist_delay.rel_value_us)
1027 hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
1029 hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
1030 if (hostlist_delay.rel_value_us >
1031 GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
1033 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
1034 (1 + stat_connection_count));
1035 GNUNET_STATISTICS_set (stats,
1037 ("# milliseconds between hostlist downloads"),
1038 hostlist_delay.rel_value_us / 1000LL,
1042 delay = GNUNET_TIME_UNIT_ZERO;
1045 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1046 _("Have %u/%u connections. Will consider downloading hostlist in %s\n"),
1047 stat_connection_count, MIN_CONNECTIONS,
1048 GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
1049 ti_check_download = GNUNET_SCHEDULER_add_delayed (delay, &task_check, NULL);
1054 * This tasks sets hostlist testing to allowed after intervall between to testings is reached
1056 * @param cls closure
1057 * @param tc TaskContext
1060 task_testing_intervall_reset (void *cls,
1061 const struct GNUNET_SCHEDULER_TaskContext *tc)
1063 ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
1064 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1066 stat_testing_allowed = GNUNET_OK;
1067 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1068 "Testing new hostlist advertisements is allowed again\n");
1073 * Task that writes hostlist entries to a file on a regular base
1075 * @param cls closure
1076 * @param tc TaskContext
1079 task_hostlist_saving (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
1081 ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
1082 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1084 save_hostlist_file (GNUNET_NO);
1086 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1087 "Hostlists will be saved to file again in %s\n",
1088 GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVALL, GNUNET_YES));
1090 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL, &task_hostlist_saving,
1096 * Method called whenever a given peer connects.
1098 * @param cls closure
1099 * @param peer peer identity this notification is about
1102 handler_connect (void *cls, const struct GNUNET_PeerIdentity *peer)
1104 GNUNET_assert (stat_connection_count < UINT_MAX);
1105 stat_connection_count++;
1106 GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), 1,
1112 * Method called whenever a given peer disconnects.
1114 * @param cls closure
1115 * @param peer peer identity this notification is about
1118 handler_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
1120 GNUNET_assert (stat_connection_count > 0);
1121 stat_connection_count--;
1122 GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), -1,
1128 * Method called whenever an advertisement message arrives.
1130 * @param cls closure (always NULL)
1131 * @param peer the peer sending the message
1132 * @param message the actual message
1133 * @return GNUNET_OK to keep the connection open,
1134 * GNUNET_SYSERR to close it (signal serious error)
1137 handler_advertisement (void *cls, const struct GNUNET_PeerIdentity *peer,
1138 const struct GNUNET_MessageHeader *message)
1142 const struct GNUNET_MessageHeader *incoming;
1144 struct Hostlist *hostlist;
1146 GNUNET_assert (ntohs (message->type) ==
1147 GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
1148 size = ntohs (message->size);
1149 if (size <= sizeof (struct GNUNET_MessageHeader))
1151 GNUNET_break_op (0);
1152 return GNUNET_SYSERR;
1154 incoming = (const struct GNUNET_MessageHeader *) message;
1155 uri = (const char *) &incoming[1];
1156 uri_size = size - sizeof (struct GNUNET_MessageHeader);
1157 if (uri[uri_size - 1] != '\0')
1159 GNUNET_break_op (0);
1160 return GNUNET_SYSERR;
1162 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1163 "Hostlist client recieved advertisement from `%s' containing URI `%s'\n",
1164 GNUNET_i2s (peer), uri);
1165 if (GNUNET_NO != linked_list_contains (uri))
1167 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "URI `%s' is already known\n", uri);
1171 if (GNUNET_NO == stat_testing_allowed)
1173 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1174 "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
1175 return GNUNET_SYSERR;
1177 if (GNUNET_YES == stat_testing_hostlist)
1179 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1180 "Currently not accepting new advertisements: we are already testing a hostlist\n");
1181 return GNUNET_SYSERR;
1184 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
1185 hostlist->hostlist_uri = (const char *) &hostlist[1];
1186 memcpy (&hostlist[1], uri, uri_size);
1187 hostlist->time_creation = GNUNET_TIME_absolute_get ();
1188 hostlist->quality = HOSTLIST_INITIAL;
1189 hostlist_to_test = hostlist;
1191 stat_testing_hostlist = GNUNET_YES;
1192 stat_testing_allowed = GNUNET_NO;
1193 ti_testing_intervall_task =
1194 GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
1195 &task_testing_intervall_reset, NULL);
1197 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1198 "Testing new hostlist advertisements is locked for the next %s\n",
1199 GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
1202 ti_download_dispatcher_task =
1203 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1210 * Continuation called by the statistics code once
1211 * we go the stat. Initiates hostlist download scheduling.
1213 * @param cls closure
1214 * @param success GNUNET_OK if statistics were
1215 * successfully obtained, GNUNET_SYSERR if not.
1218 primary_task (void *cls, int success)
1221 GNUNET_assert (stats != NULL);
1222 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1223 "Statistics request done, scheduling hostlist download\n");
1224 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
1229 * We've received the previous delay value from statistics. Remember it.
1231 * @param cls NULL, unused
1232 * @param subsystem should be "hostlist", unused
1233 * @param name will be "milliseconds between hostlist downloads", unused
1234 * @param value previous delay value, in milliseconds (!)
1235 * @param is_persistent unused, will be GNUNET_YES
1238 process_stat (void *cls, const char *subsystem, const char *name,
1239 uint64_t value, int is_persistent)
1241 hostlist_delay.rel_value_us = value * 1000LL;
1242 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1243 "Initial time between hostlist downloads is %s\n",
1244 GNUNET_STRINGS_relative_time_to_string (hostlist_delay, GNUNET_YES));
1250 * Method to load persistent hostlist file during hostlist client startup
1253 load_hostlist_file ()
1258 struct Hostlist *hostlist;
1261 uint32_t times_used;
1262 uint32_t hellos_returned;
1269 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1272 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1273 "hostlist", "HOSTLISTFILE");
1277 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1278 _("Loading saved hostlist entries from file `%s' \n"), filename);
1279 if (GNUNET_NO == GNUNET_DISK_file_test (filename))
1281 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1282 _("Hostlist file `%s' does not exist\n"), filename);
1283 GNUNET_free (filename);
1287 struct GNUNET_BIO_ReadHandle *rh = GNUNET_BIO_read_open (filename);
1291 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1293 ("Could not open file `%s' for reading to load hostlists: %s\n"),
1294 filename, STRERROR (errno));
1295 GNUNET_free (filename);
1300 while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
1301 (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, ×_used))
1302 && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
1303 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
1304 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
1305 (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
1307 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
1308 hostlist->hello_count = hellos_returned;
1309 hostlist->hostlist_uri = (const char *) &hostlist[1];
1310 memcpy (&hostlist[1], uri, strlen (uri) + 1);
1311 hostlist->quality = quality;
1312 hostlist->time_creation.abs_value_us = created;
1313 hostlist->time_last_usage.abs_value_us = last_used;
1314 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
1316 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1317 "Added hostlist entry eith URI `%s' \n",
1318 hostlist->hostlist_uri);
1322 if (counter >= MAX_NUMBER_HOSTLISTS)
1326 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%u hostlist URIs loaded from file\n"),
1328 GNUNET_STATISTICS_set (stats, gettext_noop ("# hostlist URIs read from file"),
1329 counter, GNUNET_YES);
1330 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
1331 linked_list_size, GNUNET_NO);
1333 GNUNET_free_non_null (uri);
1335 (void) GNUNET_BIO_read_close (rh, &emsg);
1338 GNUNET_free (filename);
1343 * Method to save persistent hostlist file during hostlist client shutdown
1344 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
1347 save_hostlist_file (int shutdown)
1350 struct Hostlist *pos;
1351 struct GNUNET_BIO_WriteHandle *wh;
1356 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1359 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1360 "hostlist", "HOSTLISTFILE");
1363 if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
1365 GNUNET_free (filename);
1368 wh = GNUNET_BIO_write_open (filename);
1371 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1373 ("Could not open file `%s' for writing to save hostlists: %s\n"),
1374 filename, STRERROR (errno));
1375 GNUNET_free (filename);
1378 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Writing %u hostlist URIs to `%s'\n"),
1379 linked_list_size, filename);
1380 /* add code to write hostlists to file using bio */
1383 while (NULL != (pos = linked_list_head))
1385 if (GNUNET_YES == shutdown)
1387 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
1390 if (GNUNET_YES == ok)
1392 if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
1393 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
1394 (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
1396 GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
1398 GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
1399 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
1401 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1402 _("Error writing hostlist URIs to file `%s'\n"), filename);
1407 if (GNUNET_YES == shutdown)
1410 if (counter >= MAX_NUMBER_HOSTLISTS)
1413 GNUNET_STATISTICS_set (stats,
1414 gettext_noop ("# hostlist URIs written to file"),
1415 counter, GNUNET_YES);
1417 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
1418 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1419 _("Error writing hostlist URIs to file `%s'\n"), filename);
1420 GNUNET_free (filename);
1425 * Start downloading hostlists from hostlist servers as necessary.
1428 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
1429 struct GNUNET_STATISTICS_Handle *st,
1430 GNUNET_CORE_ConnectEventHandler *ch,
1431 GNUNET_CORE_DisconnectEventHandler *dh,
1432 GNUNET_CORE_MessageCallback *msgh, int learn)
1437 GNUNET_assert (NULL != st);
1438 if (0 != curl_global_init (CURL_GLOBAL_WIN32))
1441 return GNUNET_SYSERR;
1443 transport = GNUNET_TRANSPORT_connect (c, NULL, NULL, NULL, NULL, NULL);
1444 if (NULL == transport)
1446 curl_global_cleanup ();
1447 return GNUNET_SYSERR;
1452 GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "HTTP-PROXY",
1455 stat_learning = learn;
1456 *ch = &handler_connect;
1457 *dh = &handler_disconnect;
1458 linked_list_head = NULL;
1459 linked_list_tail = NULL;
1460 stat_use_bootstrap = GNUNET_YES;
1461 stat_testing_hostlist = GNUNET_NO;
1462 stat_testing_allowed = GNUNET_YES;
1464 if (GNUNET_YES == stat_learning)
1466 *msgh = &handler_advertisement;
1467 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1468 _("Learning is enabled on this peer\n"));
1469 load_hostlist_file ();
1470 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1471 "Hostlists will be saved to file again in %s\n",
1472 GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVALL, GNUNET_YES));
1474 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL, &task_hostlist_saving,
1479 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1480 _("Learning is not enabled on this peer\n"));
1483 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST",
1484 "HOSTLISTFILE", &filename))
1486 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1488 result = remove (filename);
1490 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1492 ("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
1495 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1496 _("Hostlist file `%s' could not be removed\n"), filename);
1499 GNUNET_free (filename);
1501 sget = GNUNET_STATISTICS_get (stats, "hostlist",
1503 ("# milliseconds between hostlist downloads"),
1504 GNUNET_TIME_UNIT_MINUTES, &primary_task, &process_stat,
1511 * Stop downloading hostlists from hostlist servers as necessary.
1514 GNUNET_HOSTLIST_client_stop ()
1516 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Hostlist client shutdown\n");
1519 GNUNET_STATISTICS_get_cancel (sget);
1523 if (GNUNET_YES == stat_learning)
1524 save_hostlist_file (GNUNET_YES);
1525 if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
1527 GNUNET_SCHEDULER_cancel (ti_saving_task);
1528 ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
1531 if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
1533 GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
1534 ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
1536 if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
1538 GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
1539 ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
1541 if (ti_download != GNUNET_SCHEDULER_NO_TASK)
1543 GNUNET_SCHEDULER_cancel (ti_download);
1544 ti_download = GNUNET_SCHEDULER_NO_TASK;
1546 if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
1548 GNUNET_SCHEDULER_cancel (ti_check_download);
1549 ti_check_download = GNUNET_SCHEDULER_NO_TASK;
1550 curl_global_cleanup ();
1552 if (transport != NULL)
1554 GNUNET_TRANSPORT_disconnect (transport);
1557 GNUNET_free_non_null (proxy);
1562 /* end of hostlist-client.c */