2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009, 2010 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file hostlist/hostlist-client.c
23 * @brief hostlist support. Downloads HELLOs via HTTP.
24 * @author Christian Grothoff
25 * @author Matthias Wachs
29 #include "hostlist-client.h"
30 #include "gnunet_core_service.h"
31 #include "gnunet_hello_lib.h"
32 #include "gnunet_statistics_service.h"
33 #include "gnunet_transport_service.h"
34 #include "gnunet-daemon-hostlist.h"
35 #include <curl/curl.h>
36 #include "gnunet_util_lib.h"
40 * Number of connections that we must have to NOT download
43 #define MIN_CONNECTIONS 4
46 * Interval between two advertised hostlist tests
48 #define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 5)
51 * A single hostlist obtained by hostlist advertisements
56 * previous entry, used to manage entries in a double linked list
58 struct Hostlist *prev;
61 * next entry, used to manage entries in a double linked list
63 struct Hostlist *next;
66 * URI where hostlist can be obtained
68 const char *hostlist_uri;
71 * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
72 * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
73 * intial value = HOSTLIST_INITIAL
74 * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
75 * increased every successful download by number of obtained HELLO messages
76 * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
81 * Time the hostlist advertisement was recieved and the entry was created
83 struct GNUNET_TIME_Absolute time_creation;
86 * Last time the hostlist was obtained
88 struct GNUNET_TIME_Absolute time_last_usage;
91 * Number of HELLO messages obtained during last download
96 * Number of times the hostlist was successfully obtained
106 static const struct GNUNET_CONFIGURATION_Handle *cfg;
111 static struct GNUNET_STATISTICS_Handle *stats;
116 static struct GNUNET_TRANSPORT_Handle *transport;
119 * Proxy hostname or ip we are using (can be NULL).
124 * Proxy username we are using (can be NULL).
126 static char *proxy_username;
129 * Proxy password we are using (can be NULL).
131 static char *proxy_password;
134 * Proxy type we are using (can be NULL).
136 static curl_proxytype proxy_type;
139 * Number of bytes valid in 'download_buffer'.
141 static size_t download_pos;
144 * Current URL that we are using.
146 static char *current_url;
149 * Current CURL handle.
154 * Current multi-CURL handle.
159 * How many bytes did we download from the current hostlist URL?
161 static uint32_t stat_bytes_downloaded;
164 * Amount of time we wait between hostlist downloads.
166 static struct GNUNET_TIME_Relative hostlist_delay;
169 * ID of the task, checking if hostlist download should take plate
171 static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;
174 * ID of the task downloading the hostlist
176 static GNUNET_SCHEDULER_TaskIdentifier ti_download;
179 * ID of the task saving the hostlsit in a regular intervall
181 static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;
184 * ID of the task called to initiate a download
186 static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;
189 * ID of the task controlling the locking between two hostlist tests
191 static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;
194 * At what time MUST the current hostlist request be done?
196 static struct GNUNET_TIME_Absolute end_time;
199 * Head of the linked list used to store hostlists
201 static struct Hostlist *linked_list_head;
204 * Tail of the linked list used to store hostlists
206 static struct Hostlist *linked_list_tail;
209 * Current hostlist used for downloading
211 static struct Hostlist *current_hostlist;
214 * Size of the linke list used to store hostlists
216 static unsigned int linked_list_size;
219 * Head of the linked list used to store hostlists
221 static struct Hostlist *hostlist_to_test;
224 * Handle for our statistics GET operation.
226 static struct GNUNET_STATISTICS_GetHandle *sget;
229 * Set to GNUNET_YES if the current URL had some problems.
231 static int stat_bogus_url;
234 * Value controlling if a hostlist is tested at the moment
236 static int stat_testing_hostlist;
239 * Value controlling if a hostlist testing is allowed at the moment
241 static int stat_testing_allowed;
244 * Value controlling if a hostlist download is running at the moment
246 static int stat_download_in_progress;
249 * Value saying if a preconfigured bootstrap server is used
251 static unsigned int stat_use_bootstrap;
254 * Set if we are allowed to learn new hostlists and use them
256 static int stat_learning;
259 * Value saying if hostlist download was successful
261 static unsigned int stat_download_successful;
264 * Value saying how many valid HELLO messages were obtained during download
266 static unsigned int stat_hellos_obtained;
269 * Number of active connections (according to core service).
271 static unsigned int stat_connection_count;
275 * Process downloaded bits by calling callback on each HELLO.
277 * @param ptr buffer with downloaded data
278 * @param size size of a record
279 * @param nmemb number of records downloaded
281 * @return number of bytes that were processed (always size*nmemb)
284 callback_download (void *ptr, size_t size, size_t nmemb, void *ctx)
286 static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
287 const char *cbuf = ptr;
288 const struct GNUNET_MessageHeader *msg;
294 total = size * nmemb;
295 stat_bytes_downloaded += total;
296 if ((total == 0) || (stat_bogus_url))
298 return total; /* ok, no data or bogus data */
301 GNUNET_STATISTICS_update (stats,
303 ("# bytes downloaded from hostlist servers"),
304 (int64_t) total, GNUNET_NO);
306 while ((left > 0) || (download_pos > 0))
308 cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
309 memcpy (&download_buffer[download_pos], cbuf, cpy);
313 if (download_pos < sizeof (struct GNUNET_MessageHeader))
315 GNUNET_assert (0 == left);
318 msg = (const struct GNUNET_MessageHeader *) download_buffer;
319 msize = ntohs (msg->size);
320 if (msize < sizeof (struct GNUNET_MessageHeader))
322 GNUNET_STATISTICS_update (stats,
324 ("# invalid HELLOs downloaded from hostlist servers"),
326 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
327 _("Invalid `%s' message received from hostlist at `%s'\n"),
328 "HELLO", current_url);
329 stat_hellos_obtained++;
333 if (download_pos < msize)
335 GNUNET_assert (left == 0);
338 if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
340 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
341 "Received valid `%s' message from hostlist server.\n",
343 GNUNET_STATISTICS_update (stats,
345 ("# valid HELLOs downloaded from hostlist servers"),
347 stat_hellos_obtained++;
348 GNUNET_TRANSPORT_offer_hello (transport, msg, NULL, NULL);
352 GNUNET_STATISTICS_update (stats,
354 ("# invalid HELLOs downloaded from hostlist servers"),
356 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
357 _("Invalid `%s' message received from hostlist at `%s'\n"),
358 "HELLO", current_url);
359 stat_bogus_url = GNUNET_YES;
360 stat_hellos_obtained++;
363 memmove (download_buffer, &download_buffer[msize], download_pos - msize);
364 download_pos -= msize;
371 * Obtain a hostlist URL that we should use.
373 * @return NULL if there is no URL available
376 get_bootstrap_server ()
384 GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
387 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
388 "hostlist", "SERVERS");
393 if (strlen (servers) > 0)
396 pos = strlen (servers) - 1;
399 if (servers[pos] == ' ')
406 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
407 "hostlist", "SERVERS");
408 GNUNET_free (servers);
412 urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
413 pos = strlen (servers) - 1;
416 if (servers[pos] == ' ')
428 ret = GNUNET_strdup (&servers[pos]);
429 GNUNET_free (servers);
435 * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
436 * @return uri to use, NULL if there is no URL available
442 unsigned int counter;
443 struct Hostlist *pos;
445 if (GNUNET_NO == stat_learning)
447 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
448 "Using preconfigured bootstrap server\n");
449 current_hostlist = NULL;
450 return get_bootstrap_server ();
453 if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
455 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
456 "Testing new advertised hostlist if it is obtainable\n");
457 current_hostlist = hostlist_to_test;
458 return GNUNET_strdup (hostlist_to_test->hostlist_uri);
461 if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
463 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
464 "Using preconfigured bootstrap server\n");
465 current_hostlist = NULL;
466 return get_bootstrap_server ();
469 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
471 pos = linked_list_head;
472 while (counter < index)
477 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
479 current_hostlist = pos;
480 return GNUNET_strdup (pos->hostlist_uri);
484 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)
488 * Method to save hostlist to a file during hostlist client shutdown
489 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
492 save_hostlist_file (int shutdown);
496 * add val2 to val1 with overflow check
498 * @param val1 value 1
499 * @param val2 value 2
503 checked_add (uint64_t val1, uint64_t val2)
505 static uint64_t temp;
506 static uint64_t maxv;
519 * Subtract val2 from val1 with underflow check
521 * @param val1 value 1
522 * @param val2 value 2
526 checked_sub (uint64_t val1, uint64_t val2)
530 return (val1 - val2);
535 * Method to check if a URI is in hostlist linked list
537 * @param uri uri to check
538 * @return #GNUNET_YES if existing in linked list, #GNUNET_NO if not
541 linked_list_contains (const char *uri)
543 struct Hostlist *pos;
545 pos = linked_list_head;
548 if (0 == strcmp (pos->hostlist_uri, uri))
557 * Method returning the hostlist element with the lowest quality in the datastore
558 * @return hostlist with lowest quality
560 static struct Hostlist *
561 linked_list_get_lowest_quality ()
563 struct Hostlist *pos;
564 struct Hostlist *lowest;
566 if (linked_list_size == 0)
568 lowest = linked_list_head;
569 pos = linked_list_head->next;
572 if (pos->quality < lowest->quality)
581 * Method to insert a hostlist into the datastore. If datastore
582 * contains maximum number of elements, the elements with lowest
583 * quality is dismissed
588 struct Hostlist *lowest_quality;
590 if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
592 /* No free entries available, replace existing entry */
593 lowest_quality = linked_list_get_lowest_quality ();
594 GNUNET_assert (lowest_quality != NULL);
595 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
596 "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
597 lowest_quality->hostlist_uri,
598 (unsigned long long) lowest_quality->quality);
599 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
602 GNUNET_free (lowest_quality);
604 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
607 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
608 linked_list_size, GNUNET_NO);
609 stat_testing_hostlist = GNUNET_NO;
614 * Method updating hostlist statistics
621 if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
622 ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
624 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
625 "Updating hostlist statics for URI `%s'\n",
626 current_hostlist->hostlist_uri);
627 current_hostlist->hello_count = stat_hellos_obtained;
628 current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
629 current_hostlist->quality =
630 checked_add (current_hostlist->quality,
631 (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
632 if (GNUNET_YES == stat_download_successful)
634 current_hostlist->times_used++;
635 current_hostlist->quality =
636 checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
637 GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
638 current_hostlist->hostlist_uri);
640 GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
644 current_hostlist->quality =
645 checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
647 current_hostlist = NULL;
648 /* Alternating the usage of preconfigured and learned hostlists */
650 if (stat_testing_hostlist == GNUNET_YES)
653 if (GNUNET_YES == stat_learning)
655 if (stat_use_bootstrap == GNUNET_YES)
656 stat_use_bootstrap = GNUNET_NO;
658 stat_use_bootstrap = GNUNET_YES;
661 stat_use_bootstrap = GNUNET_YES;
666 * Clean up the state from the task that downloaded the
667 * hostlist and schedule the next task.
674 if ((stat_testing_hostlist == GNUNET_YES) &&
675 (GNUNET_NO == stat_download_successful) && (NULL != hostlist_to_test))
677 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
679 ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
680 hostlist_to_test->hostlist_uri);
683 if (stat_testing_hostlist == GNUNET_YES)
685 stat_testing_hostlist = GNUNET_NO;
687 if (NULL != hostlist_to_test)
689 GNUNET_free (hostlist_to_test);
690 hostlist_to_test = NULL;
695 mret = curl_multi_remove_handle (multi, curl);
696 if (mret != CURLM_OK)
698 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
699 "curl_multi_remove_handle", __FILE__, __LINE__,
700 curl_multi_strerror (mret));
702 mret = curl_multi_cleanup (multi);
703 if (mret != CURLM_OK)
704 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
705 "curl_multi_cleanup", __FILE__, __LINE__,
706 curl_multi_strerror (mret));
711 curl_easy_cleanup (curl);
714 GNUNET_free_non_null (current_url);
716 stat_bytes_downloaded = 0;
717 stat_download_in_progress = GNUNET_NO;
722 * Task that is run when we are ready to receive more data from the hostlist
725 * @param cls closure, unused
726 * @param tc task context, unused
729 task_download (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc);
733 * Ask CURL for the select set and then schedule the
734 * receiving task with the scheduler.
744 struct GNUNET_NETWORK_FDSet *grs;
745 struct GNUNET_NETWORK_FDSet *gws;
747 struct GNUNET_TIME_Relative rtime;
753 mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
754 if (mret != CURLM_OK)
756 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
757 "curl_multi_fdset", __FILE__, __LINE__,
758 curl_multi_strerror (mret));
762 mret = curl_multi_timeout (multi, &timeout);
763 if (mret != CURLM_OK)
765 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
766 "curl_multi_timeout", __FILE__, __LINE__,
767 curl_multi_strerror (mret));
772 GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
773 GNUNET_TIME_relative_multiply
774 (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
775 grs = GNUNET_NETWORK_fdset_create ();
776 gws = GNUNET_NETWORK_fdset_create ();
777 GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
778 GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
779 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
780 "Scheduling task for hostlist download using cURL\n");
782 GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
784 &task_download, multi);
785 GNUNET_NETWORK_fdset_destroy (gws);
786 GNUNET_NETWORK_fdset_destroy (grs);
791 * Task that is run when we are ready to receive more data from the hostlist
794 * @param cls closure, unused
795 * @param tc task context, unused
798 task_download (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
804 ti_download = GNUNET_SCHEDULER_NO_TASK;
805 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
807 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
808 "Shutdown requested while trying to download hostlist from `%s'\n",
814 if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
816 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
817 _("Timeout trying to download hostlist from `%s'\n"),
823 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
824 "Ready for processing hostlist client request\n");
828 if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
830 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
831 _("Download limit of %u bytes exceeded, stopping download\n"),
832 MAX_BYTES_PER_HOSTLISTS);
836 mret = curl_multi_perform (multi, &running);
841 msg = curl_multi_info_read (multi, &running);
842 GNUNET_break (msg != NULL);
848 if ((msg->data.result != CURLE_OK) &&
849 (msg->data.result != CURLE_GOT_NOTHING))
850 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
851 _("Download of hostlist from `%s' failed: `%s'\n"),
853 curl_easy_strerror (msg->data.result));
856 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
857 _("Download of hostlist `%s' completed.\n"),
859 stat_download_successful = GNUNET_YES;
861 if (GNUNET_YES == stat_testing_hostlist)
863 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
865 ("Adding successfully tested hostlist `%s' datastore.\n"),
868 hostlist_to_test = NULL;
869 stat_testing_hostlist = GNUNET_NO;
879 while ((running > 0));
882 while (mret == CURLM_CALL_MULTI_PERFORM);
884 if (mret != CURLM_OK)
886 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
887 "curl_multi_perform", __FILE__, __LINE__,
888 curl_multi_strerror (mret));
896 * Main function that will download a hostlist and process its
906 current_url = download_get_url ();
907 if (current_url == NULL)
909 curl = curl_easy_init ();
917 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
918 _("Bootstrapping using hostlist at `%s'.\n"), current_url);
920 stat_download_in_progress = GNUNET_YES;
921 stat_download_successful = GNUNET_NO;
922 stat_hellos_obtained = 0;
923 stat_bytes_downloaded = 0;
925 GNUNET_STATISTICS_update (stats,
926 gettext_noop ("# hostlist downloads initiated"), 1,
930 CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
931 CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
932 if (NULL != proxy_username)
933 CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
934 if (NULL != proxy_password)
935 CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
939 CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
945 CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
951 CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
952 CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
953 CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
954 CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
955 /* no need to abort if the above failed */
956 CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
962 CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
964 CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
966 CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
967 if (0 == strncmp (current_url, "http", 4))
968 CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
969 CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
970 CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
972 /* this should no longer be needed; we're now single-threaded! */
973 CURL_EASY_SETOPT (curl, CURLOPT_NOSIGNAL, 1);
975 multi = curl_multi_init ();
982 mret = curl_multi_add_handle (multi, curl);
983 if (mret != CURLM_OK)
985 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
986 "curl_multi_add_handle", __FILE__, __LINE__,
987 curl_multi_strerror (mret));
988 mret = curl_multi_cleanup (multi);
989 if (mret != CURLM_OK)
990 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
991 "curl_multi_cleanup", __FILE__, __LINE__,
992 curl_multi_strerror (mret));
997 end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
1003 task_download_dispatcher (void *cls,
1004 const struct GNUNET_SCHEDULER_TaskContext *tc)
1006 ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
1007 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1009 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
1010 if (GNUNET_NO == stat_download_in_progress)
1012 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
1013 download_hostlist ();
1017 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1018 "Download in progess, have to wait...\n");
1019 ti_download_dispatcher_task =
1020 GNUNET_SCHEDULER_add_delayed (WAITING_INTERVALL,
1021 &task_download_dispatcher, NULL);
1027 * Task that checks if we should try to download a hostlist.
1028 * If so, we initiate the download, otherwise we schedule
1029 * this task again for a later time.
1032 task_check (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
1035 struct GNUNET_TIME_Relative delay;
1037 ti_check_download = GNUNET_SCHEDULER_NO_TASK;
1038 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1042 curl_global_cleanup ();
1043 return; /* in shutdown */
1045 if ( (stat_connection_count < MIN_CONNECTIONS) &&
1046 (GNUNET_SCHEDULER_NO_TASK == ti_download_dispatcher_task) )
1047 ti_download_dispatcher_task =
1048 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1050 delay = hostlist_delay;
1051 if (0 == hostlist_delay.rel_value_us)
1052 hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
1054 hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
1055 if (hostlist_delay.rel_value_us >
1056 GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
1058 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
1059 (1 + stat_connection_count));
1060 GNUNET_STATISTICS_set (stats,
1062 ("# milliseconds between hostlist downloads"),
1063 hostlist_delay.rel_value_us / 1000LL,
1067 delay = GNUNET_TIME_UNIT_ZERO;
1070 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1071 _("Have %u/%u connections. Will consider downloading hostlist in %s\n"),
1072 stat_connection_count, MIN_CONNECTIONS,
1073 GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
1074 ti_check_download = GNUNET_SCHEDULER_add_delayed (delay, &task_check, NULL);
1079 * This tasks sets hostlist testing to allowed after intervall between to testings is reached
1081 * @param cls closure
1082 * @param tc TaskContext
1085 task_testing_intervall_reset (void *cls,
1086 const struct GNUNET_SCHEDULER_TaskContext *tc)
1088 ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
1089 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1091 stat_testing_allowed = GNUNET_OK;
1092 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1093 "Testing new hostlist advertisements is allowed again\n");
1098 * Task that writes hostlist entries to a file on a regular base
1100 * @param cls closure
1101 * @param tc TaskContext
1104 task_hostlist_saving (void *cls, const struct GNUNET_SCHEDULER_TaskContext *tc)
1106 ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
1107 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1109 save_hostlist_file (GNUNET_NO);
1111 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1112 "Hostlists will be saved to file again in %s\n",
1113 GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVALL, GNUNET_YES));
1115 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL, &task_hostlist_saving,
1121 * Method called whenever a given peer connects.
1123 * @param cls closure
1124 * @param peer peer identity this notification is about
1127 handler_connect (void *cls, const struct GNUNET_PeerIdentity *peer)
1129 GNUNET_assert (stat_connection_count < UINT_MAX);
1130 stat_connection_count++;
1131 GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), 1,
1137 * Method called whenever a given peer disconnects.
1139 * @param cls closure
1140 * @param peer peer identity this notification is about
1143 handler_disconnect (void *cls, const struct GNUNET_PeerIdentity *peer)
1145 GNUNET_assert (stat_connection_count > 0);
1146 stat_connection_count--;
1147 GNUNET_STATISTICS_update (stats, gettext_noop ("# active connections"), -1,
1153 * Method called whenever an advertisement message arrives.
1155 * @param cls closure (always NULL)
1156 * @param peer the peer sending the message
1157 * @param message the actual message
1158 * @return #GNUNET_OK to keep the connection open,
1159 * #GNUNET_SYSERR to close it (signal serious error)
1162 handler_advertisement (void *cls, const struct GNUNET_PeerIdentity *peer,
1163 const struct GNUNET_MessageHeader *message)
1167 const struct GNUNET_MessageHeader *incoming;
1169 struct Hostlist *hostlist;
1171 GNUNET_assert (ntohs (message->type) ==
1172 GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
1173 size = ntohs (message->size);
1174 if (size <= sizeof (struct GNUNET_MessageHeader))
1176 GNUNET_break_op (0);
1177 return GNUNET_SYSERR;
1179 incoming = (const struct GNUNET_MessageHeader *) message;
1180 uri = (const char *) &incoming[1];
1181 uri_size = size - sizeof (struct GNUNET_MessageHeader);
1182 if (uri[uri_size - 1] != '\0')
1184 GNUNET_break_op (0);
1185 return GNUNET_SYSERR;
1187 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1188 "Hostlist client recieved advertisement from `%s' containing URI `%s'\n",
1189 GNUNET_i2s (peer), uri);
1190 if (GNUNET_NO != linked_list_contains (uri))
1192 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "URI `%s' is already known\n", uri);
1196 if (GNUNET_NO == stat_testing_allowed)
1198 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1199 "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
1200 return GNUNET_SYSERR;
1202 if (GNUNET_YES == stat_testing_hostlist)
1204 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1205 "Currently not accepting new advertisements: we are already testing a hostlist\n");
1206 return GNUNET_SYSERR;
1209 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
1210 hostlist->hostlist_uri = (const char *) &hostlist[1];
1211 memcpy (&hostlist[1], uri, uri_size);
1212 hostlist->time_creation = GNUNET_TIME_absolute_get ();
1213 hostlist->quality = HOSTLIST_INITIAL;
1214 hostlist_to_test = hostlist;
1216 stat_testing_hostlist = GNUNET_YES;
1217 stat_testing_allowed = GNUNET_NO;
1218 ti_testing_intervall_task =
1219 GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
1220 &task_testing_intervall_reset, NULL);
1222 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1223 "Testing new hostlist advertisements is locked for the next %s\n",
1224 GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
1227 ti_download_dispatcher_task =
1228 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1235 * Continuation called by the statistics code once
1236 * we go the stat. Initiates hostlist download scheduling.
1238 * @param cls closure
1239 * @param success #GNUNET_OK if statistics were
1240 * successfully obtained, #GNUNET_SYSERR if not.
1243 primary_task (void *cls, int success)
1246 GNUNET_assert (stats != NULL);
1247 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1248 "Statistics request done, scheduling hostlist download\n");
1249 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
1254 * We've received the previous delay value from statistics. Remember it.
1256 * @param cls NULL, unused
1257 * @param subsystem should be "hostlist", unused
1258 * @param name will be "milliseconds between hostlist downloads", unused
1259 * @param value previous delay value, in milliseconds (!)
1260 * @param is_persistent unused, will be GNUNET_YES
1263 process_stat (void *cls, const char *subsystem, const char *name,
1264 uint64_t value, int is_persistent)
1266 hostlist_delay.rel_value_us = value * 1000LL;
1267 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1268 "Initial time between hostlist downloads is %s\n",
1269 GNUNET_STRINGS_relative_time_to_string (hostlist_delay, GNUNET_YES));
1275 * Method to load persistent hostlist file during hostlist client startup
1278 load_hostlist_file ()
1283 struct Hostlist *hostlist;
1284 uint32_t times_used;
1285 uint32_t hellos_returned;
1293 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1296 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1297 "hostlist", "HOSTLISTFILE");
1301 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1302 _("Loading saved hostlist entries from file `%s' \n"),
1304 if (GNUNET_NO == GNUNET_DISK_file_test (filename))
1306 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1307 _("Hostlist file `%s' does not exist\n"), filename);
1308 GNUNET_free (filename);
1312 struct GNUNET_BIO_ReadHandle *rh = GNUNET_BIO_read_open (filename);
1316 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1318 ("Could not open file `%s' for reading to load hostlists: %s\n"),
1319 filename, STRERROR (errno));
1320 GNUNET_free (filename);
1325 while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
1326 (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, ×_used))
1327 && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
1328 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
1329 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
1330 (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
1332 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
1333 hostlist->hello_count = hellos_returned;
1334 hostlist->hostlist_uri = (const char *) &hostlist[1];
1335 memcpy (&hostlist[1], uri, strlen (uri) + 1);
1336 hostlist->quality = quality;
1337 hostlist->time_creation.abs_value_us = created;
1338 hostlist->time_last_usage.abs_value_us = last_used;
1339 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
1341 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1342 "Added hostlist entry eith URI `%s' \n",
1343 hostlist->hostlist_uri);
1347 if (counter >= MAX_NUMBER_HOSTLISTS)
1351 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%u hostlist URIs loaded from file\n"),
1353 GNUNET_STATISTICS_set (stats, gettext_noop ("# hostlist URIs read from file"),
1354 counter, GNUNET_YES);
1355 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
1356 linked_list_size, GNUNET_NO);
1358 GNUNET_free_non_null (uri);
1360 (void) GNUNET_BIO_read_close (rh, &emsg);
1363 GNUNET_free (filename);
1368 * Method to save persistent hostlist file during hostlist client shutdown
1369 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
1372 save_hostlist_file (int shutdown)
1375 struct Hostlist *pos;
1376 struct GNUNET_BIO_WriteHandle *wh;
1381 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1384 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1385 "hostlist", "HOSTLISTFILE");
1388 if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
1390 GNUNET_free (filename);
1393 wh = GNUNET_BIO_write_open (filename);
1396 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1398 ("Could not open file `%s' for writing to save hostlists: %s\n"),
1399 filename, STRERROR (errno));
1400 GNUNET_free (filename);
1403 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("Writing %u hostlist URIs to `%s'\n"),
1404 linked_list_size, filename);
1405 /* add code to write hostlists to file using bio */
1408 while (NULL != (pos = linked_list_head))
1410 if (GNUNET_YES == shutdown)
1412 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
1415 if (GNUNET_YES == ok)
1417 if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
1418 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
1419 (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
1421 GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
1423 GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
1424 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
1426 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1427 _("Error writing hostlist URIs to file `%s'\n"), filename);
1432 if (GNUNET_YES == shutdown)
1435 if (counter >= MAX_NUMBER_HOSTLISTS)
1438 GNUNET_STATISTICS_set (stats,
1439 gettext_noop ("# hostlist URIs written to file"),
1440 counter, GNUNET_YES);
1442 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
1443 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1444 _("Error writing hostlist URIs to file `%s'\n"), filename);
1445 GNUNET_free (filename);
1450 * Start downloading hostlists from hostlist servers as necessary.
1453 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
1454 struct GNUNET_STATISTICS_Handle *st,
1455 GNUNET_CORE_ConnectEventHandler *ch,
1456 GNUNET_CORE_DisconnectEventHandler *dh,
1457 GNUNET_CORE_MessageCallback *msgh, int learn)
1460 char *proxytype_str;
1463 GNUNET_assert (NULL != st);
1464 if (0 != curl_global_init (CURL_GLOBAL_WIN32))
1467 return GNUNET_SYSERR;
1469 transport = GNUNET_TRANSPORT_connect (c, NULL, NULL, NULL, NULL, NULL);
1470 if (NULL == transport)
1472 curl_global_cleanup ();
1473 return GNUNET_SYSERR;
1478 /* Read proxy configuration */
1479 if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
1480 "HOSTLIST", "PROXY", &proxy))
1482 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1483 "Found proxy host: `%s'\n",
1485 /* proxy username */
1486 if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
1487 "HOSTLIST", "PROXY_USERNAME", &proxy_username))
1489 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1490 "Found proxy username name: `%s'\n",
1494 /* proxy password */
1495 if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
1496 "HOSTLIST", "PROXY_PASSWORD", &proxy_password))
1498 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1499 "Found proxy password name: `%s'\n",
1504 if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
1505 "HOSTLIST", "PROXY_TYPE", &proxytype_str))
1507 GNUNET_STRINGS_utf8_toupper (proxytype_str, proxytype_str);
1509 proxy_type = CURLPROXY_HTTP;
1510 if (0 == strcmp(proxytype_str, "HTTP"))
1511 proxy_type = CURLPROXY_HTTP;
1512 else if (0 == strcmp(proxytype_str, "HTTP_1_0"))
1513 proxy_type = CURLPROXY_HTTP_1_0;
1514 else if (0 == strcmp(proxytype_str, "SOCKS4"))
1515 proxy_type = CURLPROXY_SOCKS4;
1516 else if (0 == strcmp(proxytype_str, "SOCKS5"))
1517 proxy_type = CURLPROXY_SOCKS5;
1518 else if (0 == strcmp(proxytype_str, "SOCKS4A"))
1519 proxy_type = CURLPROXY_SOCKS4A;
1520 else if (0 == strcmp(proxytype_str, "SOCKS5_HOSTNAME"))
1521 proxy_type = CURLPROXY_SOCKS5_HOSTNAME ;
1524 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1525 _("Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
1527 GNUNET_free (proxytype_str);
1529 GNUNET_free (proxy);
1531 GNUNET_free_non_null (proxy_username);
1532 proxy_username = NULL;
1533 GNUNET_free_non_null (proxy_password);
1534 proxy_password = NULL;
1536 return GNUNET_SYSERR;
1539 GNUNET_free_non_null (proxytype_str);
1542 stat_learning = learn;
1543 *ch = &handler_connect;
1544 *dh = &handler_disconnect;
1545 linked_list_head = NULL;
1546 linked_list_tail = NULL;
1547 stat_use_bootstrap = GNUNET_YES;
1548 stat_testing_hostlist = GNUNET_NO;
1549 stat_testing_allowed = GNUNET_YES;
1551 if (GNUNET_YES == stat_learning)
1553 *msgh = &handler_advertisement;
1554 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1555 _("Learning is enabled on this peer\n"));
1556 load_hostlist_file ();
1557 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1558 "Hostlists will be saved to file again in %s\n",
1559 GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVALL, GNUNET_YES));
1561 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVALL, &task_hostlist_saving,
1566 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1567 _("Learning is not enabled on this peer\n"));
1570 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST",
1571 "HOSTLISTFILE", &filename))
1573 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1575 result = remove (filename);
1577 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1579 ("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
1582 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1583 _("Hostlist file `%s' could not be removed\n"), filename);
1586 GNUNET_free (filename);
1588 sget = GNUNET_STATISTICS_get (stats, "hostlist",
1590 ("# milliseconds between hostlist downloads"),
1591 GNUNET_TIME_UNIT_MINUTES, &primary_task, &process_stat,
1598 * Stop downloading hostlists from hostlist servers as necessary.
1601 GNUNET_HOSTLIST_client_stop ()
1603 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Hostlist client shutdown\n");
1606 GNUNET_STATISTICS_get_cancel (sget);
1610 if (GNUNET_YES == stat_learning)
1611 save_hostlist_file (GNUNET_YES);
1612 if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
1614 GNUNET_SCHEDULER_cancel (ti_saving_task);
1615 ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
1618 if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
1620 GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
1621 ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
1623 if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
1625 GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
1626 ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
1628 if (ti_download != GNUNET_SCHEDULER_NO_TASK)
1630 GNUNET_SCHEDULER_cancel (ti_download);
1631 ti_download = GNUNET_SCHEDULER_NO_TASK;
1633 if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
1635 GNUNET_SCHEDULER_cancel (ti_check_download);
1636 ti_check_download = GNUNET_SCHEDULER_NO_TASK;
1637 curl_global_cleanup ();
1639 if (transport != NULL)
1641 GNUNET_TRANSPORT_disconnect (transport);
1644 GNUNET_free_non_null (proxy);
1646 GNUNET_free_non_null (proxy_username);
1647 proxy_username = NULL;
1648 GNUNET_free_non_null (proxy_password);
1649 proxy_password = NULL;
1654 /* end of hostlist-client.c */