2 This file is part of GNUnet.
3 Copyright (C) 2001-2010, 2014, 2016 GNUnet e.V.
5 GNUnet is free software: you can redistribute it and/or modify it
6 under the terms of the GNU Affero General Public License as published
7 by the Free Software Foundation, either version 3 of the License,
8 or (at your option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Affero General Public License for more details.
15 You should have received a copy of the GNU Affero General Public License
16 along with this program. If not, see <http://www.gnu.org/licenses/>.
18 SPDX-License-Identifier: AGPL3.0-or-later
21 * @file hostlist/gnunet-daemon-hostlist_client.c
22 * @brief hostlist support. Downloads HELLOs via HTTP.
23 * @author Christian Grothoff
24 * @author Matthias Wachs
27 #include "gnunet-daemon-hostlist_client.h"
28 #include "gnunet_hello_lib.h"
29 #include "gnunet_statistics_service.h"
30 #include "gnunet_transport_service.h"
31 #include "gnunet_peerinfo_service.h"
32 #include "gnunet-daemon-hostlist.h"
34 #include <curl/curl.h>
35 #elif HAVE_GNURL_CURL_H
36 #include <gnurl/curl.h>
42 * Number of connections that we must have to NOT download
45 #define MIN_CONNECTIONS 4
48 * Maximum number of hostlist that are saved
50 #define MAX_NUMBER_HOSTLISTS 30
53 * Time interval hostlists are saved to disk
55 #define SAVING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
58 * Time interval between two hostlist tests
60 #define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)
63 * Time interval for download dispatcher before a download is re-scheduled
65 #define WAITING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
68 * Defines concerning the hostlist quality metric
72 * Initial quality of a new created hostlist
74 #define HOSTLIST_INITIAL 10000
77 * Value subtracted each time a hostlist download fails
79 #define HOSTLIST_FAILED_DOWNLOAD 100
82 * Value added each time a hostlist download is successful
84 #define HOSTLIST_SUCCESSFUL_DOWNLOAD 100
87 * Value added for each valid HELLO recived during a hostlist download
89 #define HOSTLIST_SUCCESSFUL_HELLO 1
94 * A single hostlist obtained by hostlist advertisements
99 * previous entry, used to manage entries in a double linked list
101 struct Hostlist *prev;
104 * next entry, used to manage entries in a double linked list
106 struct Hostlist *next;
109 * URI where hostlist can be obtained
111 const char *hostlist_uri;
114 * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
115 * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
116 * intial value = HOSTLIST_INITIAL
117 * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
118 * increased every successful download by number of obtained HELLO messages
119 * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
124 * Time the hostlist advertisement was recieved and the entry was created
126 struct GNUNET_TIME_Absolute time_creation;
129 * Last time the hostlist was obtained
131 struct GNUNET_TIME_Absolute time_last_usage;
134 * Number of HELLO messages obtained during last download
136 uint32_t hello_count;
139 * Number of times the hostlist was successfully obtained
149 static const struct GNUNET_CONFIGURATION_Handle *cfg;
154 static struct GNUNET_STATISTICS_Handle *stats;
157 * Proxy hostname or ip we are using (can be NULL).
162 * Proxy username we are using (can be NULL).
164 static char *proxy_username;
167 * Proxy password we are using (can be NULL).
169 static char *proxy_password;
172 * Proxy type we are using (can be NULL).
174 static curl_proxytype proxy_type;
177 * Number of bytes valid in 'download_buffer'.
179 static size_t download_pos;
182 * Current URL that we are using.
184 static char *current_url;
187 * Current CURL handle.
192 * Current multi-CURL handle.
197 * How many bytes did we download from the current hostlist URL?
199 static uint32_t stat_bytes_downloaded;
202 * Amount of time we wait between hostlist downloads.
204 static struct GNUNET_TIME_Relative hostlist_delay;
207 * ID of the task, checking if hostlist download should take plate
209 static struct GNUNET_SCHEDULER_Task *ti_check_download;
212 * ID of the task downloading the hostlist
214 static struct GNUNET_SCHEDULER_Task *ti_download;
217 * ID of the task saving the hostlsit in a regular intervall
219 static struct GNUNET_SCHEDULER_Task *ti_saving_task;
222 * ID of the task called to initiate a download
224 static struct GNUNET_SCHEDULER_Task *ti_download_dispatcher_task;
227 * ID of the task controlling the locking between two hostlist tests
229 static struct GNUNET_SCHEDULER_Task *ti_testing_intervall_task;
232 * At what time MUST the current hostlist request be done?
234 static struct GNUNET_TIME_Absolute end_time;
237 * Head of the linked list used to store hostlists
239 static struct Hostlist *linked_list_head;
242 * Tail of the linked list used to store hostlists
244 static struct Hostlist *linked_list_tail;
247 * Current hostlist used for downloading
249 static struct Hostlist *current_hostlist;
252 * Size of the linke list used to store hostlists
254 static unsigned int linked_list_size;
257 * Head of the linked list used to store hostlists
259 static struct Hostlist *hostlist_to_test;
262 * Handle for our statistics GET operation.
264 static struct GNUNET_STATISTICS_GetHandle *sget;
267 * Set to GNUNET_YES if the current URL had some problems.
269 static int stat_bogus_url;
272 * Value controlling if a hostlist is tested at the moment
274 static int stat_testing_hostlist;
277 * Value controlling if a hostlist testing is allowed at the moment
279 static int stat_testing_allowed;
282 * Value controlling if a hostlist download is running at the moment
284 static int stat_download_in_progress;
287 * Value saying if a preconfigured bootstrap server is used
289 static unsigned int stat_use_bootstrap;
292 * Set if we are allowed to learn new hostlists and use them
294 static int stat_learning;
297 * Value saying if hostlist download was successful
299 static unsigned int stat_download_successful;
302 * Value saying how many valid HELLO messages were obtained during download
304 static unsigned int stat_hellos_obtained;
307 * Number of active connections (according to core service).
309 static unsigned int stat_connection_count;
312 * Handle to peerinfo service.
314 static struct GNUNET_PEERINFO_Handle *pi;
318 * Process downloaded bits by calling callback on each HELLO.
320 * @param ptr buffer with downloaded data
321 * @param size size of a record
322 * @param nmemb number of records downloaded
324 * @return number of bytes that were processed (always size*nmemb)
327 callback_download (void *ptr,
332 static char download_buffer[GNUNET_MAX_MESSAGE_SIZE - 1];
333 const char *cbuf = ptr;
334 const struct GNUNET_MessageHeader *msg;
340 total = size * nmemb;
341 stat_bytes_downloaded += total;
342 if ((total == 0) || (stat_bogus_url))
344 return total; /* ok, no data or bogus data */
347 GNUNET_STATISTICS_update (stats,
349 ("# bytes downloaded from hostlist servers"),
350 (int64_t) total, GNUNET_NO);
352 while ((left > 0) || (download_pos > 0))
354 cpy = GNUNET_MIN (left, GNUNET_MAX_MESSAGE_SIZE - 1 - download_pos);
355 GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
359 if (download_pos < sizeof (struct GNUNET_MessageHeader))
361 GNUNET_assert (0 == left);
364 msg = (const struct GNUNET_MessageHeader *) download_buffer;
365 msize = ntohs (msg->size);
366 if (msize < sizeof (struct GNUNET_MessageHeader))
368 GNUNET_STATISTICS_update (stats,
370 ("# invalid HELLOs downloaded from hostlist servers"),
372 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
373 _("Invalid `%s' message received from hostlist at `%s'\n"),
374 "HELLO", current_url);
375 stat_hellos_obtained++;
379 if (download_pos < msize)
381 GNUNET_assert (left == 0);
384 if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
386 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
387 "Received valid `%s' message from hostlist server.\n",
389 GNUNET_STATISTICS_update (stats,
391 ("# valid HELLOs downloaded from hostlist servers"),
393 stat_hellos_obtained++;
394 (void) GNUNET_PEERINFO_add_peer (pi,
395 (const struct GNUNET_HELLO_Message *) msg,
401 GNUNET_STATISTICS_update (stats,
403 ("# invalid HELLOs downloaded from hostlist servers"),
405 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
406 _("Invalid `%s' message received from hostlist at `%s'\n"),
407 "HELLO", current_url);
408 stat_bogus_url = GNUNET_YES;
409 stat_hellos_obtained++;
412 memmove (download_buffer,
413 &download_buffer[msize],
414 download_pos - msize);
415 download_pos -= msize;
422 * Obtain a hostlist URL that we should use.
424 * @return NULL if there is no URL available
427 get_bootstrap_server ()
435 GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
438 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
439 "hostlist", "SERVERS");
444 if (strlen (servers) > 0)
447 pos = strlen (servers) - 1;
450 if (servers[pos] == ' ')
457 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
458 "hostlist", "SERVERS");
459 GNUNET_free (servers);
463 urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
464 pos = strlen (servers) - 1;
467 if (servers[pos] == ' ')
479 ret = GNUNET_strdup (&servers[pos]);
480 GNUNET_free (servers);
486 * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
487 * @return uri to use, NULL if there is no URL available
493 unsigned int counter;
494 struct Hostlist *pos;
496 if (GNUNET_NO == stat_learning)
498 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
499 "Using preconfigured bootstrap server\n");
500 current_hostlist = NULL;
501 return get_bootstrap_server ();
504 if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
506 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
507 "Testing new advertised hostlist if it is obtainable\n");
508 current_hostlist = hostlist_to_test;
509 return GNUNET_strdup (hostlist_to_test->hostlist_uri);
512 if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
514 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
515 "Using preconfigured bootstrap server\n");
516 current_hostlist = NULL;
517 return get_bootstrap_server ();
520 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
522 pos = linked_list_head;
523 while (counter < index)
528 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
530 current_hostlist = pos;
531 return GNUNET_strdup (pos->hostlist_uri);
535 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)
539 * Method to save hostlist to a file during hostlist client shutdown
541 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
544 save_hostlist_file (int shutdown);
548 * Add val2 to val1 with overflow check
550 * @param val1 value 1
551 * @param val2 value 2
555 checked_add (uint64_t val1,
558 static uint64_t temp;
559 static uint64_t maxv;
572 * Subtract val2 from val1 with underflow check
574 * @param val1 value 1
575 * @param val2 value 2
579 checked_sub (uint64_t val1,
584 return (val1 - val2);
589 * Method to check if a URI is in hostlist linked list
591 * @param uri uri to check
592 * @return #GNUNET_YES if existing in linked list, #GNUNET_NO if not
595 linked_list_contains (const char *uri)
597 struct Hostlist *pos;
599 pos = linked_list_head;
602 if (0 == strcmp (pos->hostlist_uri, uri))
611 * Method returning the hostlist element with the lowest quality in the datastore
612 * @return hostlist with lowest quality
614 static struct Hostlist *
615 linked_list_get_lowest_quality ()
617 struct Hostlist *pos;
618 struct Hostlist *lowest;
620 if (linked_list_size == 0)
622 lowest = linked_list_head;
623 pos = linked_list_head->next;
626 if (pos->quality < lowest->quality)
635 * Method to insert a hostlist into the datastore. If datastore
636 * contains maximum number of elements, the elements with lowest
637 * quality is dismissed
642 struct Hostlist *lowest_quality;
644 if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
646 /* No free entries available, replace existing entry */
647 lowest_quality = linked_list_get_lowest_quality ();
648 GNUNET_assert (lowest_quality != NULL);
649 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
650 "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
651 lowest_quality->hostlist_uri,
652 (unsigned long long) lowest_quality->quality);
653 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
656 GNUNET_free (lowest_quality);
658 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
661 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
662 linked_list_size, GNUNET_NO);
663 stat_testing_hostlist = GNUNET_NO;
668 * Method updating hostlist statistics
675 if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
676 ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
678 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
679 "Updating hostlist statics for URI `%s'\n",
680 current_hostlist->hostlist_uri);
681 current_hostlist->hello_count = stat_hellos_obtained;
682 current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
683 current_hostlist->quality =
684 checked_add (current_hostlist->quality,
685 (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
686 if (GNUNET_YES == stat_download_successful)
688 current_hostlist->times_used++;
689 current_hostlist->quality =
690 checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
691 GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
692 current_hostlist->hostlist_uri);
694 GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
698 current_hostlist->quality =
699 checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
701 current_hostlist = NULL;
702 /* Alternating the usage of preconfigured and learned hostlists */
704 if (stat_testing_hostlist == GNUNET_YES)
707 if (GNUNET_YES == stat_learning)
709 if (stat_use_bootstrap == GNUNET_YES)
710 stat_use_bootstrap = GNUNET_NO;
712 stat_use_bootstrap = GNUNET_YES;
715 stat_use_bootstrap = GNUNET_YES;
720 * Clean up the state from the task that downloaded the
721 * hostlist and schedule the next task.
728 if ( (stat_testing_hostlist == GNUNET_YES) &&
729 (GNUNET_NO == stat_download_successful) &&
730 (NULL != hostlist_to_test) )
732 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
734 ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
735 hostlist_to_test->hostlist_uri);
738 if (stat_testing_hostlist == GNUNET_YES)
740 stat_testing_hostlist = GNUNET_NO;
742 if (NULL != hostlist_to_test)
744 GNUNET_free (hostlist_to_test);
745 hostlist_to_test = NULL;
750 mret = curl_multi_remove_handle (multi, curl);
751 if (mret != CURLM_OK)
753 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
754 "curl_multi_remove_handle", __FILE__, __LINE__,
755 curl_multi_strerror (mret));
757 mret = curl_multi_cleanup (multi);
758 if (mret != CURLM_OK)
759 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
760 "curl_multi_cleanup", __FILE__, __LINE__,
761 curl_multi_strerror (mret));
766 curl_easy_cleanup (curl);
769 GNUNET_free_non_null (current_url);
771 stat_bytes_downloaded = 0;
772 stat_download_in_progress = GNUNET_NO;
777 * Task that is run when we are ready to receive more data from the hostlist
780 * @param cls closure, unused
781 * @param tc task context, unused
784 task_download (void *cls);
788 * Ask CURL for the select set and then schedule the
789 * receiving task with the scheduler.
799 struct GNUNET_NETWORK_FDSet *grs;
800 struct GNUNET_NETWORK_FDSet *gws;
802 struct GNUNET_TIME_Relative rtime;
808 mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
809 if (mret != CURLM_OK)
811 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
812 "curl_multi_fdset", __FILE__, __LINE__,
813 curl_multi_strerror (mret));
817 mret = curl_multi_timeout (multi, &timeout);
818 if (mret != CURLM_OK)
820 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
821 "curl_multi_timeout", __FILE__, __LINE__,
822 curl_multi_strerror (mret));
827 GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
828 GNUNET_TIME_relative_multiply
829 (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
830 grs = GNUNET_NETWORK_fdset_create ();
831 gws = GNUNET_NETWORK_fdset_create ();
832 GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
833 GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
834 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
835 "Scheduling task for hostlist download using cURL\n");
837 GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
839 &task_download, multi);
840 GNUNET_NETWORK_fdset_destroy (gws);
841 GNUNET_NETWORK_fdset_destroy (grs);
846 * Task that is run when we are ready to receive more data from the hostlist
849 * @param cls closure, unused
852 task_download (void *cls)
859 if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
861 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
862 _("Timeout trying to download hostlist from `%s'\n"),
868 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
869 "Ready for processing hostlist client request\n");
873 if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
875 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
876 _("Download limit of %u bytes exceeded, stopping download\n"),
877 MAX_BYTES_PER_HOSTLISTS);
881 mret = curl_multi_perform (multi, &running);
886 msg = curl_multi_info_read (multi, &running);
887 GNUNET_break (msg != NULL);
893 if ((msg->data.result != CURLE_OK) &&
894 (msg->data.result != CURLE_GOT_NOTHING))
895 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
896 _("Download of hostlist from `%s' failed: `%s'\n"),
898 curl_easy_strerror (msg->data.result));
901 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
902 _("Download of hostlist `%s' completed.\n"),
904 stat_download_successful = GNUNET_YES;
906 if (GNUNET_YES == stat_testing_hostlist)
908 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
910 ("Adding successfully tested hostlist `%s' datastore.\n"),
913 hostlist_to_test = NULL;
914 stat_testing_hostlist = GNUNET_NO;
924 while ((running > 0));
927 while (mret == CURLM_CALL_MULTI_PERFORM);
929 if (mret != CURLM_OK)
931 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
932 "curl_multi_perform", __FILE__, __LINE__,
933 curl_multi_strerror (mret));
941 * Main function that will download a hostlist and process its
951 current_url = download_get_url ();
952 if (current_url == NULL)
954 curl = curl_easy_init ();
962 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
963 _("Bootstrapping using hostlist at `%s'.\n"), current_url);
965 stat_download_in_progress = GNUNET_YES;
966 stat_download_successful = GNUNET_NO;
967 stat_hellos_obtained = 0;
968 stat_bytes_downloaded = 0;
970 GNUNET_STATISTICS_update (stats,
971 gettext_noop ("# hostlist downloads initiated"), 1,
975 CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
976 CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
977 if (NULL != proxy_username)
978 CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
979 if (NULL != proxy_password)
980 CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
984 CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
990 CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
996 CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
997 CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
998 CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
999 CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
1000 /* no need to abort if the above failed */
1001 CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
1002 if (ret != CURLE_OK)
1007 CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
1009 CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
1011 CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_MAX_MESSAGE_SIZE);
1012 if (0 == strncmp (current_url, "http", 4))
1013 CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
1014 CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
1015 CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
1016 multi = curl_multi_init ();
1023 mret = curl_multi_add_handle (multi, curl);
1024 if (mret != CURLM_OK)
1026 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
1027 "curl_multi_add_handle", __FILE__, __LINE__,
1028 curl_multi_strerror (mret));
1029 mret = curl_multi_cleanup (multi);
1030 if (mret != CURLM_OK)
1031 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
1032 "curl_multi_cleanup", __FILE__, __LINE__,
1033 curl_multi_strerror (mret));
1038 end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
1039 download_prepare ();
1044 task_download_dispatcher (void *cls)
1046 ti_download_dispatcher_task = NULL;
1047 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
1048 if (GNUNET_NO == stat_download_in_progress)
1050 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
1051 download_hostlist ();
1055 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1056 "Download in progess, have to wait...\n");
1057 ti_download_dispatcher_task =
1058 GNUNET_SCHEDULER_add_delayed (WAITING_INTERVAL,
1059 &task_download_dispatcher, NULL);
1065 * Task that checks if we should try to download a hostlist.
1066 * If so, we initiate the download, otherwise we schedule
1067 * this task again for a later time.
1070 task_check (void *cls)
1073 struct GNUNET_TIME_Relative delay;
1075 ti_check_download = NULL;
1078 curl_global_cleanup ();
1079 return; /* in shutdown */
1081 if ( (stat_connection_count < MIN_CONNECTIONS) &&
1082 (NULL == ti_download_dispatcher_task) )
1083 ti_download_dispatcher_task =
1084 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1086 delay = hostlist_delay;
1087 if (0 == hostlist_delay.rel_value_us)
1088 hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
1090 hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
1091 if (hostlist_delay.rel_value_us >
1092 GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
1094 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
1095 (1 + stat_connection_count));
1096 GNUNET_STATISTICS_set (stats,
1098 ("# milliseconds between hostlist downloads"),
1099 hostlist_delay.rel_value_us / 1000LL,
1103 delay = GNUNET_TIME_UNIT_ZERO;
1106 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1107 _("Have %u/%u connections. Will consider downloading hostlist in %s\n"),
1108 stat_connection_count, MIN_CONNECTIONS,
1109 GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
1110 ti_check_download = GNUNET_SCHEDULER_add_delayed (delay,
1116 * This tasks sets hostlist testing to allowed after intervall between to testings is reached
1118 * @param cls closure
1121 task_testing_intervall_reset (void *cls)
1123 ti_testing_intervall_task = NULL;
1124 stat_testing_allowed = GNUNET_OK;
1125 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1126 "Testing new hostlist advertisements is allowed again\n");
1131 * Task that writes hostlist entries to a file on a regular base
1133 * @param cls closure
1136 task_hostlist_saving (void *cls)
1138 ti_saving_task = NULL;
1139 save_hostlist_file (GNUNET_NO);
1141 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1142 "Hostlists will be saved to file again in %s\n",
1143 GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVAL, GNUNET_YES));
1145 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
1146 &task_hostlist_saving,
1152 * Method called whenever a given peer connects.
1154 * @param cls closure
1155 * @param peer peer identity this notification is about
1156 * @param mq message queue for transmissions to @a peer
1159 handler_connect (void *cls,
1160 const struct GNUNET_PeerIdentity *peer,
1161 struct GNUNET_MQ_Handle *mq)
1163 GNUNET_assert (stat_connection_count < UINT_MAX);
1164 stat_connection_count++;
1165 GNUNET_STATISTICS_update (stats,
1166 gettext_noop ("# active connections"),
1174 * Method called whenever a given peer disconnects.
1176 * @param cls closure
1177 * @param peer peer identity this notification is about
1180 handler_disconnect (void *cls,
1181 const struct GNUNET_PeerIdentity *peer,
1184 GNUNET_assert (stat_connection_count > 0);
1185 stat_connection_count--;
1186 GNUNET_STATISTICS_update (stats,
1187 gettext_noop ("# active connections"),
1194 * Method called whenever an advertisement message arrives.
1196 * @param uri the advertised URI
1199 handler_advertisement (const char *uri)
1202 struct Hostlist *hostlist;
1204 uri_size = strlen (uri) + 1;
1205 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1206 "Hostlist client recieved advertisement containing URI `%s'\n",
1208 if (GNUNET_NO != linked_list_contains (uri))
1210 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1211 "URI `%s' is already known\n",
1216 if (GNUNET_NO == stat_testing_allowed)
1218 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1219 "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
1222 if (GNUNET_YES == stat_testing_hostlist)
1224 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1225 "Currently not accepting new advertisements: we are already testing a hostlist\n");
1229 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
1230 hostlist->hostlist_uri = (const char *) &hostlist[1];
1231 GNUNET_memcpy (&hostlist[1],
1234 hostlist->time_creation = GNUNET_TIME_absolute_get ();
1235 hostlist->quality = HOSTLIST_INITIAL;
1236 hostlist_to_test = hostlist;
1238 stat_testing_hostlist = GNUNET_YES;
1239 stat_testing_allowed = GNUNET_NO;
1240 ti_testing_intervall_task =
1241 GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
1242 &task_testing_intervall_reset,
1245 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1246 "Testing new hostlist advertisements is locked for the next %s\n",
1247 GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
1250 ti_download_dispatcher_task =
1251 GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
1257 * Continuation called by the statistics code once
1258 * we go the stat. Initiates hostlist download scheduling.
1260 * @param cls closure
1261 * @param success #GNUNET_OK if statistics were
1262 * successfully obtained, #GNUNET_SYSERR if not.
1265 primary_task (void *cls,
1268 if (NULL != ti_check_download)
1270 GNUNET_SCHEDULER_cancel (ti_check_download);
1271 ti_check_download = NULL;
1274 GNUNET_assert (NULL != stats);
1275 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1276 "Statistics request done, scheduling hostlist download\n");
1277 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
1282 * Continuation called by the statistics code once
1283 * we go the stat. Initiates hostlist download scheduling.
1285 * @param cls closure
1286 * @param success #GNUNET_OK if statistics were
1287 * successfully obtained, #GNUNET_SYSERR if not.
1290 stat_timeout_task (void *cls)
1292 GNUNET_STATISTICS_get_cancel (sget);
1294 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
1300 * We've received the previous delay value from statistics. Remember it.
1302 * @param cls NULL, unused
1303 * @param subsystem should be "hostlist", unused
1304 * @param name will be "milliseconds between hostlist downloads", unused
1305 * @param value previous delay value, in milliseconds (!)
1306 * @param is_persistent unused, will be #GNUNET_YES
1309 process_stat (void *cls,
1310 const char *subsystem,
1315 hostlist_delay.rel_value_us = value * 1000LL;
1316 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1317 "Initial time between hostlist downloads is %s\n",
1318 GNUNET_STRINGS_relative_time_to_string (hostlist_delay,
1325 * Method to load persistent hostlist file during hostlist client startup
1328 load_hostlist_file ()
1333 struct Hostlist *hostlist;
1334 uint32_t times_used;
1335 uint32_t hellos_returned;
1340 struct GNUNET_BIO_ReadHandle *rh;
1344 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1347 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1348 "hostlist", "HOSTLISTFILE");
1352 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1353 _("Loading saved hostlist entries from file `%s' \n"),
1355 if (GNUNET_NO == GNUNET_DISK_file_test (filename))
1357 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1358 _("Hostlist file `%s' does not exist\n"), filename);
1359 GNUNET_free (filename);
1363 rh = GNUNET_BIO_read_open (filename);
1366 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1367 _("Could not open file `%s' for reading to load hostlists: %s\n"),
1368 filename, STRERROR (errno));
1369 GNUNET_free (filename);
1374 while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
1375 (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, ×_used))
1376 && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
1377 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
1378 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
1379 (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
1381 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
1382 hostlist->hello_count = hellos_returned;
1383 hostlist->hostlist_uri = (const char *) &hostlist[1];
1384 GNUNET_memcpy (&hostlist[1], uri, strlen (uri) + 1);
1385 hostlist->quality = quality;
1386 hostlist->time_creation.abs_value_us = created;
1387 hostlist->time_last_usage.abs_value_us = last_used;
1388 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
1390 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1391 "Added hostlist entry eith URI `%s' \n",
1392 hostlist->hostlist_uri);
1396 if (counter >= MAX_NUMBER_HOSTLISTS)
1400 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1401 _("%u hostlist URIs loaded from file\n"),
1403 GNUNET_STATISTICS_set (stats,
1404 gettext_noop ("# hostlist URIs read from file"),
1405 counter, GNUNET_YES);
1406 GNUNET_STATISTICS_set (stats,
1407 gettext_noop ("# advertised hostlist URIs"),
1408 linked_list_size, GNUNET_NO);
1410 GNUNET_free_non_null (uri);
1412 (void) GNUNET_BIO_read_close (rh, &emsg);
1415 GNUNET_free (filename);
1420 * Method to save persistent hostlist file during hostlist client shutdown
1422 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
1425 save_hostlist_file (int shutdown)
1428 struct Hostlist *pos;
1429 struct GNUNET_BIO_WriteHandle *wh;
1434 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1437 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1438 "hostlist", "HOSTLISTFILE");
1441 if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
1443 GNUNET_free (filename);
1446 wh = GNUNET_BIO_write_open (filename);
1449 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1450 _("Could not open file `%s' for writing to save hostlists: %s\n"),
1453 GNUNET_free (filename);
1456 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1457 _("Writing %u hostlist URIs to `%s'\n"),
1458 linked_list_size, filename);
1459 /* add code to write hostlists to file using bio */
1462 while (NULL != (pos = linked_list_head))
1464 if (GNUNET_YES == shutdown)
1466 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
1469 if (GNUNET_YES == ok)
1471 if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
1472 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
1473 (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
1475 GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
1477 GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
1478 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
1480 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1481 _("Error writing hostlist URIs to file `%s'\n"), filename);
1486 if (GNUNET_YES == shutdown)
1489 if (counter >= MAX_NUMBER_HOSTLISTS)
1492 GNUNET_STATISTICS_set (stats,
1493 gettext_noop ("# hostlist URIs written to file"),
1494 counter, GNUNET_YES);
1496 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
1497 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1498 _("Error writing hostlist URIs to file `%s'\n"), filename);
1499 GNUNET_free (filename);
1504 * Start downloading hostlists from hostlist servers as necessary.
1506 * @param c configuration to use
1507 * @param st statistics handle to use
1508 * @param[out] ch set to handler for CORE connect events
1509 * @param[out] dh set to handler for CORE disconnect events
1510 * @param[out] msgh set to handler for CORE advertisement messages
1511 * @param learn should we learn hostlist URLs from CORE
1512 * @return #GNUNET_OK on success
1515 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
1516 struct GNUNET_STATISTICS_Handle *st,
1517 GNUNET_CORE_ConnectEventHandler *ch,
1518 GNUNET_CORE_DisconnectEventHandler *dh,
1519 GNUNET_HOSTLIST_UriHandler *msgh,
1523 char *proxytype_str;
1526 GNUNET_assert (NULL != st);
1527 if (0 != curl_global_init (CURL_GLOBAL_WIN32))
1530 return GNUNET_SYSERR;
1535 /* Read proxy configuration */
1536 pi = GNUNET_PEERINFO_connect (c);
1538 GNUNET_CONFIGURATION_get_value_string (cfg,
1543 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1544 "Found proxy host: `%s'\n",
1546 /* proxy username */
1548 GNUNET_CONFIGURATION_get_value_string (cfg,
1553 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1554 "Found proxy username name: `%s'\n",
1558 /* proxy password */
1560 GNUNET_CONFIGURATION_get_value_string (cfg,
1565 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1566 "Found proxy password name: `%s'\n",
1572 GNUNET_CONFIGURATION_get_value_string (cfg,
1577 GNUNET_STRINGS_utf8_toupper (proxytype_str,
1579 proxy_type = CURLPROXY_HTTP;
1580 if (0 == strcmp (proxytype_str, "HTTP"))
1581 proxy_type = CURLPROXY_HTTP;
1582 else if (0 == strcmp (proxytype_str, "HTTP_1_0"))
1583 proxy_type = CURLPROXY_HTTP_1_0;
1584 else if (0 == strcmp (proxytype_str, "SOCKS4"))
1585 proxy_type = CURLPROXY_SOCKS4;
1586 else if (0 == strcmp (proxytype_str, "SOCKS5"))
1587 proxy_type = CURLPROXY_SOCKS5;
1588 else if (0 == strcmp (proxytype_str, "SOCKS4A"))
1589 proxy_type = CURLPROXY_SOCKS4A;
1590 else if (0 == strcmp (proxytype_str, "SOCKS5_HOSTNAME"))
1591 proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
1594 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1595 _("Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
1597 GNUNET_free (proxytype_str);
1598 GNUNET_free (proxy);
1600 GNUNET_free_non_null (proxy_username);
1601 proxy_username = NULL;
1602 GNUNET_free_non_null (proxy_password);
1603 proxy_password = NULL;
1605 return GNUNET_SYSERR;
1608 GNUNET_free_non_null (proxytype_str);
1611 stat_learning = learn;
1612 *ch = &handler_connect;
1613 *dh = &handler_disconnect;
1614 linked_list_head = NULL;
1615 linked_list_tail = NULL;
1616 stat_use_bootstrap = GNUNET_YES;
1617 stat_testing_hostlist = GNUNET_NO;
1618 stat_testing_allowed = GNUNET_YES;
1620 if (GNUNET_YES == stat_learning)
1622 *msgh = &handler_advertisement;
1623 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1624 _("Learning is enabled on this peer\n"));
1625 load_hostlist_file ();
1626 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1627 "Hostlists will be saved to file again in %s\n",
1628 GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL,
1631 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
1632 &task_hostlist_saving,
1637 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1638 _("Learning is not enabled on this peer\n"));
1641 GNUNET_CONFIGURATION_get_value_filename (cfg,
1646 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1648 result = remove (filename);
1650 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1651 _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
1654 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1659 GNUNET_free (filename);
1661 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1662 "Loading stats value on hostlist download frequency\n");
1663 sget = GNUNET_STATISTICS_get (stats, "hostlist",
1665 ("# milliseconds between hostlist downloads"),
1671 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1672 "Statistics request failed, scheduling hostlist download\n");
1673 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
1679 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
1688 * Stop downloading hostlists from hostlist servers as necessary.
1691 GNUNET_HOSTLIST_client_stop ()
1693 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1694 "Hostlist client shutdown\n");
1697 GNUNET_STATISTICS_get_cancel (sget);
1701 if (GNUNET_YES == stat_learning)
1702 save_hostlist_file (GNUNET_YES);
1703 if (NULL != ti_saving_task)
1705 GNUNET_SCHEDULER_cancel (ti_saving_task);
1706 ti_saving_task = NULL;
1708 if (NULL != ti_download_dispatcher_task)
1710 GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
1711 ti_download_dispatcher_task = NULL;
1713 if (NULL != ti_testing_intervall_task)
1715 GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
1716 ti_testing_intervall_task = NULL;
1718 if (NULL != ti_download)
1720 GNUNET_SCHEDULER_cancel (ti_download);
1725 if (NULL != ti_check_download)
1727 GNUNET_SCHEDULER_cancel (ti_check_download);
1728 ti_check_download = NULL;
1729 curl_global_cleanup ();
1731 GNUNET_free_non_null (proxy);
1733 GNUNET_free_non_null (proxy_username);
1734 proxy_username = NULL;
1735 GNUNET_free_non_null (proxy_password);
1736 proxy_password = NULL;
1739 GNUNET_PEERINFO_disconnect (pi);
1745 /* end of gnunet-daemon-hostlist_client.c */