2 This file is part of GNUnet.
3 Copyright (C) 2001-2010, 2014, 2016 GNUnet e.V.
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 3, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
18 Boston, MA 02110-1301, USA.
21 * @file hostlist/gnunet-daemon-hostlist_client.c
22 * @brief hostlist support. Downloads HELLOs via HTTP.
23 * @author Christian Grothoff
24 * @author Matthias Wachs
27 #include "gnunet-daemon-hostlist_client.h"
28 #include "gnunet_hello_lib.h"
29 #include "gnunet_statistics_service.h"
30 #include "gnunet_transport_service.h"
31 #include "gnunet-daemon-hostlist.h"
33 #include <curl/curl.h>
34 #elif HAVE_GNURL_CURL_H
35 #include <gnurl/curl.h>
41 * Number of connections that we must have to NOT download
44 #define MIN_CONNECTIONS 4
47 * Maximum number of hostlist that are saved
49 #define MAX_NUMBER_HOSTLISTS 30
52 * Time interval hostlists are saved to disk
54 #define SAVING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MINUTES, 30)
57 * Time interval between two hostlist tests
59 #define TESTING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 3)
62 * Time interval for download dispatcher before a download is re-scheduled
64 #define WAITING_INTERVAL GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_SECONDS, 1)
67 * Defines concerning the hostlist quality metric
71 * Initial quality of a new created hostlist
73 #define HOSTLIST_INITIAL 10000
76 * Value subtracted each time a hostlist download fails
78 #define HOSTLIST_FAILED_DOWNLOAD 100
81 * Value added each time a hostlist download is successful
83 #define HOSTLIST_SUCCESSFUL_DOWNLOAD 100
86 * Value added for each valid HELLO recived during a hostlist download
88 #define HOSTLIST_SUCCESSFUL_HELLO 1
93 * A single hostlist obtained by hostlist advertisements
98 * previous entry, used to manage entries in a double linked list
100 struct Hostlist *prev;
103 * next entry, used to manage entries in a double linked list
105 struct Hostlist *next;
108 * URI where hostlist can be obtained
110 const char *hostlist_uri;
113 * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
114 * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
115 * intial value = HOSTLIST_INITIAL
116 * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
117 * increased every successful download by number of obtained HELLO messages
118 * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
123 * Time the hostlist advertisement was recieved and the entry was created
125 struct GNUNET_TIME_Absolute time_creation;
128 * Last time the hostlist was obtained
130 struct GNUNET_TIME_Absolute time_last_usage;
133 * Number of HELLO messages obtained during last download
135 uint32_t hello_count;
138 * Number of times the hostlist was successfully obtained
147 struct HelloOffer *next;
148 struct HelloOffer *prev;
149 struct GNUNET_TRANSPORT_OfferHelloHandle *ohh;
156 static const struct GNUNET_CONFIGURATION_Handle *cfg;
161 static struct GNUNET_STATISTICS_Handle *stats;
164 * Proxy hostname or ip we are using (can be NULL).
169 * Proxy username we are using (can be NULL).
171 static char *proxy_username;
174 * Proxy password we are using (can be NULL).
176 static char *proxy_password;
179 * Proxy type we are using (can be NULL).
181 static curl_proxytype proxy_type;
184 * Number of bytes valid in 'download_buffer'.
186 static size_t download_pos;
189 * Current URL that we are using.
191 static char *current_url;
194 * Current CURL handle.
199 * Current multi-CURL handle.
204 * How many bytes did we download from the current hostlist URL?
206 static uint32_t stat_bytes_downloaded;
209 * Amount of time we wait between hostlist downloads.
211 static struct GNUNET_TIME_Relative hostlist_delay;
214 * ID of the task, checking if hostlist download should take plate
216 static struct GNUNET_SCHEDULER_Task *ti_check_download;
219 * ID of the task downloading the hostlist
221 static struct GNUNET_SCHEDULER_Task *ti_download;
224 * ID of the task saving the hostlsit in a regular intervall
226 static struct GNUNET_SCHEDULER_Task *ti_saving_task;
229 * ID of the task called to initiate a download
231 static struct GNUNET_SCHEDULER_Task *ti_download_dispatcher_task;
234 * ID of the task controlling the locking between two hostlist tests
236 static struct GNUNET_SCHEDULER_Task *ti_testing_intervall_task;
239 * At what time MUST the current hostlist request be done?
241 static struct GNUNET_TIME_Absolute end_time;
244 * Head of the linked list used to store hostlists
246 static struct Hostlist *linked_list_head;
249 * Tail of the linked list used to store hostlists
251 static struct Hostlist *linked_list_tail;
254 * Current hostlist used for downloading
256 static struct Hostlist *current_hostlist;
259 * Size of the linke list used to store hostlists
261 static unsigned int linked_list_size;
264 * Head of the linked list used to store hostlists
266 static struct Hostlist *hostlist_to_test;
269 * Handle for our statistics GET operation.
271 static struct GNUNET_STATISTICS_GetHandle *sget;
274 * Set to GNUNET_YES if the current URL had some problems.
276 static int stat_bogus_url;
279 * Value controlling if a hostlist is tested at the moment
281 static int stat_testing_hostlist;
284 * Value controlling if a hostlist testing is allowed at the moment
286 static int stat_testing_allowed;
289 * Value controlling if a hostlist download is running at the moment
291 static int stat_download_in_progress;
294 * Value saying if a preconfigured bootstrap server is used
296 static unsigned int stat_use_bootstrap;
299 * Set if we are allowed to learn new hostlists and use them
301 static int stat_learning;
304 * Value saying if hostlist download was successful
306 static unsigned int stat_download_successful;
309 * Value saying how many valid HELLO messages were obtained during download
311 static unsigned int stat_hellos_obtained;
314 * Number of active connections (according to core service).
316 static unsigned int stat_connection_count;
318 static struct HelloOffer *ho_head;
320 static struct HelloOffer *ho_tail;
324 * Hello offer complete. Clean up.
327 done_offer_hello (void *cls)
329 struct HelloOffer *ho = cls;
331 GNUNET_CONTAINER_DLL_remove (ho_head,
339 * Process downloaded bits by calling callback on each HELLO.
341 * @param ptr buffer with downloaded data
342 * @param size size of a record
343 * @param nmemb number of records downloaded
345 * @return number of bytes that were processed (always size*nmemb)
348 callback_download (void *ptr,
353 static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE - 1];
354 const char *cbuf = ptr;
355 const struct GNUNET_MessageHeader *msg;
356 struct HelloOffer *ho;
362 total = size * nmemb;
363 stat_bytes_downloaded += total;
364 if ((total == 0) || (stat_bogus_url))
366 return total; /* ok, no data or bogus data */
369 GNUNET_STATISTICS_update (stats,
371 ("# bytes downloaded from hostlist servers"),
372 (int64_t) total, GNUNET_NO);
374 while ((left > 0) || (download_pos > 0))
376 cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - 1 - download_pos);
377 GNUNET_memcpy (&download_buffer[download_pos], cbuf, cpy);
381 if (download_pos < sizeof (struct GNUNET_MessageHeader))
383 GNUNET_assert (0 == left);
386 msg = (const struct GNUNET_MessageHeader *) download_buffer;
387 msize = ntohs (msg->size);
388 if (msize < sizeof (struct GNUNET_MessageHeader))
390 GNUNET_STATISTICS_update (stats,
392 ("# invalid HELLOs downloaded from hostlist servers"),
394 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
395 _("Invalid `%s' message received from hostlist at `%s'\n"),
396 "HELLO", current_url);
397 stat_hellos_obtained++;
401 if (download_pos < msize)
403 GNUNET_assert (left == 0);
406 if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message *) msg) == msize)
408 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
409 "Received valid `%s' message from hostlist server.\n",
411 GNUNET_STATISTICS_update (stats,
413 ("# valid HELLOs downloaded from hostlist servers"),
415 stat_hellos_obtained++;
417 ho = GNUNET_new (struct HelloOffer);
418 ho->ohh = GNUNET_TRANSPORT_offer_hello (cfg,
428 GNUNET_CONTAINER_DLL_insert (ho_head,
435 GNUNET_STATISTICS_update (stats,
437 ("# invalid HELLOs downloaded from hostlist servers"),
439 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
440 _("Invalid `%s' message received from hostlist at `%s'\n"),
441 "HELLO", current_url);
442 stat_bogus_url = GNUNET_YES;
443 stat_hellos_obtained++;
446 memmove (download_buffer,
447 &download_buffer[msize],
448 download_pos - msize);
449 download_pos -= msize;
456 * Obtain a hostlist URL that we should use.
458 * @return NULL if there is no URL available
461 get_bootstrap_server ()
469 GNUNET_CONFIGURATION_get_value_string (cfg, "HOSTLIST", "SERVERS",
472 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
473 "hostlist", "SERVERS");
478 if (strlen (servers) > 0)
481 pos = strlen (servers) - 1;
484 if (servers[pos] == ' ')
491 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
492 "hostlist", "SERVERS");
493 GNUNET_free (servers);
497 urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
498 pos = strlen (servers) - 1;
501 if (servers[pos] == ' ')
513 ret = GNUNET_strdup (&servers[pos]);
514 GNUNET_free (servers);
520 * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
521 * @return uri to use, NULL if there is no URL available
527 unsigned int counter;
528 struct Hostlist *pos;
530 if (GNUNET_NO == stat_learning)
532 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
533 "Using preconfigured bootstrap server\n");
534 current_hostlist = NULL;
535 return get_bootstrap_server ();
538 if ((GNUNET_YES == stat_testing_hostlist) && (NULL != hostlist_to_test))
540 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
541 "Testing new advertised hostlist if it is obtainable\n");
542 current_hostlist = hostlist_to_test;
543 return GNUNET_strdup (hostlist_to_test->hostlist_uri);
546 if ((GNUNET_YES == stat_use_bootstrap) || (linked_list_size == 0))
548 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
549 "Using preconfigured bootstrap server\n");
550 current_hostlist = NULL;
551 return get_bootstrap_server ();
554 GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
556 pos = linked_list_head;
557 while (counter < index)
562 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Using learned hostlist `%s'\n",
564 current_hostlist = pos;
565 return GNUNET_strdup (pos->hostlist_uri);
569 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt (c, a, b); if (CURLE_OK != ret) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror (ret)); } while (0)
573 * Method to save hostlist to a file during hostlist client shutdown
575 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
578 save_hostlist_file (int shutdown);
582 * Add val2 to val1 with overflow check
584 * @param val1 value 1
585 * @param val2 value 2
589 checked_add (uint64_t val1,
592 static uint64_t temp;
593 static uint64_t maxv;
606 * Subtract val2 from val1 with underflow check
608 * @param val1 value 1
609 * @param val2 value 2
613 checked_sub (uint64_t val1,
618 return (val1 - val2);
623 * Method to check if a URI is in hostlist linked list
625 * @param uri uri to check
626 * @return #GNUNET_YES if existing in linked list, #GNUNET_NO if not
629 linked_list_contains (const char *uri)
631 struct Hostlist *pos;
633 pos = linked_list_head;
636 if (0 == strcmp (pos->hostlist_uri, uri))
645 * Method returning the hostlist element with the lowest quality in the datastore
646 * @return hostlist with lowest quality
648 static struct Hostlist *
649 linked_list_get_lowest_quality ()
651 struct Hostlist *pos;
652 struct Hostlist *lowest;
654 if (linked_list_size == 0)
656 lowest = linked_list_head;
657 pos = linked_list_head->next;
660 if (pos->quality < lowest->quality)
669 * Method to insert a hostlist into the datastore. If datastore
670 * contains maximum number of elements, the elements with lowest
671 * quality is dismissed
676 struct Hostlist *lowest_quality;
678 if (MAX_NUMBER_HOSTLISTS <= linked_list_size)
680 /* No free entries available, replace existing entry */
681 lowest_quality = linked_list_get_lowest_quality ();
682 GNUNET_assert (lowest_quality != NULL);
683 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
684 "Removing hostlist with URI `%s' which has the worst quality of all (%llu)\n",
685 lowest_quality->hostlist_uri,
686 (unsigned long long) lowest_quality->quality);
687 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail,
690 GNUNET_free (lowest_quality);
692 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail,
695 GNUNET_STATISTICS_set (stats, gettext_noop ("# advertised hostlist URIs"),
696 linked_list_size, GNUNET_NO);
697 stat_testing_hostlist = GNUNET_NO;
702 * Method updating hostlist statistics
709 if (((stat_use_bootstrap == GNUNET_NO) && (NULL != current_hostlist)) ||
710 ((stat_testing_hostlist == GNUNET_YES) && (NULL != current_hostlist)))
712 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
713 "Updating hostlist statics for URI `%s'\n",
714 current_hostlist->hostlist_uri);
715 current_hostlist->hello_count = stat_hellos_obtained;
716 current_hostlist->time_last_usage = GNUNET_TIME_absolute_get ();
717 current_hostlist->quality =
718 checked_add (current_hostlist->quality,
719 (stat_hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
720 if (GNUNET_YES == stat_download_successful)
722 current_hostlist->times_used++;
723 current_hostlist->quality =
724 checked_add (current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
725 GNUNET_asprintf (&stat, gettext_noop ("# advertised URI `%s' downloaded"),
726 current_hostlist->hostlist_uri);
728 GNUNET_STATISTICS_update (stats, stat, 1, GNUNET_YES);
732 current_hostlist->quality =
733 checked_sub (current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD);
735 current_hostlist = NULL;
736 /* Alternating the usage of preconfigured and learned hostlists */
738 if (stat_testing_hostlist == GNUNET_YES)
741 if (GNUNET_YES == stat_learning)
743 if (stat_use_bootstrap == GNUNET_YES)
744 stat_use_bootstrap = GNUNET_NO;
746 stat_use_bootstrap = GNUNET_YES;
749 stat_use_bootstrap = GNUNET_YES;
754 * Clean up the state from the task that downloaded the
755 * hostlist and schedule the next task.
762 if ( (stat_testing_hostlist == GNUNET_YES) &&
763 (GNUNET_NO == stat_download_successful) &&
764 (NULL != hostlist_to_test) )
766 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
768 ("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),
769 hostlist_to_test->hostlist_uri);
772 if (stat_testing_hostlist == GNUNET_YES)
774 stat_testing_hostlist = GNUNET_NO;
776 if (NULL != hostlist_to_test)
778 GNUNET_free (hostlist_to_test);
779 hostlist_to_test = NULL;
784 mret = curl_multi_remove_handle (multi, curl);
785 if (mret != CURLM_OK)
787 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
788 "curl_multi_remove_handle", __FILE__, __LINE__,
789 curl_multi_strerror (mret));
791 mret = curl_multi_cleanup (multi);
792 if (mret != CURLM_OK)
793 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
794 "curl_multi_cleanup", __FILE__, __LINE__,
795 curl_multi_strerror (mret));
800 curl_easy_cleanup (curl);
803 GNUNET_free_non_null (current_url);
805 stat_bytes_downloaded = 0;
806 stat_download_in_progress = GNUNET_NO;
811 * Task that is run when we are ready to receive more data from the hostlist
814 * @param cls closure, unused
815 * @param tc task context, unused
818 task_download (void *cls);
822 * Ask CURL for the select set and then schedule the
823 * receiving task with the scheduler.
833 struct GNUNET_NETWORK_FDSet *grs;
834 struct GNUNET_NETWORK_FDSet *gws;
836 struct GNUNET_TIME_Relative rtime;
842 mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
843 if (mret != CURLM_OK)
845 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
846 "curl_multi_fdset", __FILE__, __LINE__,
847 curl_multi_strerror (mret));
851 mret = curl_multi_timeout (multi, &timeout);
852 if (mret != CURLM_OK)
854 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
855 "curl_multi_timeout", __FILE__, __LINE__,
856 curl_multi_strerror (mret));
861 GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
862 GNUNET_TIME_relative_multiply
863 (GNUNET_TIME_UNIT_MILLISECONDS, timeout));
864 grs = GNUNET_NETWORK_fdset_create ();
865 gws = GNUNET_NETWORK_fdset_create ();
866 GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
867 GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
868 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
869 "Scheduling task for hostlist download using cURL\n");
871 GNUNET_SCHEDULER_add_select (GNUNET_SCHEDULER_PRIORITY_DEFAULT,
873 &task_download, multi);
874 GNUNET_NETWORK_fdset_destroy (gws);
875 GNUNET_NETWORK_fdset_destroy (grs);
880 * Task that is run when we are ready to receive more data from the hostlist
883 * @param cls closure, unused
886 task_download (void *cls)
893 if (0 == GNUNET_TIME_absolute_get_remaining (end_time).rel_value_us)
895 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
896 _("Timeout trying to download hostlist from `%s'\n"),
902 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
903 "Ready for processing hostlist client request\n");
907 if (stat_bytes_downloaded > MAX_BYTES_PER_HOSTLISTS)
909 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
910 _("Download limit of %u bytes exceeded, stopping download\n"),
911 MAX_BYTES_PER_HOSTLISTS);
915 mret = curl_multi_perform (multi, &running);
920 msg = curl_multi_info_read (multi, &running);
921 GNUNET_break (msg != NULL);
927 if ((msg->data.result != CURLE_OK) &&
928 (msg->data.result != CURLE_GOT_NOTHING))
929 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
930 _("Download of hostlist from `%s' failed: `%s'\n"),
932 curl_easy_strerror (msg->data.result));
935 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
936 _("Download of hostlist `%s' completed.\n"),
938 stat_download_successful = GNUNET_YES;
940 if (GNUNET_YES == stat_testing_hostlist)
942 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
944 ("Adding successfully tested hostlist `%s' datastore.\n"),
947 hostlist_to_test = NULL;
948 stat_testing_hostlist = GNUNET_NO;
958 while ((running > 0));
961 while (mret == CURLM_CALL_MULTI_PERFORM);
963 if (mret != CURLM_OK)
965 GNUNET_log (GNUNET_ERROR_TYPE_INFO, _("%s failed at %s:%d: `%s'\n"),
966 "curl_multi_perform", __FILE__, __LINE__,
967 curl_multi_strerror (mret));
975 * Main function that will download a hostlist and process its
985 current_url = download_get_url ();
986 if (current_url == NULL)
988 curl = curl_easy_init ();
996 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
997 _("Bootstrapping using hostlist at `%s'.\n"), current_url);
999 stat_download_in_progress = GNUNET_YES;
1000 stat_download_successful = GNUNET_NO;
1001 stat_hellos_obtained = 0;
1002 stat_bytes_downloaded = 0;
1004 GNUNET_STATISTICS_update (stats,
1005 gettext_noop ("# hostlist downloads initiated"), 1,
1009 CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
1010 CURL_EASY_SETOPT (curl, CURLOPT_PROXYTYPE, proxy_type);
1011 if (NULL != proxy_username)
1012 CURL_EASY_SETOPT (curl, CURLOPT_PROXYUSERNAME, proxy_username);
1013 if (NULL != proxy_password)
1014 CURL_EASY_SETOPT (curl, CURLOPT_PROXYPASSWORD, proxy_password);
1018 CURL_EASY_SETOPT (curl, CURLOPT_WRITEFUNCTION, &callback_download);
1019 if (ret != CURLE_OK)
1024 CURL_EASY_SETOPT (curl, CURLOPT_WRITEDATA, NULL);
1025 if (ret != CURLE_OK)
1030 CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
1031 CURL_EASY_SETOPT (curl, CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
1032 CURL_EASY_SETOPT (curl, CURLOPT_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS);
1033 CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
1034 /* no need to abort if the above failed */
1035 CURL_EASY_SETOPT (curl, CURLOPT_URL, current_url);
1036 if (ret != CURLE_OK)
1041 CURL_EASY_SETOPT (curl, CURLOPT_FAILONERROR, 1);
1043 CURL_EASY_SETOPT (curl, CURLOPT_VERBOSE, 1);
1045 CURL_EASY_SETOPT (curl, CURLOPT_BUFFERSIZE, GNUNET_SERVER_MAX_MESSAGE_SIZE);
1046 if (0 == strncmp (current_url, "http", 4))
1047 CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
1048 CURL_EASY_SETOPT (curl, CURLOPT_CONNECTTIMEOUT, 60L);
1049 CURL_EASY_SETOPT (curl, CURLOPT_TIMEOUT, 60L);
1050 multi = curl_multi_init ();
1057 mret = curl_multi_add_handle (multi, curl);
1058 if (mret != CURLM_OK)
1060 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
1061 "curl_multi_add_handle", __FILE__, __LINE__,
1062 curl_multi_strerror (mret));
1063 mret = curl_multi_cleanup (multi);
1064 if (mret != CURLM_OK)
1065 GNUNET_log (GNUNET_ERROR_TYPE_ERROR, _("%s failed at %s:%d: `%s'\n"),
1066 "curl_multi_cleanup", __FILE__, __LINE__,
1067 curl_multi_strerror (mret));
1072 end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
1073 download_prepare ();
1078 task_download_dispatcher (void *cls)
1080 ti_download_dispatcher_task = NULL;
1081 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download is initiated...\n");
1082 if (GNUNET_NO == stat_download_in_progress)
1084 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG, "Download can start immediately...\n");
1085 download_hostlist ();
1089 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1090 "Download in progess, have to wait...\n");
1091 ti_download_dispatcher_task =
1092 GNUNET_SCHEDULER_add_delayed (WAITING_INTERVAL,
1093 &task_download_dispatcher, NULL);
1099 * Task that checks if we should try to download a hostlist.
1100 * If so, we initiate the download, otherwise we schedule
1101 * this task again for a later time.
1104 task_check (void *cls)
1107 struct GNUNET_TIME_Relative delay;
1109 ti_check_download = NULL;
1112 curl_global_cleanup ();
1113 return; /* in shutdown */
1115 if ( (stat_connection_count < MIN_CONNECTIONS) &&
1116 (NULL == ti_download_dispatcher_task) )
1117 ti_download_dispatcher_task =
1118 GNUNET_SCHEDULER_add_now (&task_download_dispatcher, NULL);
1120 delay = hostlist_delay;
1121 if (0 == hostlist_delay.rel_value_us)
1122 hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
1124 hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
1125 if (hostlist_delay.rel_value_us >
1126 GNUNET_TIME_UNIT_HOURS.rel_value_us * (1 + stat_connection_count))
1128 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
1129 (1 + stat_connection_count));
1130 GNUNET_STATISTICS_set (stats,
1132 ("# milliseconds between hostlist downloads"),
1133 hostlist_delay.rel_value_us / 1000LL,
1137 delay = GNUNET_TIME_UNIT_ZERO;
1140 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1141 _("Have %u/%u connections. Will consider downloading hostlist in %s\n"),
1142 stat_connection_count, MIN_CONNECTIONS,
1143 GNUNET_STRINGS_relative_time_to_string (delay, GNUNET_YES));
1144 ti_check_download = GNUNET_SCHEDULER_add_delayed (delay,
1150 * This tasks sets hostlist testing to allowed after intervall between to testings is reached
1152 * @param cls closure
1155 task_testing_intervall_reset (void *cls)
1157 ti_testing_intervall_task = NULL;
1158 stat_testing_allowed = GNUNET_OK;
1159 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1160 "Testing new hostlist advertisements is allowed again\n");
1165 * Task that writes hostlist entries to a file on a regular base
1167 * @param cls closure
1170 task_hostlist_saving (void *cls)
1172 ti_saving_task = NULL;
1173 save_hostlist_file (GNUNET_NO);
1175 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1176 "Hostlists will be saved to file again in %s\n",
1177 GNUNET_STRINGS_relative_time_to_string(SAVING_INTERVAL, GNUNET_YES));
1179 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
1180 &task_hostlist_saving,
1186 * Method called whenever a given peer connects.
1188 * @param cls closure
1189 * @param peer peer identity this notification is about
1190 * @param mq message queue for transmissions to @a peer
1193 handler_connect (void *cls,
1194 const struct GNUNET_PeerIdentity *peer,
1195 struct GNUNET_MQ_Handle *mq)
1197 GNUNET_assert (stat_connection_count < UINT_MAX);
1198 stat_connection_count++;
1199 GNUNET_STATISTICS_update (stats,
1200 gettext_noop ("# active connections"),
1208 * Method called whenever a given peer disconnects.
1210 * @param cls closure
1211 * @param peer peer identity this notification is about
1214 handler_disconnect (void *cls,
1215 const struct GNUNET_PeerIdentity *peer,
1218 GNUNET_assert (stat_connection_count > 0);
1219 stat_connection_count--;
1220 GNUNET_STATISTICS_update (stats,
1221 gettext_noop ("# active connections"),
1228 * Method called whenever an advertisement message arrives.
1230 * @param uri the advertised URI
1233 handler_advertisement (const char *uri)
1236 struct Hostlist *hostlist;
1238 uri_size = strlen (uri) + 1;
1239 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1240 "Hostlist client recieved advertisement containing URI `%s'\n",
1242 if (GNUNET_NO != linked_list_contains (uri))
1244 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1245 "URI `%s' is already known\n",
1250 if (GNUNET_NO == stat_testing_allowed)
1252 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1253 "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
1256 if (GNUNET_YES == stat_testing_hostlist)
1258 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1259 "Currently not accepting new advertisements: we are already testing a hostlist\n");
1263 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
1264 hostlist->hostlist_uri = (const char *) &hostlist[1];
1265 GNUNET_memcpy (&hostlist[1],
1268 hostlist->time_creation = GNUNET_TIME_absolute_get ();
1269 hostlist->quality = HOSTLIST_INITIAL;
1270 hostlist_to_test = hostlist;
1272 stat_testing_hostlist = GNUNET_YES;
1273 stat_testing_allowed = GNUNET_NO;
1274 ti_testing_intervall_task =
1275 GNUNET_SCHEDULER_add_delayed (TESTING_INTERVAL,
1276 &task_testing_intervall_reset,
1279 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1280 "Testing new hostlist advertisements is locked for the next %s\n",
1281 GNUNET_STRINGS_relative_time_to_string (TESTING_INTERVAL,
1284 ti_download_dispatcher_task =
1285 GNUNET_SCHEDULER_add_now (&task_download_dispatcher,
1291 * Continuation called by the statistics code once
1292 * we go the stat. Initiates hostlist download scheduling.
1294 * @param cls closure
1295 * @param success #GNUNET_OK if statistics were
1296 * successfully obtained, #GNUNET_SYSERR if not.
1299 primary_task (void *cls,
1302 if (NULL != ti_check_download)
1304 GNUNET_SCHEDULER_cancel (ti_check_download);
1305 ti_check_download = NULL;
1308 GNUNET_assert (NULL != stats);
1309 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1310 "Statistics request done, scheduling hostlist download\n");
1311 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check, NULL);
1316 * Continuation called by the statistics code once
1317 * we go the stat. Initiates hostlist download scheduling.
1319 * @param cls closure
1320 * @param success #GNUNET_OK if statistics were
1321 * successfully obtained, #GNUNET_SYSERR if not.
1324 stat_timeout_task (void *cls)
1326 GNUNET_STATISTICS_get_cancel (sget);
1328 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
1334 * We've received the previous delay value from statistics. Remember it.
1336 * @param cls NULL, unused
1337 * @param subsystem should be "hostlist", unused
1338 * @param name will be "milliseconds between hostlist downloads", unused
1339 * @param value previous delay value, in milliseconds (!)
1340 * @param is_persistent unused, will be #GNUNET_YES
1343 process_stat (void *cls,
1344 const char *subsystem,
1349 hostlist_delay.rel_value_us = value * 1000LL;
1350 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1351 "Initial time between hostlist downloads is %s\n",
1352 GNUNET_STRINGS_relative_time_to_string (hostlist_delay,
1359 * Method to load persistent hostlist file during hostlist client startup
1362 load_hostlist_file ()
1367 struct Hostlist *hostlist;
1368 uint32_t times_used;
1369 uint32_t hellos_returned;
1374 struct GNUNET_BIO_ReadHandle *rh;
1378 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1381 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1382 "hostlist", "HOSTLISTFILE");
1386 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1387 _("Loading saved hostlist entries from file `%s' \n"),
1389 if (GNUNET_NO == GNUNET_DISK_file_test (filename))
1391 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1392 _("Hostlist file `%s' does not exist\n"), filename);
1393 GNUNET_free (filename);
1397 rh = GNUNET_BIO_read_open (filename);
1400 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1401 _("Could not open file `%s' for reading to load hostlists: %s\n"),
1402 filename, STRERROR (errno));
1403 GNUNET_free (filename);
1408 while ((GNUNET_OK == GNUNET_BIO_read_string (rh, "url", &uri, MAX_URL_LEN)) &&
1409 (NULL != uri) && (GNUNET_OK == GNUNET_BIO_read_int32 (rh, ×_used))
1410 && (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
1411 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
1412 (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
1413 (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)))
1415 hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
1416 hostlist->hello_count = hellos_returned;
1417 hostlist->hostlist_uri = (const char *) &hostlist[1];
1418 GNUNET_memcpy (&hostlist[1], uri, strlen (uri) + 1);
1419 hostlist->quality = quality;
1420 hostlist->time_creation.abs_value_us = created;
1421 hostlist->time_last_usage.abs_value_us = last_used;
1422 GNUNET_CONTAINER_DLL_insert (linked_list_head, linked_list_tail, hostlist);
1424 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1425 "Added hostlist entry eith URI `%s' \n",
1426 hostlist->hostlist_uri);
1430 if (counter >= MAX_NUMBER_HOSTLISTS)
1434 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1435 _("%u hostlist URIs loaded from file\n"),
1437 GNUNET_STATISTICS_set (stats,
1438 gettext_noop ("# hostlist URIs read from file"),
1439 counter, GNUNET_YES);
1440 GNUNET_STATISTICS_set (stats,
1441 gettext_noop ("# advertised hostlist URIs"),
1442 linked_list_size, GNUNET_NO);
1444 GNUNET_free_non_null (uri);
1446 (void) GNUNET_BIO_read_close (rh, &emsg);
1449 GNUNET_free (filename);
1454 * Method to save persistent hostlist file during hostlist client shutdown
1456 * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
1459 save_hostlist_file (int shutdown)
1462 struct Hostlist *pos;
1463 struct GNUNET_BIO_WriteHandle *wh;
1468 GNUNET_CONFIGURATION_get_value_filename (cfg, "HOSTLIST", "HOSTLISTFILE",
1471 GNUNET_log_config_missing (GNUNET_ERROR_TYPE_WARNING,
1472 "hostlist", "HOSTLISTFILE");
1475 if (GNUNET_SYSERR == GNUNET_DISK_directory_create_for_file (filename))
1477 GNUNET_free (filename);
1480 wh = GNUNET_BIO_write_open (filename);
1483 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1484 _("Could not open file `%s' for writing to save hostlists: %s\n"),
1487 GNUNET_free (filename);
1490 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1491 _("Writing %u hostlist URIs to `%s'\n"),
1492 linked_list_size, filename);
1493 /* add code to write hostlists to file using bio */
1496 while (NULL != (pos = linked_list_head))
1498 if (GNUNET_YES == shutdown)
1500 GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
1503 if (GNUNET_YES == ok)
1505 if ((GNUNET_OK != GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
1506 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
1507 (GNUNET_OK != GNUNET_BIO_write_int64 (wh, pos->quality)) ||
1509 GNUNET_BIO_write_int64 (wh, pos->time_last_usage.abs_value_us)) ||
1511 GNUNET_BIO_write_int64 (wh, pos->time_creation.abs_value_us)) ||
1512 (GNUNET_OK != GNUNET_BIO_write_int32 (wh, pos->hello_count)))
1514 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1515 _("Error writing hostlist URIs to file `%s'\n"), filename);
1520 if (GNUNET_YES == shutdown)
1523 if (counter >= MAX_NUMBER_HOSTLISTS)
1526 GNUNET_STATISTICS_set (stats,
1527 gettext_noop ("# hostlist URIs written to file"),
1528 counter, GNUNET_YES);
1530 if (GNUNET_OK != GNUNET_BIO_write_close (wh))
1531 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1532 _("Error writing hostlist URIs to file `%s'\n"), filename);
1533 GNUNET_free (filename);
1538 * Start downloading hostlists from hostlist servers as necessary.
1540 * @param c configuration to use
1541 * @param st statistics handle to use
1542 * @param[out] ch set to handler for CORE connect events
1543 * @param[out] dh set to handler for CORE disconnect events
1544 * @param[out] msgh set to handler for CORE advertisement messages
1545 * @param learn should we learn hostlist URLs from CORE
1546 * @return #GNUNET_OK on success
1549 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
1550 struct GNUNET_STATISTICS_Handle *st,
1551 GNUNET_CORE_ConnecTEventHandler *ch,
1552 GNUNET_CORE_DisconnecTEventHandler *dh,
1553 GNUNET_HOSTLIST_UriHandler *msgh,
1557 char *proxytype_str;
1560 GNUNET_assert (NULL != st);
1561 if (0 != curl_global_init (CURL_GLOBAL_WIN32))
1564 return GNUNET_SYSERR;
1569 /* Read proxy configuration */
1571 GNUNET_CONFIGURATION_get_value_string (cfg,
1576 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1577 "Found proxy host: `%s'\n",
1579 /* proxy username */
1581 GNUNET_CONFIGURATION_get_value_string (cfg,
1586 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1587 "Found proxy username name: `%s'\n",
1591 /* proxy password */
1593 GNUNET_CONFIGURATION_get_value_string (cfg,
1598 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1599 "Found proxy password name: `%s'\n",
1605 GNUNET_CONFIGURATION_get_value_string (cfg,
1610 GNUNET_STRINGS_utf8_toupper (proxytype_str,
1612 proxy_type = CURLPROXY_HTTP;
1613 if (0 == strcmp (proxytype_str, "HTTP"))
1614 proxy_type = CURLPROXY_HTTP;
1615 else if (0 == strcmp (proxytype_str, "HTTP_1_0"))
1616 proxy_type = CURLPROXY_HTTP_1_0;
1617 else if (0 == strcmp (proxytype_str, "SOCKS4"))
1618 proxy_type = CURLPROXY_SOCKS4;
1619 else if (0 == strcmp (proxytype_str, "SOCKS5"))
1620 proxy_type = CURLPROXY_SOCKS5;
1621 else if (0 == strcmp (proxytype_str, "SOCKS4A"))
1622 proxy_type = CURLPROXY_SOCKS4A;
1623 else if (0 == strcmp (proxytype_str, "SOCKS5_HOSTNAME"))
1624 proxy_type = CURLPROXY_SOCKS5_HOSTNAME;
1627 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1628 _("Invalid proxy type: `%s', disabling proxy! Check configuration!\n"),
1630 GNUNET_free (proxytype_str);
1631 GNUNET_free (proxy);
1633 GNUNET_free_non_null (proxy_username);
1634 proxy_username = NULL;
1635 GNUNET_free_non_null (proxy_password);
1636 proxy_password = NULL;
1638 return GNUNET_SYSERR;
1641 GNUNET_free_non_null (proxytype_str);
1644 stat_learning = learn;
1645 *ch = &handler_connect;
1646 *dh = &handler_disconnect;
1647 linked_list_head = NULL;
1648 linked_list_tail = NULL;
1649 stat_use_bootstrap = GNUNET_YES;
1650 stat_testing_hostlist = GNUNET_NO;
1651 stat_testing_allowed = GNUNET_YES;
1653 if (GNUNET_YES == stat_learning)
1655 *msgh = &handler_advertisement;
1656 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1657 _("Learning is enabled on this peer\n"));
1658 load_hostlist_file ();
1659 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1660 "Hostlists will be saved to file again in %s\n",
1661 GNUNET_STRINGS_relative_time_to_string (SAVING_INTERVAL,
1664 GNUNET_SCHEDULER_add_delayed (SAVING_INTERVAL,
1665 &task_hostlist_saving,
1670 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1671 _("Learning is not enabled on this peer\n"));
1674 GNUNET_CONFIGURATION_get_value_filename (cfg,
1679 if (GNUNET_YES == GNUNET_DISK_file_test (filename))
1681 result = remove (filename);
1683 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1684 _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),
1687 GNUNET_log_strerror_file (GNUNET_ERROR_TYPE_ERROR,
1692 GNUNET_free (filename);
1694 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1695 "Loading stats value on hostlist download frequency\n");
1696 sget = GNUNET_STATISTICS_get (stats, "hostlist",
1698 ("# milliseconds between hostlist downloads"),
1704 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1705 "Statistics request failed, scheduling hostlist download\n");
1706 ti_check_download = GNUNET_SCHEDULER_add_now (&task_check,
1712 = GNUNET_SCHEDULER_add_delayed (GNUNET_TIME_UNIT_MINUTES,
1721 * Stop downloading hostlists from hostlist servers as necessary.
1724 GNUNET_HOSTLIST_client_stop ()
1726 struct HelloOffer *ho;
1728 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1729 "Hostlist client shutdown\n");
1730 while (NULL != (ho = ho_head))
1732 GNUNET_CONTAINER_DLL_remove (ho_head,
1735 GNUNET_TRANSPORT_offer_hello_cancel (ho->ohh);
1740 GNUNET_STATISTICS_get_cancel (sget);
1744 if (GNUNET_YES == stat_learning)
1745 save_hostlist_file (GNUNET_YES);
1746 if (NULL != ti_saving_task)
1748 GNUNET_SCHEDULER_cancel (ti_saving_task);
1749 ti_saving_task = NULL;
1751 if (NULL != ti_download_dispatcher_task)
1753 GNUNET_SCHEDULER_cancel (ti_download_dispatcher_task);
1754 ti_download_dispatcher_task = NULL;
1756 if (NULL != ti_testing_intervall_task)
1758 GNUNET_SCHEDULER_cancel (ti_testing_intervall_task);
1759 ti_testing_intervall_task = NULL;
1761 if (NULL != ti_download)
1763 GNUNET_SCHEDULER_cancel (ti_download);
1768 if (NULL != ti_check_download)
1770 GNUNET_SCHEDULER_cancel (ti_check_download);
1771 ti_check_download = NULL;
1772 curl_global_cleanup ();
1774 GNUNET_free_non_null (proxy);
1776 GNUNET_free_non_null (proxy_username);
1777 proxy_username = NULL;
1778 GNUNET_free_non_null (proxy_password);
1779 proxy_password = NULL;
1784 /* end of gnunet-daemon-hostlist_client.c */