2 This file is part of GNUnet.
3 (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009, 2010 Christian Grothoff (and other contributing authors)
5 GNUnet is free software; you can redistribute it and/or modify
6 it under the terms of the GNU General Public License as published
7 by the Free Software Foundation; either version 2, or (at your
8 option) any later version.
10 GNUnet is distributed in the hope that it will be useful, but
11 WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 General Public License for more details.
15 You should have received a copy of the GNU General Public License
16 along with GNUnet; see the file COPYING. If not, write to the
17 Free Software Foundation, Inc., 59 Temple Place - Suite 330,
18 Boston, MA 02111-1307, USA.
22 * @file hostlist/hostlist-client.c
23 * @brief hostlist support. Downloads HELLOs via HTTP.
24 * @author Christian Grothoff
28 #include "hostlist-client.h"
29 #include "gnunet_core_service.h"
30 #include "gnunet_hello_lib.h"
31 #include "gnunet_statistics_service.h"
32 #include "gnunet_transport_service.h"
33 #include <curl/curl.h>
35 #define DEBUG_HOSTLIST_CLIENT GNUNET_NO
38 * Number of connections that we must have to NOT download
41 #define MIN_CONNECTIONS 4
46 static const struct GNUNET_CONFIGURATION_Handle *cfg;
51 static struct GNUNET_SCHEDULER_Handle *sched;
56 struct GNUNET_STATISTICS_Handle *stats;
61 struct GNUNET_TRANSPORT_Handle *transport;
64 * Proxy that we are using (can be NULL).
69 * Buffer for data downloaded via HTTP.
71 static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE];
74 * Number of bytes valid in 'download_buffer'.
76 static size_t download_pos;
79 * Current URL that we are using.
81 static char *current_url;
84 * Current CURL handle.
89 * Current multi-CURL handle.
94 * ID of the current task scheduled.
96 static GNUNET_SCHEDULER_TaskIdentifier current_task;
99 * Amount of time we wait between hostlist downloads.
101 static struct GNUNET_TIME_Relative hostlist_delay;
104 * Set to GNUNET_YES if the current URL had some problems.
106 static int bogus_url;
109 * Number of active connections (according to core service).
111 static unsigned int connection_count;
114 * At what time MUST the current hostlist request be done?
116 static struct GNUNET_TIME_Absolute end_time;
120 * Process downloaded bits by calling callback on each HELLO.
122 * @param ptr buffer with downloaded data
123 * @param size size of a record
124 * @param nmemb number of records downloaded
126 * @return number of bytes that were processed (always size*nmemb)
129 download_hostlist_processor (void *ptr,
134 const char * cbuf = ptr;
135 const struct GNUNET_MessageHeader *msg;
141 total = size * nmemb;
142 if ( (total == 0) || (bogus_url) )
144 return total; /* ok, no data or bogus data */
146 GNUNET_STATISTICS_update (stats,
147 gettext_noop ("# bytes downloaded from hostlist servers"),
151 while ( (left > 0) ||
154 cpy = GNUNET_MIN (total, GNUNET_SERVER_MAX_MESSAGE_SIZE - download_pos);
155 GNUNET_assert (cpy > 0);
156 memcpy (&download_buffer[download_pos],
162 if (download_pos < sizeof(struct GNUNET_MessageHeader))
164 GNUNET_assert (left == 0);
167 msg = (const struct GNUNET_MessageHeader *) download_buffer;
168 msize = ntohs(msg->size);
169 if (msize < sizeof(struct GNUNET_MessageHeader))
171 GNUNET_STATISTICS_update (stats,
172 gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
175 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
176 _("Invalid `%s' message received from hostlist at `%s'\n"),
182 if (download_pos < msize)
184 GNUNET_assert (left == 0);
187 if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message*)msg) == msize)
189 #if DEBUG_HOSTLIST_CLIENT
190 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
191 "Received valid `%s' message from hostlist server.\n",
194 GNUNET_STATISTICS_update (stats,
195 gettext_noop ("# valid HELLOs downloaded from hostlist servers"),
198 GNUNET_TRANSPORT_offer_hello (transport, msg);
202 GNUNET_STATISTICS_update (stats,
203 gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
206 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
207 _("Invalid `%s' message received from hostlist at `%s'\n"),
210 bogus_url = GNUNET_YES;
213 memmove (download_buffer,
214 &download_buffer[msize],
215 download_pos - msize);
216 download_pos -= msize;
223 * Obtain a hostlist URL that we should use.
225 * @return NULL if there is no URL available
236 GNUNET_CONFIGURATION_get_value_string (cfg,
241 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
242 _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
243 "SERVERS", "HOSTLIST");
248 if (strlen (servers) > 0)
251 pos = strlen (servers) - 1;
254 if (servers[pos] == ' ')
261 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
262 _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
263 "SERVERS", "HOSTLIST");
264 GNUNET_free (servers);
268 urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
269 pos = strlen (servers) - 1;
272 if (servers[pos] == ' ')
284 ret = GNUNET_strdup (&servers[pos]);
285 GNUNET_free (servers);
290 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt(c, a, b); if (ret != CURLE_OK) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror(ret)); } while (0);
294 * Schedule the background task that will (possibly)
295 * download a hostlist.
298 schedule_hostlist_task (void);
302 * Clean up the state from the task that downloaded the
303 * hostlist and schedule the next task.
312 mret = curl_multi_remove_handle (multi, curl);
313 if (mret != CURLM_OK)
315 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
316 _("%s failed at %s:%d: `%s'\n"),
317 "curl_multi_remove_handle", __FILE__, __LINE__,
318 curl_multi_strerror (mret));
320 mret = curl_multi_cleanup (multi);
321 if (mret != CURLM_OK)
322 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
323 _("%s failed at %s:%d: `%s'\n"),
324 "curl_multi_cleanup", __FILE__, __LINE__,
325 curl_multi_strerror (mret));
330 curl_easy_cleanup (curl);
333 GNUNET_free_non_null (current_url);
335 schedule_hostlist_task ();
340 * Ask CURL for the select set and then schedule the
341 * receiving task with the scheduler.
348 * Task that is run when we are ready to receive more data from the hostlist
351 * @param cls closure, unused
352 * @param tc task context, unused
355 multi_ready (void *cls,
356 const struct GNUNET_SCHEDULER_TaskContext *tc)
362 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
364 #if DEBUG_HOSTLIST_CLIENT
365 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
366 "Shutdown requested while trying to download hostlist from `%s'\n",
372 if (GNUNET_TIME_absolute_get_remaining (end_time).value == 0)
374 GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
375 _("Timeout trying to download hostlist from `%s'\n"),
380 #if DEBUG_HOSTLIST_CLIENT
381 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
382 "Ready for processing hostlist client request\n");
387 mret = curl_multi_perform (multi, &running);
392 msg = curl_multi_info_read (multi, &running);
393 GNUNET_break (msg != NULL);
399 if ( (msg->data.result != CURLE_OK) &&
400 (msg->data.result != CURLE_GOT_NOTHING) )
401 GNUNET_log(GNUNET_ERROR_TYPE_INFO,
402 _("%s failed for `%s' at %s:%d: `%s'\n"),
403 "curl_multi_perform",
407 curl_easy_strerror (msg->data.result));
409 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
410 _("Download of hostlist `%s' completed.\n"),
421 while (mret == CURLM_CALL_MULTI_PERFORM);
422 if (mret != CURLM_OK)
424 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
425 _("%s failed at %s:%d: `%s'\n"),
426 "curl_multi_perform", __FILE__, __LINE__,
427 curl_multi_strerror (mret));
435 * Ask CURL for the select set and then schedule the
436 * receiving task with the scheduler.
446 struct GNUNET_NETWORK_FDSet *grs;
447 struct GNUNET_NETWORK_FDSet *gws;
449 struct GNUNET_TIME_Relative rtime;
455 mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
456 if (mret != CURLM_OK)
458 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
459 _("%s failed at %s:%d: `%s'\n"),
460 "curl_multi_fdset", __FILE__, __LINE__,
461 curl_multi_strerror (mret));
465 mret = curl_multi_timeout (multi, &timeout);
466 if (mret != CURLM_OK)
468 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
469 _("%s failed at %s:%d: `%s'\n"),
470 "curl_multi_timeout", __FILE__, __LINE__,
471 curl_multi_strerror (mret));
475 rtime = GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
476 GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
478 grs = GNUNET_NETWORK_fdset_create ();
479 gws = GNUNET_NETWORK_fdset_create ();
480 GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
481 GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
482 #if DEBUG_HOSTLIST_CLIENT
483 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
484 "Scheduling task for hostlist download using cURL\n");
487 = GNUNET_SCHEDULER_add_select (sched,
488 GNUNET_SCHEDULER_PRIORITY_DEFAULT,
489 GNUNET_SCHEDULER_NO_TASK,
495 GNUNET_NETWORK_fdset_destroy (gws);
496 GNUNET_NETWORK_fdset_destroy (grs);
501 * Main function that will download a hostlist and process its
510 curl = curl_easy_init ();
518 current_url = get_url ();
519 GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
520 _("Bootstrapping using hostlist at `%s'.\n"),
522 GNUNET_STATISTICS_update (stats,
523 gettext_noop ("# hostlist downloads initiated"),
527 CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
530 CURL_EASY_SETOPT (curl,
531 CURLOPT_WRITEFUNCTION,
532 &download_hostlist_processor);
538 CURL_EASY_SETOPT (curl,
546 CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
547 CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
548 /* no need to abort if the above failed */
549 CURL_EASY_SETOPT (curl,
557 CURL_EASY_SETOPT (curl,
561 CURL_EASY_SETOPT (curl,
565 CURL_EASY_SETOPT (curl,
567 GNUNET_SERVER_MAX_MESSAGE_SIZE);
568 if (0 == strncmp (current_url, "http", 4))
569 CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
570 CURL_EASY_SETOPT (curl,
571 CURLOPT_CONNECTTIMEOUT,
573 CURL_EASY_SETOPT (curl,
577 /* this should no longer be needed; we're now single-threaded! */
578 CURL_EASY_SETOPT (curl,
582 multi = curl_multi_init ();
589 mret = curl_multi_add_handle (multi, curl);
590 if (mret != CURLM_OK)
592 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
593 _("%s failed at %s:%d: `%s'\n"),
594 "curl_multi_add_handle", __FILE__, __LINE__,
595 curl_multi_strerror (mret));
596 mret = curl_multi_cleanup (multi);
597 if (mret != CURLM_OK)
598 GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
599 _("%s failed at %s:%d: `%s'\n"),
600 "curl_multi_cleanup", __FILE__, __LINE__,
601 curl_multi_strerror (mret));
606 end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
612 * Task that checks if we should try to download a hostlist.
613 * If so, we initiate the download, otherwise we schedule
614 * this task again for a later time.
617 check_task (void *cls,
618 const struct GNUNET_SCHEDULER_TaskContext *tc)
620 current_task = GNUNET_SCHEDULER_NO_TASK;
621 if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
623 if (connection_count < MIN_CONNECTIONS)
624 download_hostlist ();
626 schedule_hostlist_task ();
631 * Compute when we should check the next time about downloading
632 * a hostlist; then schedule the task accordingly.
635 schedule_hostlist_task ()
638 struct GNUNET_TIME_Relative delay;
642 curl_global_cleanup ();
643 return; /* in shutdown */
645 delay = hostlist_delay;
646 if (hostlist_delay.value == 0)
647 hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
649 hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
650 if (hostlist_delay.value > GNUNET_TIME_UNIT_HOURS.value * (1 + connection_count))
651 hostlist_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
652 (1 + connection_count));
653 GNUNET_STATISTICS_set (stats,
654 gettext_noop("Minimum time between hostlist downloads"),
655 hostlist_delay.value,
659 delay = GNUNET_TIME_UNIT_ZERO;
662 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
663 _("Have %u/%u connections. Will consider downloading hostlist in %llums\n"),
666 (unsigned long long) delay.value);
667 current_task = GNUNET_SCHEDULER_add_delayed (sched,
675 * Method called whenever a given peer connects.
678 * @param peer peer identity this notification is about
679 * @param latency reported latency of the connection with 'other'
680 * @param distance reported distance (DV) to 'other'
683 connect_handler (void *cls,
685 GNUNET_PeerIdentity * peer,
686 struct GNUNET_TIME_Relative latency,
690 GNUNET_STATISTICS_update (stats,
691 gettext_noop ("# active connections"),
698 * Method called whenever a given peer connects.
701 * @param peer peer identity this notification is about
704 disconnect_handler (void *cls,
706 GNUNET_PeerIdentity * peer)
709 GNUNET_STATISTICS_update (stats,
710 gettext_noop ("# active connections"),
717 * Continuation called by the statistics code once
718 * we go the stat. Initiates hostlist download scheduling.
721 * @param success GNUNET_OK if statistics were
722 * successfully obtained, GNUNET_SYSERR if not.
725 primary_task (void *cls, int success)
728 return; /* in shutdown */
729 #if DEBUG_HOSTLIST_CLIENT
730 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
731 "Statistics request done, scheduling hostlist download\n");
733 schedule_hostlist_task ();
738 process_stat (void *cls,
739 const char *subsystem,
744 GNUNET_log (GNUNET_ERROR_TYPE_INFO,
745 _("Initial time between hostlist downloads is %llums\n"),
746 (unsigned long long) value);
747 hostlist_delay.value = value;
753 * Start downloading hostlists from hostlist servers as necessary.
756 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
757 struct GNUNET_SCHEDULER_Handle *s,
758 struct GNUNET_STATISTICS_Handle *st,
759 GNUNET_CORE_ConnectEventHandler *ch,
760 GNUNET_CORE_DisconnectEventHandler *dh)
762 if (0 != curl_global_init (CURL_GLOBAL_WIN32))
765 return GNUNET_SYSERR;
767 transport = GNUNET_TRANSPORT_connect (s, c, NULL, NULL, NULL, NULL);
768 if (NULL == transport)
770 curl_global_cleanup ();
771 return GNUNET_SYSERR;
777 GNUNET_CONFIGURATION_get_value_string (cfg,
782 *ch = &connect_handler;
783 *dh = &disconnect_handler;
784 GNUNET_STATISTICS_get (stats,
786 gettext_noop("Minimum time between hostlist downloads"),
787 GNUNET_TIME_UNIT_MINUTES,
796 * Stop downloading hostlists from hostlist servers as necessary.
799 GNUNET_HOSTLIST_client_stop ()
801 #if DEBUG_HOSTLIST_CLIENT
802 GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
803 "Hostlist client shutdown\n");
805 if (current_task != GNUNET_SCHEDULER_NO_TASK)
807 GNUNET_SCHEDULER_cancel (sched,
809 curl_global_cleanup ();
811 if (transport != NULL)
813 GNUNET_TRANSPORT_disconnect (transport);
816 GNUNET_assert (NULL == transport);
817 GNUNET_free_non_null (proxy);
823 /* end of hostlist-client.c */