src/hostlist/hostlist-client.c

   1 /*
   2      This file is part of GNUnet.
   3      (C) 2001, 2002, 2003, 2004, 2005, 2006, 2009, 2010 Christian Grothoff (and other contributing authors)
   4
   5      GNUnet is free software; you can redistribute it and/or modify
   6      it under the terms of the GNU General Public License as published
   7      by the Free Software Foundation; either version 2, or (at your
   8      option) any later version.
   9
  10      GNUnet is distributed in the hope that it will be useful, but
  11      WITHOUT ANY WARRANTY; without even the implied warranty of
  12      MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13      General Public License for more details.
  14
  15      You should have received a copy of the GNU General Public License
  16      along with GNUnet; see the file COPYING.  If not, write to the
  17      Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  18      Boston, MA 02111-1307, USA.
  19 */
  20
  21 /**
  22  * @file hostlist/hostlist-client.c
  23  * @brief hostlist support.  Downloads HELLOs via HTTP.
  24  * @author Christian Grothoff
  25  * @author Matthias Wachs
  26  */
  27
  28 #include "platform.h"
  29 #include "hostlist-client.h"
  30 #include "gnunet_core_service.h"
  31 #include "gnunet_hello_lib.h"
  32 #include "gnunet_statistics_service.h"
  33 #include "gnunet_transport_service.h"
  34 #include "gnunet-daemon-hostlist.h"
  35 #include <curl/curl.h>
  36 #include "gnunet_common.h"
  37 #include "gnunet_bio_lib.h"
  38
  39 #define DEBUG_HOSTLIST_CLIENT GNUNET_YES
  40
  41
  42 /**
  43  * Number of connections that we must have to NOT download
  44  * hostlists anymore.
  45  */
  46 #define MIN_CONNECTIONS 4
  47
  48 /**
  49  * A single hostlist obtained by hostlist advertisements
  50  */
  51 struct Hostlist
  52 {
  53   /**
  54    * previous entry, used to manage entries in a double linked list
  55    */
  56   struct Hostlist * prev;
  57
  58   /**
  59    * next entry, used to manage entries in a double linked list
  60    */
  61   struct Hostlist * next;
  62
  63   /**
  64    * URI where hostlist can be obtained
  65    */
  66   const char *hostlist_uri;
  67
  68   /**
  69    * Value describing the quality of the hostlist, the bigger the better but (should) never < 0
  70    * used for deciding which hostlist is replaced if MAX_NUMBER_HOSTLISTS in data structure is reached
  71    * intial value = HOSTLIST_INITIAL
  72    * increased every successful download by HOSTLIST_SUCCESSFULL_DOWNLOAD
  73    * increased every successful download by number of obtained HELLO messages
  74    * decreased every failed download by HOSTLIST_SUCCESSFULL_DOWNLOAD
  75    */
  76   uint64_t quality;
  77
  78   /**
  79    * Time the hostlist advertisement was recieved and the entry was created
  80    */
  81   struct GNUNET_TIME_Absolute time_creation;
  82
  83   /**
  84    * Last time the hostlist was obtained
  85    */
  86   struct GNUNET_TIME_Absolute time_last_usage;
  87
  88   /**
  89    * Number of HELLO messages obtained during last download
  90    */
  91   uint32_t hello_count;
  92
  93   /**
  94    * Number of times the hostlist was successfully obtained
  95    */
  96   uint32_t times_used;
  97
  98 };
  99
 100
 101 /**
 102  * Our configuration.
 103  */
 104 static const struct GNUNET_CONFIGURATION_Handle *cfg;
 105
 106 /**
 107  * Our scheduler.
 108  */
 109 static struct GNUNET_SCHEDULER_Handle *sched;
 110
 111 /**
 112  * Statistics handle.
 113  */
 114 struct GNUNET_STATISTICS_Handle *stats;
 115
 116 /**
 117  * Transport handle.
 118  */
 119 struct GNUNET_TRANSPORT_Handle *transport;
 120
 121 /**
 122  * Proxy that we are using (can be NULL).
 123  */
 124 static char *proxy;
 125
 126 /**
 127  * Buffer for data downloaded via HTTP.
 128  */
 129 static char download_buffer[GNUNET_SERVER_MAX_MESSAGE_SIZE];
 130
 131 /**
 132  * Number of bytes valid in 'download_buffer'.
 133  */
 134 static size_t download_pos;
 135
 136 /**
 137  * Current URL that we are using.
 138  */
 139 static char *current_url;
 140
 141 /**
 142  * Current CURL handle.
 143  */
 144 static CURL *curl;
 145
 146 /**
 147  * Current multi-CURL handle.
 148  */
 149 static CURLM *multi;
 150
 151 /**
 152  * ID of the current task scheduled.
 153  */
 154 static GNUNET_SCHEDULER_TaskIdentifier ti_check_download;
 155
 156 /**
 157  * ID of the current task scheduled.
 158  */
 159 static GNUNET_SCHEDULER_TaskIdentifier ti_download;
 160
 161 /**
 162  * ID of the current hostlist saving task scheduled.
 163  */
 164 static GNUNET_SCHEDULER_TaskIdentifier ti_saving_task;
 165
 166 /**
 167  * ID of the current hostlist saving task scheduled.
 168  */
 169 static GNUNET_SCHEDULER_TaskIdentifier ti_download_dispatcher_task;
 170
 171
 172 /**
 173  * ID of the task checking the intervall between to hostlist tests
 174  */
 175 static GNUNET_SCHEDULER_TaskIdentifier ti_testing_intervall_task;
 176
 177 /**
 178  * Amount of time we wait between hostlist downloads.
 179  */
 180 static struct GNUNET_TIME_Relative hostlist_delay;
 181
 182 /**
 183  * Set to GNUNET_YES if the current URL had some problems.
 184  */
 185 static int bogus_url;
 186
 187 /**
 188  * Number of active connections (according to core service).
 189  */
 190 static unsigned int connection_count;
 191
 192 /**
 193  * At what time MUST the current hostlist request be done?
 194  */
 195 static struct GNUNET_TIME_Absolute end_time;
 196
 197 /**
 198  * Head of the linked list used to store hostlists
 199  */
 200 static struct Hostlist * linked_list_head;
 201
 202 /**
 203  *  Tail of the linked list used to store hostlists
 204  */
 205 static struct Hostlist * linked_list_tail;
 206
 207 /**
 208  *  Current hostlist used for downloading
 209  */
 210 static struct Hostlist * current_hostlist;
 211
 212 /**
 213  *  Size of the linke list  used to store hostlists
 214  */
 215 static unsigned int linked_list_size;
 216
 217 /**
 218  * Head of the linked list used to store hostlists
 219  */
 220 static struct Hostlist * hostlist_to_test;
 221
 222 static int testing_hostlist;
 223
 224 static int testing_allowed;
 225
 226 static int download_in_progress;
 227
 228 /**
 229  * Value saying if preconfigured  is used
 230  */
 231 static unsigned int use_preconfigured_list;
 232
 233 /**
 234  * Set if we are allowed to learn new hostlists and use them
 235  */
 236 static int learning;
 237
 238 /**
 239  * Value saying how many valid HELLO messages were obtained during download
 240  */
 241 static unsigned int hellos_obtained;
 242
 243 /**
 244  * Value saying if hostlist download was successful
 245  */
 246 static unsigned int download_successful;
 247
 248 /**
 249  * Process downloaded bits by calling callback on each HELLO.
 250  *
 251  * @param ptr buffer with downloaded data
 252  * @param size size of a record
 253  * @param nmemb number of records downloaded
 254  * @param ctx unused
 255  * @return number of bytes that were processed (always size*nmemb)
 256  */
 257 static size_t
 258 download_hostlist_processor (void *ptr,
 259                              size_t size,
 260                              size_t nmemb,
 261                              void *ctx)
 262 {
 263   const char * cbuf = ptr;
 264   const struct GNUNET_MessageHeader *msg;
 265   size_t total;
 266   size_t cpy;
 267   size_t left;
 268   uint16_t msize;
 269
 270   total = size * nmemb;
 271   if ( (total == 0) || (bogus_url) )
 272     {
 273       return total;  /* ok, no data or bogus data */
 274     }
 275   GNUNET_STATISTICS_update (stats,
 276                             gettext_noop ("# bytes downloaded from hostlist servers"),
 277                             (int64_t) total,
 278                             GNUNET_NO);
 279   left = total;
 280   while ( (left > 0) ||
 281           (download_pos > 0) )
 282     {
 283       cpy = GNUNET_MIN (left, GNUNET_SERVER_MAX_MESSAGE_SIZE - download_pos);
 284       memcpy (&download_buffer[download_pos],
 285               cbuf,
 286               cpy);
 287       cbuf += cpy;
 288       download_pos += cpy;
 289       left -= cpy;
 290       if (download_pos < sizeof(struct GNUNET_MessageHeader))
 291         {
 292           GNUNET_assert (left == 0);
 293           break;
 294         }
 295       msg = (const struct GNUNET_MessageHeader *) download_buffer;
 296       msize = ntohs(msg->size);
 297       if (msize < sizeof(struct GNUNET_MessageHeader))
 298         {
 299           GNUNET_STATISTICS_update (stats,
 300                                     gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
 301                                     1,
 302                                     GNUNET_NO);
 303           GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 304                       _("Invalid `%s' message received from hostlist at `%s'\n"),
 305                       "HELLO",
 306                       current_url);
 307           bogus_url = 1;
 308           return total;
 309         }
 310       if (download_pos < msize)
 311         {
 312           GNUNET_assert (left == 0);
 313           break;
 314         }
 315       if (GNUNET_HELLO_size ((const struct GNUNET_HELLO_Message*)msg) == msize)
 316         {
 317 #if DEBUG_HOSTLIST_CLIENT
 318           GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 319                       "Received valid `%s' message from hostlist server.\n",
 320                       "HELLO");
 321 #endif
 322           GNUNET_STATISTICS_update (stats,
 323                                     gettext_noop ("# valid HELLOs downloaded from hostlist servers"),
 324                                     1,
 325                                     GNUNET_NO);
 326           hellos_obtained++;
 327           GNUNET_TRANSPORT_offer_hello (transport, msg);
 328         }
 329       else
 330         {
 331           GNUNET_STATISTICS_update (stats,
 332                                     gettext_noop ("# invalid HELLOs downloaded from hostlist servers"),
 333                                     1,
 334                                     GNUNET_NO);
 335           GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 336                       _("Invalid `%s' message received from hostlist at `%s'\n"),
 337                       "HELLO",
 338                       current_url);
 339           bogus_url = GNUNET_YES;
 340           return total;
 341         }
 342       memmove (download_buffer,
 343                &download_buffer[msize],
 344                download_pos - msize);
 345       download_pos -= msize;
 346     }
 347   return total;
 348 }
 349
 350
 351 /**
 352  * Obtain a hostlist URL that we should use.
 353  *
 354  * @return NULL if there is no URL available
 355  */
 356 static char *
 357 get_bootstrap_url ()
 358 {
 359   char *servers;
 360   char *ret;
 361   size_t urls;
 362   size_t pos;
 363
 364   if (GNUNET_OK !=
 365       GNUNET_CONFIGURATION_get_value_string (cfg,
 366                                              "HOSTLIST",
 367                                              "SERVERS",
 368                                              &servers))
 369     {
 370       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 371                   _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
 372                   "SERVERS", "HOSTLIST");
 373       return NULL;
 374     }
 375
 376   urls = 0;
 377   if (strlen (servers) > 0)
 378     {
 379       urls++;
 380       pos = strlen (servers) - 1;
 381       while (pos > 0)
 382         {
 383           if (servers[pos] == ' ')
 384             urls++;
 385           pos--;
 386         }
 387     }
 388   if (urls == 0)
 389     {
 390       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 391                   _("No `%s' specified in `%s' configuration, will not bootstrap.\n"),
 392                   "SERVERS", "HOSTLIST");
 393       GNUNET_free (servers);
 394       return NULL;
 395     }
 396
 397   urls = GNUNET_CRYPTO_random_u32 (GNUNET_CRYPTO_QUALITY_WEAK, urls) + 1;
 398   pos = strlen (servers) - 1;
 399   while (pos > 0)
 400     {
 401       if (servers[pos] == ' ')
 402         {
 403           urls--;
 404           servers[pos] = '\0';
 405         }
 406       if (urls == 0)
 407         {
 408           pos++;
 409           break;
 410         }
 411       pos--;
 412     }
 413   ret = GNUNET_strdup (&servers[pos]);
 414   GNUNET_free (servers);
 415   return ret;
 416 }
 417
 418 /**
 419  * Method deciding if a preconfigured or advertisied hostlist is used on a 50:50 ratio
 420  * @return uri to use, NULL if there is no URL available
 421  */
 422 static char *
 423 get_list_url ()
 424 {
 425   uint32_t index;
 426   unsigned int counter;
 427   struct Hostlist * pos;
 428
 429   if ( GNUNET_NO == learning)
 430   {
 431     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 432                 "Using preconfigured bootstrap server\n");
 433     current_hostlist = NULL;
 434     return get_bootstrap_url();
 435   }
 436
 437   if ( ( GNUNET_YES == testing_hostlist) && (NULL != hostlist_to_test) )
 438   {
 439     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 440                 "Testing new advertised hostlist if it is obtainable\n");
 441     current_hostlist = hostlist_to_test;
 442     return strdup(hostlist_to_test->hostlist_uri);
 443   }
 444
 445   if ( (GNUNET_YES == use_preconfigured_list) ||
 446        (linked_list_size == 0) )
 447   {
 448     GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 449                 "Using preconfigured bootstrap server\n");
 450     current_hostlist = NULL;
 451     return get_bootstrap_url();
 452   }
 453   index = GNUNET_CRYPTO_random_u32 ( GNUNET_CRYPTO_QUALITY_WEAK, linked_list_size);
 454   counter = 0;
 455   pos = linked_list_head;
 456   while ( counter < index )
 457     {
 458       pos = pos->next;
 459       counter ++;
 460     }
 461   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 462               "Using learned hostlist `%s'\n", pos->hostlist_uri);
 463   current_hostlist = pos;
 464   return strdup(pos->hostlist_uri);
 465 }
 466
 467
 468 #define CURL_EASY_SETOPT(c, a, b) do { ret = curl_easy_setopt(c, a, b); if (ret != CURLE_OK) GNUNET_log(GNUNET_ERROR_TYPE_WARNING, _("%s failed at %s:%d: `%s'\n"), "curl_easy_setopt", __FILE__, __LINE__, curl_easy_strerror(ret)); } while (0);
 469
 470 /**
 471  * Method to load persistent hostlist file during hostlist client shutdown
 472  * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
 473  */
 474 static void save_hostlist_file ( int shutdown );
 475
 476 /**
 477  * add val2 to val1 with overflow check
 478  * @param val1 value 1
 479  * @param val2 value 2
 480  * @return result
 481  */
 482 static uint64_t checked_add (uint64_t val1, uint64_t val2)
 483 {
 484   static uint64_t temp;
 485   static uint64_t maxv;
 486
 487   maxv = 0;
 488   maxv--;
 489
 490   temp = val1+val2;
 491   if ( temp < val1)
 492     return maxv;
 493   else
 494     return temp;
 495 }
 496
 497 /**
 498  * Subtract val2 from val1 with underflow check
 499  * @param val1 value 1
 500  * @param val2 value 2
 501  * @return result
 502  */
 503 static uint64_t checked_sub (uint64_t val1, uint64_t val2)
 504 {
 505   if ( val1 <= val2)
 506     return 0;
 507   else
 508     return (val1-val2);
 509 }
 510
 511 /**
 512  * Method to check if URI is in hostlist linked list
 513  * @param uri uri to check
 514  * @return GNUNET_YES if existing in linked list, GNUNET_NO if not
 515  */
 516 static int
 517 linked_list_contains (const char * uri)
 518 {
 519   struct Hostlist * pos;
 520
 521   pos = linked_list_head;
 522   while (pos != NULL)
 523     {
 524       if (0 == strcmp(pos->hostlist_uri, uri) )
 525         return GNUNET_YES;
 526       pos = pos->next;
 527     }
 528   return GNUNET_NO;
 529 }
 530
 531
 532 /**
 533  * Method returning the uri with the lowest quality in the datastore
 534  * @return hostlist with lowest quality
 535  */
 536 static struct Hostlist *
 537 linked_list_get_lowest_quality ( )
 538 {
 539   struct Hostlist * pos;
 540   struct Hostlist * lowest;
 541
 542   if (linked_list_size == 0)
 543     return NULL;
 544   lowest = linked_list_head;
 545   pos = linked_list_head->next;
 546   while (pos != NULL)
 547     {
 548       if (pos->quality < lowest->quality)
 549         lowest = pos;
 550       pos = pos->next;
 551     }
 552   return lowest;
 553 }
 554
 555
 556 /**
 557  * Method to insert a hostlist into the datastore. If datastore contains maximum number of elements, the elements with lowest quality is dismissed
 558  */
 559 static void
 560 insert_hostlist ( void )
 561 {
 562   GNUNET_CONTAINER_DLL_insert(linked_list_head, linked_list_tail, hostlist_to_test);
 563   linked_list_size++;
 564
 565   GNUNET_STATISTICS_set (stats,
 566                          gettext_noop("# advertised hostlist URIs"),
 567                          linked_list_size,
 568                          GNUNET_NO);
 569
 570   if (MAX_NUMBER_HOSTLISTS >= linked_list_size)
 571     return;
 572
 573   /* No free entries available, replace existing entry  */
 574   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 575               "Removing lowest quality entry\n" );
 576   struct Hostlist * lowest_quality = linked_list_get_lowest_quality();
 577   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 578               "Hostlist with URI `%s' has the worst quality of all with value %llu\n",
 579               lowest_quality->hostlist_uri,
 580               (unsigned long long) lowest_quality->quality);
 581   GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, lowest_quality);
 582   linked_list_size--;
 583
 584   GNUNET_STATISTICS_set (stats,
 585                          gettext_noop("# advertised hostlist URIs"),
 586                          linked_list_size,
 587                          GNUNET_NO);
 588
 589   GNUNET_free (lowest_quality);
 590
 591   testing_hostlist = GNUNET_NO;
 592   return;
 593 }
 594
 595
 596 /**
 597  * Method updating hostlist statistics
 598  */
 599 static void update_hostlist ( )
 600 {
 601   char *stat;
 602   if ( ((use_preconfigured_list == GNUNET_NO) && ( NULL != current_hostlist )) ||
 603        ((testing_hostlist == GNUNET_YES) && ( NULL != current_hostlist )) )
 604   {
 605     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 606                 "Updating hostlist statics for URI `%s'\n",current_hostlist->hostlist_uri );
 607      current_hostlist->hello_count = hellos_obtained;
 608      current_hostlist->time_last_usage = GNUNET_TIME_absolute_get();
 609      current_hostlist->quality = checked_add ( current_hostlist->quality, (hellos_obtained * HOSTLIST_SUCCESSFUL_HELLO));
 610      if ( GNUNET_YES == download_successful )
 611      {
 612        current_hostlist->times_used++;
 613        current_hostlist->quality = checked_add ( current_hostlist->quality, HOSTLIST_SUCCESSFUL_DOWNLOAD);
 614        GNUNET_asprintf (&stat,
 615                         gettext_noop("# advertised URI `%s' downloaded"),
 616                         current_hostlist->hostlist_uri);
 617
 618        GNUNET_STATISTICS_update ( stats,
 619                                   stat,
 620                                   1,
 621                                   GNUNET_YES);
 622        GNUNET_free (stat);
 623      }
 624      else
 625        current_hostlist->quality = checked_sub ( current_hostlist->quality, HOSTLIST_FAILED_DOWNLOAD );
 626   }
 627   current_hostlist = NULL;
 628   /* Alternating the usage of preconfigured and learned hostlists */
 629
 630   if (testing_hostlist == GNUNET_YES)
 631     return;
 632
 633   if ( GNUNET_YES == learning)
 634     {
 635     if (use_preconfigured_list == GNUNET_YES)
 636       use_preconfigured_list = GNUNET_NO;
 637     else
 638       use_preconfigured_list = GNUNET_YES;
 639     }
 640   else
 641     use_preconfigured_list = GNUNET_YES;
 642 }
 643
 644 /**
 645  * Clean up the state from the task that downloaded the
 646  * hostlist and schedule the next task.
 647  */
 648 static void
 649 clean_up ()
 650 {
 651   CURLMcode mret;
 652
 653   if ( ( testing_hostlist == GNUNET_YES ) && ( GNUNET_NO == download_successful) && (NULL != hostlist_to_test))
 654   {
 655     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 656                 _("Advertised hostlist with URI `%s' could not be downloaded. Advertised URI gets dismissed.\n"),hostlist_to_test->hostlist_uri);
 657   }
 658
 659   if ( testing_hostlist == GNUNET_YES )
 660     {
 661     testing_hostlist = GNUNET_NO;
 662     }
 663   if ( NULL != hostlist_to_test)
 664   {
 665     GNUNET_free (hostlist_to_test);
 666     hostlist_to_test = NULL;
 667   }
 668
 669
 670   if (multi != NULL)
 671     {
 672       mret = curl_multi_remove_handle (multi, curl);
 673       if (mret != CURLM_OK)
 674         {
 675           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 676                       _("%s failed at %s:%d: `%s'\n"),
 677                       "curl_multi_remove_handle", __FILE__, __LINE__,
 678                       curl_multi_strerror (mret));
 679         }
 680       mret = curl_multi_cleanup (multi);
 681       if (mret != CURLM_OK)
 682         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 683                     _("%s failed at %s:%d: `%s'\n"),
 684                     "curl_multi_cleanup", __FILE__, __LINE__,
 685                     curl_multi_strerror (mret));
 686       multi = NULL;
 687     }
 688   if (curl != NULL)
 689     {
 690       curl_easy_cleanup (curl);
 691       curl = NULL;
 692     }
 693   GNUNET_free_non_null (current_url);
 694   current_url = NULL;
 695
 696   download_in_progress = GNUNET_NO;
 697 }
 698
 699 /**
 700  * Task that is run when we are ready to receive more data from the hostlist
 701  * server.
 702  *
 703  * @param cls closure, unused
 704  * @param tc task context, unused
 705  */
 706 static void
 707 multi_ready (void *cls,
 708              const struct GNUNET_SCHEDULER_TaskContext *tc);
 709
 710 /**
 711  * Ask CURL for the select set and then schedule the
 712  * receiving task with the scheduler.
 713  */
 714 static void
 715 run_multi ()
 716 {
 717   CURLMcode mret;
 718   fd_set rs;
 719   fd_set ws;
 720   fd_set es;
 721   int max;
 722   struct GNUNET_NETWORK_FDSet *grs;
 723   struct GNUNET_NETWORK_FDSet *gws;
 724   long timeout;
 725   struct GNUNET_TIME_Relative rtime;
 726
 727   max = -1;
 728   FD_ZERO (&rs);
 729   FD_ZERO (&ws);
 730   FD_ZERO (&es);
 731   mret = curl_multi_fdset (multi, &rs, &ws, &es, &max);
 732   if (mret != CURLM_OK)
 733     {
 734       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 735                   _("%s failed at %s:%d: `%s'\n"),
 736                   "curl_multi_fdset", __FILE__, __LINE__,
 737                   curl_multi_strerror (mret));
 738       clean_up ();
 739       return;
 740     }
 741   mret = curl_multi_timeout (multi, &timeout);
 742   if (mret != CURLM_OK)
 743     {
 744       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 745                   _("%s failed at %s:%d: `%s'\n"),
 746                   "curl_multi_timeout", __FILE__, __LINE__,
 747                   curl_multi_strerror (mret));
 748       clean_up ();
 749       return;
 750     }
 751   rtime = GNUNET_TIME_relative_min (GNUNET_TIME_absolute_get_remaining (end_time),
 752                                     GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_MILLISECONDS,
 753                                                                    timeout));
 754   grs = GNUNET_NETWORK_fdset_create ();
 755   gws = GNUNET_NETWORK_fdset_create ();
 756   GNUNET_NETWORK_fdset_copy_native (grs, &rs, max + 1);
 757   GNUNET_NETWORK_fdset_copy_native (gws, &ws, max + 1);
 758 #if DEBUG_HOSTLIST_CLIENT
 759   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 760               "Scheduling task for hostlist download using cURL\n");
 761 #endif
 762   ti_download
 763     = GNUNET_SCHEDULER_add_select (sched,
 764                                    GNUNET_SCHEDULER_PRIORITY_DEFAULT,
 765                                    GNUNET_SCHEDULER_NO_TASK,
 766                                    rtime,
 767                                    grs,
 768                                    gws,
 769                                    &multi_ready,
 770                                    multi);
 771   GNUNET_NETWORK_fdset_destroy (gws);
 772   GNUNET_NETWORK_fdset_destroy (grs);
 773 }
 774
 775
 776 /**
 777  * Task that is run when we are ready to receive more data from the hostlist
 778  * server.
 779  *
 780  * @param cls closure, unused
 781  * @param tc task context, unused
 782  */
 783 static void
 784 multi_ready (void *cls,
 785              const struct GNUNET_SCHEDULER_TaskContext *tc)
 786 {
 787   unsigned int counter;
 788   int running;
 789   struct CURLMsg *msg;
 790   CURLMcode mret;
 791
 792
 793   ti_download = GNUNET_SCHEDULER_NO_TASK;
 794   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
 795     {
 796 #if DEBUG_HOSTLIST_CLIENT
 797       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 798                   "Shutdown requested while trying to download hostlist from `%s'\n",
 799                   current_url);
 800 #endif
 801       update_hostlist();
 802       clean_up ();
 803       return;
 804     }
 805   if (GNUNET_TIME_absolute_get_remaining (end_time).value == 0)
 806     {
 807       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
 808                   _("Timeout trying to download hostlist from `%s'\n"),
 809                   current_url);
 810       update_hostlist();
 811       clean_up ();
 812       return;
 813     }
 814 #if DEBUG_HOSTLIST_CLIENT
 815   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
 816               "Ready for processing hostlist client request\n");
 817 #endif
 818   counter = 0;
 819   do
 820     {
 821       running = 0;
 822       mret = curl_multi_perform (multi, &running);
 823       if (running == 0)
 824         {
 825           do
 826             {
 827
 828               msg = curl_multi_info_read (multi, &running);
 829               counter ++;
 830               GNUNET_break (msg != NULL);
 831               if (msg == NULL)
 832                 break;
 833               switch (msg->msg)
 834                 {
 835                 case CURLMSG_DONE:
 836                   if ( (msg->data.result != CURLE_OK) &&
 837                        (msg->data.result != CURLE_GOT_NOTHING) )
 838                     GNUNET_log(GNUNET_ERROR_TYPE_INFO,
 839                                _("%s failed for `%s' at %s:%d: `%s'\n"),
 840                                "curl_multi_perform",
 841                                current_url,
 842                                __FILE__,
 843                                __LINE__,
 844                                curl_easy_strerror (msg->data.result));
 845                   else
 846                     {
 847                     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 848                                 _("Download of hostlist `%s' completed.\n"),
 849                                 current_url);
 850                     download_successful = GNUNET_YES;
 851                     update_hostlist();
 852                     if (GNUNET_YES == testing_hostlist)
 853                      {
 854                       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 855                                         _("Adding successfully tested hostlist `%s' datastore.\n"),current_url);
 856                       insert_hostlist();
 857                       hostlist_to_test = NULL;
 858                       testing_hostlist = GNUNET_NO;
 859                      }
 860                     }
 861                   clean_up ();
 862                   return;
 863                 default:
 864                   break;
 865                 }
 866
 867             }
 868           while (running > 0);
 869         }
 870     }
 871   while (mret == CURLM_CALL_MULTI_PERFORM);
 872   if (mret != CURLM_OK)
 873     {
 874       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
 875                   _("%s failed at %s:%d: `%s'\n"),
 876                   "curl_multi_perform", __FILE__, __LINE__,
 877                   curl_multi_strerror (mret));
 878       clean_up ();
 879     }
 880   run_multi ();
 881 }
 882
 883
 884 /**
 885  * Main function that will download a hostlist and process its
 886  * data.
 887  */
 888 static void
 889 download_hostlist ()
 890 {
 891   CURLcode ret;
 892   CURLMcode mret;
 893
 894
 895   current_url = get_list_url ();
 896   if (current_url == NULL)
 897     return;
 898   curl = curl_easy_init ();
 899   multi = NULL;
 900   if (curl == NULL)
 901     {
 902       GNUNET_break (0);
 903       clean_up ();
 904       return;
 905     }
 906   GNUNET_log (GNUNET_ERROR_TYPE_INFO | GNUNET_ERROR_TYPE_BULK,
 907               _("Bootstrapping using hostlist at `%s'.\n"),
 908               current_url);
 909
 910   download_in_progress = GNUNET_YES;
 911   download_successful = GNUNET_NO;
 912   hellos_obtained = 0;
 913
 914   GNUNET_STATISTICS_update (stats,
 915                             gettext_noop ("# hostlist downloads initiated"),
 916                             1,
 917                             GNUNET_NO);
 918   if (proxy != NULL)
 919     CURL_EASY_SETOPT (curl, CURLOPT_PROXY, proxy);
 920   download_pos = 0;
 921   bogus_url = 0;
 922   CURL_EASY_SETOPT (curl,
 923                     CURLOPT_WRITEFUNCTION,
 924                     &download_hostlist_processor);
 925   if (ret != CURLE_OK)
 926     {
 927       clean_up ();
 928       return;
 929     }
 930   CURL_EASY_SETOPT (curl,
 931                     CURLOPT_WRITEDATA,
 932                     NULL);
 933   if (ret != CURLE_OK)
 934     {
 935       clean_up ();
 936       return;
 937     }
 938   CURL_EASY_SETOPT (curl, CURLOPT_FOLLOWLOCATION, 1);
 939   CURL_EASY_SETOPT (curl, CURLOPT_MAXREDIRS, 4);
 940   /* no need to abort if the above failed */
 941   CURL_EASY_SETOPT (curl,
 942                     CURLOPT_URL,
 943                     current_url);
 944   if (ret != CURLE_OK)
 945     {
 946       clean_up ();
 947       return;
 948     }
 949   CURL_EASY_SETOPT (curl,
 950                     CURLOPT_FAILONERROR,
 951                     1);
 952 #if 0
 953   CURL_EASY_SETOPT (curl,
 954                     CURLOPT_VERBOSE,
 955                     1);
 956 #endif
 957   CURL_EASY_SETOPT (curl,
 958                     CURLOPT_BUFFERSIZE,
 959                     GNUNET_SERVER_MAX_MESSAGE_SIZE);
 960   if (0 == strncmp (current_url, "http", 4))
 961     CURL_EASY_SETOPT (curl, CURLOPT_USERAGENT, "GNUnet");
 962   CURL_EASY_SETOPT (curl,
 963                     CURLOPT_CONNECTTIMEOUT,
 964                     60L);
 965   CURL_EASY_SETOPT (curl,
 966                     CURLOPT_TIMEOUT,
 967                     60L);
 968 #if 0
 969   /* this should no longer be needed; we're now single-threaded! */
 970   CURL_EASY_SETOPT (curl,
 971                     CURLOPT_NOSIGNAL,
 972                     1);
 973 #endif
 974   multi = curl_multi_init ();
 975   if (multi == NULL)
 976     {
 977       GNUNET_break (0);
 978       clean_up ();
 979       return;
 980     }
 981   mret = curl_multi_add_handle (multi, curl);
 982   if (mret != CURLM_OK)
 983     {
 984       GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 985                   _("%s failed at %s:%d: `%s'\n"),
 986                   "curl_multi_add_handle", __FILE__, __LINE__,
 987                   curl_multi_strerror (mret));
 988       mret = curl_multi_cleanup (multi);
 989       if (mret != CURLM_OK)
 990         GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
 991                     _("%s failed at %s:%d: `%s'\n"),
 992                     "curl_multi_cleanup", __FILE__, __LINE__,
 993                     curl_multi_strerror (mret));
 994       multi = NULL;
 995       clean_up ();
 996       return;
 997     }
 998   end_time = GNUNET_TIME_relative_to_absolute (GNUNET_TIME_UNIT_MINUTES);
 999   run_multi ();
1000 }
1001
1002
1003 static void
1004 download_dispatcher (void *cls,
1005             const struct GNUNET_SCHEDULER_TaskContext *tc)
1006 {
1007   ti_download_dispatcher_task = GNUNET_SCHEDULER_NO_TASK;
1008     if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1009       return;
1010    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1011              "Download is initiated...\n");
1012    if ( GNUNET_NO == download_in_progress )
1013    {
1014      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1015                "Download can start immediately...\n");
1016      download_hostlist();
1017    }
1018    else
1019    {
1020      GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1021                "Download in progess, have to wait...\n");
1022      ti_download_dispatcher_task = GNUNET_SCHEDULER_add_delayed (sched,
1023                                                               WAITING_INTERVALL,
1024                                                               &download_dispatcher,
1025                                                               NULL);
1026    }
1027 }
1028
1029 /**
1030  * Task that checks if we should try to download a hostlist.
1031  * If so, we initiate the download, otherwise we schedule
1032  * this task again for a later time.
1033  */
1034 static void
1035 check_task (void *cls,
1036             const struct GNUNET_SCHEDULER_TaskContext *tc)
1037 {
1038   ti_check_download = GNUNET_SCHEDULER_NO_TASK;
1039   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1040     return;
1041
1042   if (connection_count < MIN_CONNECTIONS)
1043   {
1044     ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now ( sched,
1045                                                           &download_dispatcher,
1046                                                           NULL);
1047   }
1048
1049   static int once;
1050   struct GNUNET_TIME_Relative delay;
1051
1052   if (stats == NULL)
1053     {
1054       curl_global_cleanup ();
1055       return; /* in shutdown */
1056     }
1057   delay = hostlist_delay;
1058   if (hostlist_delay.value == 0)
1059     hostlist_delay = GNUNET_TIME_UNIT_SECONDS;
1060   else
1061     hostlist_delay = GNUNET_TIME_relative_multiply (hostlist_delay, 2);
1062   if (hostlist_delay.value > GNUNET_TIME_UNIT_HOURS.value * (1 + connection_count))
1063     hostlist_delay = GNUNET_TIME_relative_multiply (GNUNET_TIME_UNIT_HOURS,
1064                                                     (1 + connection_count));
1065   GNUNET_STATISTICS_set (stats,
1066                          gettext_noop("# seconds between hostlist downloads"),
1067                          hostlist_delay.value,
1068                          GNUNET_YES);
1069   if (0 == once)
1070     {
1071       delay = GNUNET_TIME_UNIT_ZERO;
1072       once = 1;
1073     }
1074   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1075               _("Have %u/%u connections.  Will consider downloading hostlist in %llums\n"),
1076               connection_count,
1077               MIN_CONNECTIONS,
1078               (unsigned long long) delay.value);
1079   ti_check_download = GNUNET_SCHEDULER_add_delayed (sched,
1080                                                delay,
1081                                                &check_task,
1082                                                NULL);
1083 }
1084
1085 /**
1086  * This tasks sets hostlist testing to allowed after intervall between to testings is reached
1087  * cls closure
1088  * tc TaskContext
1089  */
1090 static void
1091 task_testing_intervall_reset (void *cls,
1092             const struct GNUNET_SCHEDULER_TaskContext *tc)
1093 {
1094   ti_testing_intervall_task = GNUNET_SCHEDULER_NO_TASK;
1095   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1096     return;
1097    testing_allowed = GNUNET_OK;
1098    GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1099              "Testing new hostlist advertisements is allowed again\n");
1100 }
1101
1102
1103 /**
1104  * Task that writes hostlist entries to a file on a regular base
1105  * cls closure
1106  * tc TaskContext
1107  */
1108 static void
1109 task_hostlist_saving (void *cls,
1110             const struct GNUNET_SCHEDULER_TaskContext *tc)
1111 {
1112   ti_saving_task = GNUNET_SCHEDULER_NO_TASK;
1113   if (0 != (tc->reason & GNUNET_SCHEDULER_REASON_SHUTDOWN))
1114     return;
1115   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1116               _("Scheduled saving of hostlists\n"));
1117   save_hostlist_file ( GNUNET_NO );
1118
1119   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1120               _("Hostlists will be saved to file again in %llums\n"),
1121               (unsigned long long) SAVING_INTERVALL.value);
1122   ti_saving_task = GNUNET_SCHEDULER_add_delayed (sched,
1123                                                SAVING_INTERVALL,
1124                                                &task_hostlist_saving,
1125                                                NULL);
1126 }
1127
1128 /**
1129  * Method called whenever a given peer connects.
1130  *
1131  * @param cls closure
1132  * @param peer peer identity this notification is about
1133  * @param latency reported latency of the connection with 'other'
1134  * @param distance reported distance (DV) to 'other'
1135  */
1136 static void
1137 handler_connect (void *cls,
1138                  const struct
1139                  GNUNET_PeerIdentity * peer,
1140                  struct GNUNET_TIME_Relative latency,
1141                  uint32_t distance)
1142 {
1143   connection_count++;
1144   GNUNET_STATISTICS_update (stats,
1145                             gettext_noop ("# active connections"),
1146                             1,
1147                             GNUNET_NO);
1148 }
1149
1150
1151 /**
1152  * Method called whenever a given peer disconnects.
1153  *
1154  * @param cls closure
1155  * @param peer peer identity this notification is about
1156  */
1157 static void
1158 handler_disconnect (void *cls,
1159                     const struct
1160                     GNUNET_PeerIdentity * peer)
1161 {
1162   connection_count--;
1163   GNUNET_STATISTICS_update (stats,
1164                             gettext_noop ("# active connections"),
1165                             -1,
1166                             GNUNET_NO);
1167 }
1168
1169 /**
1170  * Method called whenever an advertisement message arrives.
1171  *
1172  * @param cls closure (always NULL)
1173  * @param peer the peer sending the message
1174  * @param message the actual message
1175  * @param latency latency
1176  * @param distance distance
1177  * @return GNUNET_OK to keep the connection open,
1178  *         GNUNET_SYSERR to close it (signal serious error)
1179  */
1180 static int
1181 handler_advertisement (void *cls,
1182     const struct GNUNET_PeerIdentity * peer,
1183     const struct GNUNET_MessageHeader * message,
1184     struct GNUNET_TIME_Relative latency,
1185     uint32_t distance)
1186 {
1187   size_t size;
1188   size_t uri_size;
1189   const struct GNUNET_MessageHeader * incoming;
1190   const char *uri;
1191   struct Hostlist * hostlist;
1192
1193   GNUNET_assert (ntohs (message->type) == GNUNET_MESSAGE_TYPE_HOSTLIST_ADVERTISEMENT);
1194   size = ntohs (message->size);
1195   if (size <= sizeof(struct GNUNET_MessageHeader))
1196     {
1197       GNUNET_break_op (0);
1198       return GNUNET_SYSERR;
1199     }
1200   incoming = (const struct GNUNET_MessageHeader *) message;
1201   uri = (const char*) &incoming[1];
1202   uri_size = size - sizeof (struct GNUNET_MessageHeader);
1203   if (uri [uri_size - 1] != '\0')
1204     {
1205       GNUNET_break_op (0);
1206       return GNUNET_SYSERR;
1207     }
1208   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1209               "Hostlist client recieved advertisement from `%s' containing URI `%s'\n",
1210               GNUNET_i2s (peer),
1211               uri);
1212   if (GNUNET_NO != linked_list_contains (uri))
1213     {
1214       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1215                 "URI `%s' is already known\n",
1216                 uri);
1217       return GNUNET_OK;
1218     }
1219
1220   if ( GNUNET_NO == testing_allowed )
1221     {
1222       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1223                 "Currently not accepting new advertisements: interval between to advertisements is not reached\n");
1224       return GNUNET_SYSERR;
1225     }
1226   if ( GNUNET_YES == testing_hostlist )
1227     {
1228       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1229                 "Currently not accepting new advertisements: we are already testing a hostlist\n");
1230       return GNUNET_SYSERR;
1231     }
1232
1233   hostlist = GNUNET_malloc (sizeof (struct Hostlist) + uri_size);
1234   hostlist->hostlist_uri = (const char*) &hostlist[1];
1235   memcpy (&hostlist[1], uri, uri_size);
1236   hostlist->time_creation = GNUNET_TIME_absolute_get();
1237   hostlist->time_last_usage = GNUNET_TIME_absolute_get_zero();
1238   hostlist->quality = HOSTLIST_INITIAL;
1239   hostlist_to_test = hostlist;
1240
1241   testing_hostlist = GNUNET_YES;
1242   testing_allowed = GNUNET_NO;
1243   ti_testing_intervall_task = GNUNET_SCHEDULER_add_delayed (sched,
1244                                                          TESTING_INTERVALL,
1245                                                          &task_testing_intervall_reset,
1246                                                          NULL);
1247
1248   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1249             "Testing new hostlist advertisements is locked for the next %u ms\n",
1250             TESTING_INTERVALL);
1251
1252   ti_download_dispatcher_task = GNUNET_SCHEDULER_add_now (sched,
1253                                                      &download_dispatcher,
1254                                                      NULL);
1255
1256   return GNUNET_OK;
1257 }
1258
1259
1260
1261 /**
1262  * Continuation called by the statistics code once
1263  * we go the stat.  Initiates hostlist download scheduling.
1264  *
1265  * @param cls closure
1266  * @param success GNUNET_OK if statistics were
1267  *        successfully obtained, GNUNET_SYSERR if not.
1268  */
1269 static void
1270 primary_task (void *cls, int success)
1271 {
1272   if (stats == NULL)
1273     return; /* in shutdown */
1274 #if DEBUG_HOSTLIST_CLIENT
1275   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1276               "Statistics request done, scheduling hostlist download\n");
1277 #endif
1278   ti_check_download = GNUNET_SCHEDULER_add_now (sched,
1279                                            &check_task,
1280                                            NULL);
1281 }
1282
1283
1284 static int
1285 process_stat (void *cls,
1286               const char *subsystem,
1287               const char *name,
1288               uint64_t value,
1289               int is_persistent)
1290 {
1291   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1292               _("Initial time between hostlist downloads is %llums\n"),
1293               (unsigned long long) value);
1294   hostlist_delay.value = value;
1295   return GNUNET_OK;
1296 }
1297
1298 /**
1299  * Method to load persistent hostlist file during hostlist client startup
1300  */
1301 static void
1302 load_hostlist_file ()
1303 {
1304   char *filename;
1305   char *uri;
1306   char *emsg;
1307   struct Hostlist * hostlist;
1308   uri = NULL;
1309   uint32_t times_used;
1310   uint32_t hellos_returned;
1311   uint64_t quality;
1312   uint64_t last_used;
1313   uint64_t created;
1314   uint32_t counter;
1315
1316   if (GNUNET_OK !=
1317       GNUNET_CONFIGURATION_get_value_string (cfg,
1318                                              "HOSTLIST",
1319                                              "HOSTLISTFILE",
1320                                              &filename))
1321     {
1322       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1323                   _("No `%s' specified in `%s' configuration, cannot load hostlists from file.\n"),
1324                   "HOSTLISTFILE", "HOSTLIST");
1325       return;
1326     }
1327
1328   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1329               _("Loading saved hostlist entries from file `%s' \n"), filename);
1330   if ( GNUNET_NO == GNUNET_DISK_file_test (filename) )
1331   {
1332     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1333                 _("Hostlist file `%s' is not existing\n"), filename);
1334     GNUNET_free ( filename );
1335     return;
1336   }
1337
1338   struct GNUNET_BIO_ReadHandle * rh = GNUNET_BIO_read_open (filename);
1339   if (NULL == rh)
1340     {
1341       GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1342                   _("Could not open file `%s' for reading to load hostlists: %s\n"),
1343                   filename,
1344                   STRERROR (errno));
1345       GNUNET_free (filename);
1346       return;
1347     }
1348
1349   counter = 0;
1350   while ( (GNUNET_OK == GNUNET_BIO_read_string (rh, "url" , &uri, MAX_URL_LEN)) &&
1351           (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &times_used)) &&
1352           (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &quality)) &&
1353           (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &last_used)) &&
1354           (GNUNET_OK == GNUNET_BIO_read_int64 (rh, &created)) &&
1355           (GNUNET_OK == GNUNET_BIO_read_int32 (rh, &hellos_returned)) )
1356     {
1357       hostlist = GNUNET_malloc (sizeof (struct Hostlist) + strlen (uri) + 1);
1358       hostlist->hello_count = hellos_returned;
1359       hostlist->hostlist_uri = (const char *) &hostlist[1];
1360       memcpy (&hostlist[1], uri, strlen(uri)+1);
1361       hostlist->quality = quality;
1362       hostlist->time_creation.value = created;
1363       hostlist->time_last_usage.value = last_used;
1364       GNUNET_CONTAINER_DLL_insert(linked_list_head, linked_list_tail, hostlist);
1365       linked_list_size++;
1366       GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1367                   "Added hostlist entry eith URI `%s' \n", hostlist->hostlist_uri);
1368       GNUNET_free (uri);
1369       uri = NULL;
1370       counter++;
1371       if ( counter >= MAX_NUMBER_HOSTLISTS ) break;
1372     }
1373
1374   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1375               _("%u hostlist URIs loaded from file\n"), counter);
1376   GNUNET_STATISTICS_set (stats,
1377                          gettext_noop("# hostlist URIs read from file"),
1378                          counter,
1379                          GNUNET_YES);
1380   GNUNET_STATISTICS_set (stats,
1381                          gettext_noop("# advertised hostlist URIs"),
1382                          linked_list_size,
1383                          GNUNET_NO);
1384
1385   GNUNET_free_non_null (uri);
1386   emsg = NULL;
1387   GNUNET_BIO_read_close (rh, &emsg);
1388   if (emsg != NULL)
1389     GNUNET_free (emsg);
1390   GNUNET_free (filename);
1391 }
1392
1393
1394 /**
1395  * Method to save persistent hostlist file during hostlist client shutdown
1396  * @param shutdown set if called because of shutdown, entries in linked list will be destroyed
1397  */
1398 static void save_hostlist_file ( int shutdown )
1399 {
1400   char *filename;
1401   struct Hostlist *pos;
1402   struct GNUNET_BIO_WriteHandle * wh;
1403   int ok;
1404   uint32_t counter;
1405
1406   if (GNUNET_OK !=
1407       GNUNET_CONFIGURATION_get_value_string (cfg,
1408                                              "HOSTLIST",
1409                                              "HOSTLISTFILE",
1410                                              &filename))
1411     {
1412       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1413                   _("No `%s' specified in `%s' configuration, cannot save hostlists to file.\n"),
1414                   "HOSTLISTFILE", "HOSTLIST");
1415       return;
1416     }
1417   wh = GNUNET_BIO_write_open (filename);
1418   if ( NULL == wh)
1419     {
1420       GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1421                   _("Could not open file `%s' for writing to save hostlists: %s\n"),
1422                   filename,
1423                   STRERROR (errno));
1424       GNUNET_free (filename);
1425       return;
1426     }
1427   GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1428               _("Writing %u hostlist URIs to `%s'\n" ),
1429               linked_list_size, filename);
1430
1431   /* add code to write hostlists to file using bio */
1432   ok = GNUNET_YES;
1433   counter = 0;
1434   while (NULL != (pos = linked_list_head))
1435     {
1436       if ( GNUNET_YES == shutdown)
1437       {
1438         GNUNET_CONTAINER_DLL_remove (linked_list_head, linked_list_tail, pos);
1439         linked_list_size--;
1440       }
1441       if (GNUNET_YES == ok)
1442         {
1443           if ( (GNUNET_OK !=
1444                 GNUNET_BIO_write_string (wh, pos->hostlist_uri)) ||
1445                (GNUNET_OK !=
1446                 GNUNET_BIO_write_int32 (wh, pos->times_used)) ||
1447                (GNUNET_OK !=
1448                 GNUNET_BIO_write_int64 (wh, pos->quality)) ||
1449                (GNUNET_OK !=
1450                 GNUNET_BIO_write_int64 (wh, pos->time_last_usage.value)) ||
1451                (GNUNET_OK !=
1452                 GNUNET_BIO_write_int64 (wh, pos->time_creation.value)) ||
1453                (GNUNET_OK !=
1454                 GNUNET_BIO_write_int32 (wh, pos->hello_count)))
1455             {
1456               GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1457                           _("Error writing hostlist URIs to file `%s'\n"),
1458                           filename);
1459               ok = GNUNET_NO;
1460             }
1461         }
1462
1463       if ( GNUNET_YES == shutdown)
1464         GNUNET_free (pos);
1465       counter ++;
1466       if ( counter >= MAX_NUMBER_HOSTLISTS) break;
1467     }
1468   GNUNET_STATISTICS_set (stats,
1469                          gettext_noop("# hostlist URIs written to file"),
1470                          counter,
1471                          GNUNET_YES);
1472
1473   if ( GNUNET_OK != GNUNET_BIO_write_close ( wh ) )
1474     GNUNET_log (GNUNET_ERROR_TYPE_WARNING,
1475                 _("Error writing hostlist URIs to file `%s'\n"),
1476                 filename);
1477   GNUNET_free (filename);
1478 }
1479
1480 /**
1481  * Start downloading hostlists from hostlist servers as necessary.
1482  */
1483 int
1484 GNUNET_HOSTLIST_client_start (const struct GNUNET_CONFIGURATION_Handle *c,
1485                               struct GNUNET_SCHEDULER_Handle *s,
1486                               struct GNUNET_STATISTICS_Handle *st,
1487                               GNUNET_CORE_ConnectEventHandler *ch,
1488                               GNUNET_CORE_DisconnectEventHandler *dh,
1489                               GNUNET_CORE_MessageCallback *msgh,
1490                               int learn)
1491 {
1492   char *filename;
1493   int result;
1494
1495   if (0 != curl_global_init (CURL_GLOBAL_WIN32))
1496     {
1497       GNUNET_break (0);
1498       return GNUNET_SYSERR;
1499     }
1500   transport = GNUNET_TRANSPORT_connect (s, c, NULL, NULL, NULL, NULL);
1501   if (NULL == transport)
1502     {
1503       curl_global_cleanup ();
1504       return GNUNET_SYSERR;
1505     }
1506   cfg = c;
1507   sched = s;
1508   stats = st;
1509   if (GNUNET_OK !=
1510       GNUNET_CONFIGURATION_get_value_string (cfg,
1511                                              "HOSTLIST",
1512                                              "HTTP-PROXY",
1513                                              &proxy))
1514     proxy = NULL;
1515   learning = learn;
1516   *ch = &handler_connect;
1517   *dh = &handler_disconnect;
1518   linked_list_head = NULL;
1519   linked_list_tail = NULL;
1520   use_preconfigured_list = GNUNET_YES;
1521   testing_hostlist = GNUNET_NO;
1522   testing_allowed = GNUNET_YES;
1523
1524   if ( GNUNET_YES == learning )
1525   {
1526     *msgh = &handler_advertisement;
1527     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1528               _("Learning is enabled on this peer\n"));
1529     load_hostlist_file ();
1530     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1531               _("Hostlists will be saved to file again in  %llums\n"),
1532               (unsigned long long) SAVING_INTERVALL.value);
1533     ti_saving_task = GNUNET_SCHEDULER_add_delayed (sched,
1534                                                SAVING_INTERVALL,
1535                                                &task_hostlist_saving,
1536                                                NULL);
1537   }
1538   else
1539   {
1540     GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1541               _("Learning is not enabled on this peer\n"));
1542     *msgh = NULL;
1543     if (GNUNET_OK == GNUNET_CONFIGURATION_get_value_string (cfg,
1544                                                             "HOSTLIST",
1545                                                             "HOSTLISTFILE",
1546                                                             &filename))
1547     {
1548     if ( GNUNET_YES == GNUNET_DISK_file_test (filename) )
1549       {
1550         result = remove (filename);
1551         if (result == 0)
1552         GNUNET_log (GNUNET_ERROR_TYPE_INFO,
1553               _("Since learning is not enabled on this peer, hostlist file `%s' was removed\n"),filename);
1554         else
1555           GNUNET_log (GNUNET_ERROR_TYPE_ERROR,
1556                 _("Hostlist file `%s' could not be removed\n"),filename);
1557       }
1558     }
1559     GNUNET_free ( filename );
1560   }
1561   GNUNET_STATISTICS_get (stats,
1562                          "hostlist",
1563                          gettext_noop("# seconds between hostlist downloads"),
1564                          GNUNET_TIME_UNIT_MINUTES,
1565                          &primary_task,
1566                          &process_stat,
1567                          NULL);
1568   return GNUNET_OK;
1569 }
1570
1571
1572 /**
1573  * Stop downloading hostlists from hostlist servers as necessary.
1574  */
1575 void
1576 GNUNET_HOSTLIST_client_stop ()
1577 {
1578 #if DEBUG_HOSTLIST_CLIENT
1579   GNUNET_log (GNUNET_ERROR_TYPE_DEBUG,
1580               "Hostlist client shutdown\n");
1581 #endif
1582   if ( GNUNET_YES == learning )
1583     save_hostlist_file ( GNUNET_YES );
1584
1585   if (ti_saving_task != GNUNET_SCHEDULER_NO_TASK)
1586     {
1587       GNUNET_SCHEDULER_cancel (sched,
1588           ti_saving_task);
1589     }
1590
1591   if (ti_download_dispatcher_task != GNUNET_SCHEDULER_NO_TASK)
1592     {
1593       GNUNET_SCHEDULER_cancel (sched,
1594           ti_download_dispatcher_task);
1595     }
1596   if (ti_testing_intervall_task != GNUNET_SCHEDULER_NO_TASK)
1597     {
1598       GNUNET_SCHEDULER_cancel (sched,
1599           ti_testing_intervall_task);
1600     }
1601   if (ti_download != GNUNET_SCHEDULER_NO_TASK)
1602     {
1603       GNUNET_SCHEDULER_cancel (sched,
1604                                ti_download);
1605     }
1606   if (ti_check_download != GNUNET_SCHEDULER_NO_TASK)
1607     {
1608       GNUNET_SCHEDULER_cancel (sched,
1609                                ti_check_download);
1610       curl_global_cleanup ();
1611     }
1612   if (transport != NULL)
1613     {
1614       GNUNET_TRANSPORT_disconnect (transport);
1615       transport = NULL;
1616     }
1617   GNUNET_assert (NULL == transport);
1618   GNUNET_free_non_null (proxy);
1619   proxy = NULL;
1620   cfg = NULL;
1621   sched = NULL;
1622 }
1623
1624 /* end of hostlist-client.c */